diff options
Diffstat (limited to 'net')
135 files changed, 1662 insertions, 862 deletions
diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index ec2569b3f8da..2859cbac3f26 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -502,26 +502,6 @@ static const struct header_ops vlan_header_ops = { .parse_protocol = vlan_parse_protocol, }; -static int vlan_passthru_hard_header(struct sk_buff *skb, struct net_device *dev, - unsigned short type, - const void *daddr, const void *saddr, - unsigned int len) -{ - struct vlan_dev_priv *vlan = vlan_dev_priv(dev); - struct net_device *real_dev = vlan->real_dev; - - if (saddr == NULL) - saddr = dev->dev_addr; - - return dev_hard_header(skb, real_dev, type, daddr, saddr, len); -} - -static const struct header_ops vlan_passthru_header_ops = { - .create = vlan_passthru_hard_header, - .parse = eth_header_parse, - .parse_protocol = vlan_parse_protocol, -}; - static const struct device_type vlan_type = { .name = "vlan", }; @@ -580,14 +560,10 @@ static int vlan_dev_init(struct net_device *dev) dev->fcoe_ddp_xid = real_dev->fcoe_ddp_xid; #endif - dev->needed_headroom = real_dev->needed_headroom; - if (vlan_hw_offload_capable(real_dev->features, vlan->vlan_proto)) { - dev->header_ops = &vlan_passthru_header_ops; - dev->hard_header_len = real_dev->hard_header_len; - } else { - dev->header_ops = &vlan_header_ops; - dev->hard_header_len = real_dev->hard_header_len + VLAN_HLEN; - } + dev->needed_headroom = real_dev->needed_headroom + VLAN_HLEN; + dev->needed_tailroom = real_dev->needed_tailroom; + dev->header_ops = &vlan_header_ops; + dev->hard_header_len = real_dev->hard_header_len; dev->netdev_ops = &vlan_netdev_ops; @@ -1029,10 +1005,9 @@ static void vlan_transfer_features(struct net_device *dev, netif_inherit_tso_max(vlandev, dev); - if (vlan_hw_offload_capable(dev->features, vlan->vlan_proto)) - vlandev->hard_header_len = dev->hard_header_len; - else - vlandev->hard_header_len = dev->hard_header_len + VLAN_HLEN; + vlandev->needed_headroom = dev->needed_headroom + VLAN_HLEN; + vlandev->needed_tailroom = dev->needed_tailroom; + vlandev->hard_header_len = dev->hard_header_len; #if IS_ENABLED(CONFIG_FCOE) vlandev->fcoe_ddp_xid = dev->fcoe_ddp_xid; diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c index 94e074235e15..f635da4b8ca6 100644 --- a/net/batman-adv/bridge_loop_avoidance.c +++ b/net/batman-adv/bridge_loop_avoidance.c @@ -325,11 +325,6 @@ batadv_bla_del_backbone_claims(struct batadv_bla_backbone_gw *backbone_gw) } spin_unlock_bh(list_lock); } - - /* all claims gone, initialize CRC */ - spin_lock_bh(&backbone_gw->crc_lock); - backbone_gw->crc = BATADV_BLA_CRC_INIT; - spin_unlock_bh(&backbone_gw->crc_lock); } /** @@ -695,12 +690,14 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, struct batadv_bla_backbone_gw *old_backbone_gw; struct batadv_bla_claim search_claim; struct batadv_bla_claim *claim; - bool remove_crc = false; int hash_added; + u16 claim_crc; + bool changed; ether_addr_copy(search_claim.addr, mac); search_claim.vid = vid; claim = batadv_claim_hash_find(bat_priv, &search_claim); + claim_crc = crc16(0, mac, ETH_ALEN); /* create a new claim entry if it does not exist yet. */ if (!claim) { @@ -732,43 +729,56 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, kfree(claim); return; } + + spin_lock_bh(&backbone_gw->crc_lock); + backbone_gw->crc ^= claim_crc; + spin_unlock_bh(&backbone_gw->crc_lock); + + WRITE_ONCE(backbone_gw->lasttime, jiffies); + + batadv_claim_put(claim); + return; + } + + WRITE_ONCE(claim->lasttime, jiffies); + + /* replace backbone_gw atomically and adjust reference counters */ + spin_lock_bh(&claim->backbone_lock); + if (claim->backbone_gw != backbone_gw) { + changed = true; + + old_backbone_gw = claim->backbone_gw; + kref_get(&backbone_gw->refcount); + claim->backbone_gw = backbone_gw; } else { - WRITE_ONCE(claim->lasttime, jiffies); - if (claim->backbone_gw == backbone_gw) - /* no need to register a new backbone */ - goto claim_free_ref; + old_backbone_gw = NULL; + changed = false; + } + spin_unlock_bh(&claim->backbone_lock); + if (changed) { batadv_dbg(BATADV_DBG_BLA, bat_priv, "%s(): changing ownership for %pM, vid %d to gw %pM\n", __func__, mac, batadv_print_vid(vid), backbone_gw->orig); - remove_crc = true; - } + /* add claim address to new backbone_gw */ + spin_lock_bh(&backbone_gw->crc_lock); + backbone_gw->crc ^= claim_crc; + spin_unlock_bh(&backbone_gw->crc_lock); - /* replace backbone_gw atomically and adjust reference counters */ - spin_lock_bh(&claim->backbone_lock); - old_backbone_gw = claim->backbone_gw; - kref_get(&backbone_gw->refcount); - claim->backbone_gw = backbone_gw; - spin_unlock_bh(&claim->backbone_lock); + WRITE_ONCE(backbone_gw->lasttime, jiffies); + } - if (remove_crc) { + if (old_backbone_gw) { /* remove claim address from old backbone_gw */ spin_lock_bh(&old_backbone_gw->crc_lock); - old_backbone_gw->crc ^= crc16(0, claim->addr, ETH_ALEN); + old_backbone_gw->crc ^= claim_crc; spin_unlock_bh(&old_backbone_gw->crc_lock); - } - batadv_backbone_gw_put(old_backbone_gw); - - /* add claim address to new backbone_gw */ - spin_lock_bh(&backbone_gw->crc_lock); - backbone_gw->crc ^= crc16(0, claim->addr, ETH_ALEN); - spin_unlock_bh(&backbone_gw->crc_lock); - WRITE_ONCE(backbone_gw->lasttime, jiffies); + batadv_backbone_gw_put(old_backbone_gw); + } -claim_free_ref: batadv_claim_put(claim); } @@ -940,26 +950,18 @@ static bool batadv_handle_unclaim(struct batadv_priv *bat_priv, const u8 *backbone_addr, const u8 *claim_addr, unsigned short vid) { - struct batadv_bla_backbone_gw *backbone_gw; - /* unclaim in any case if it is our own */ if (primary_if && batadv_compare_eth(backbone_addr, primary_if->net_dev->dev_addr)) batadv_bla_send_claim(bat_priv, claim_addr, vid, BATADV_CLAIM_TYPE_UNCLAIM); - backbone_gw = batadv_backbone_hash_find(bat_priv, backbone_addr, vid); - - if (!backbone_gw) - return true; - /* this must be an UNCLAIM frame */ batadv_dbg(BATADV_DBG_BLA, bat_priv, "%s(): UNCLAIM %pM on vid %d (sent by %pM)...\n", __func__, - claim_addr, batadv_print_vid(vid), backbone_gw->orig); + claim_addr, batadv_print_vid(vid), backbone_addr); batadv_bla_del_claim(bat_priv, claim_addr, vid); - batadv_backbone_gw_put(backbone_gw); return true; } diff --git a/net/batman-adv/distributed-arp-table.c b/net/batman-adv/distributed-arp-table.c index 0d5a9cb0affe..011cfdc47fab 100644 --- a/net/batman-adv/distributed-arp-table.c +++ b/net/batman-adv/distributed-arp-table.c @@ -316,7 +316,10 @@ static u8 *batadv_arp_hw_src(struct sk_buff *skb, int hdr_size) */ static __be32 batadv_arp_ip_src(struct sk_buff *skb, int hdr_size) { - return *(__force __be32 *)(batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN); + u8 *src = batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN; + __be32 *ip = (__force __be32 *)src; + + return get_unaligned(ip); } /** @@ -341,8 +344,9 @@ static u8 *batadv_arp_hw_dst(struct sk_buff *skb, int hdr_size) static __be32 batadv_arp_ip_dst(struct sk_buff *skb, int hdr_size) { u8 *dst = batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN * 2 + 4; + __be32 *ip = (__force __be32 *)dst; - return *(__force __be32 *)dst; + return get_unaligned(ip); } /** @@ -440,18 +444,19 @@ batadv_dat_entry_hash_find(struct batadv_priv *bat_priv, __be32 ip, static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, u8 *mac_addr, unsigned short vid) { + u64 u64_mac = ether_addr_to_u64(mac_addr); struct batadv_dat_entry *dat_entry; int hash_added; dat_entry = batadv_dat_entry_hash_find(bat_priv, ip, vid); /* if this entry is already known, just update it */ if (dat_entry) { - if (!batadv_compare_eth(dat_entry->mac_addr, mac_addr)) - ether_addr_copy(dat_entry->mac_addr, mac_addr); + atomic64_set(&dat_entry->mac_addr, u64_mac); + dat_entry->last_update = jiffies; batadv_dbg(BATADV_DBG_DAT, bat_priv, "Entry updated: %pI4 %pM (vid: %d)\n", - &dat_entry->ip, dat_entry->mac_addr, + &dat_entry->ip, mac_addr, batadv_print_vid(vid)); goto out; } @@ -462,7 +467,7 @@ static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, dat_entry->ip = ip; dat_entry->vid = vid; - ether_addr_copy(dat_entry->mac_addr, mac_addr); + atomic64_set(&dat_entry->mac_addr, u64_mac); dat_entry->last_update = jiffies; kref_init(&dat_entry->refcount); @@ -478,7 +483,7 @@ static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, } batadv_dbg(BATADV_DBG_DAT, bat_priv, "New entry added: %pI4 %pM (vid: %d)\n", - &dat_entry->ip, dat_entry->mac_addr, batadv_print_vid(vid)); + &dat_entry->ip, mac_addr, batadv_print_vid(vid)); out: batadv_dat_entry_put(dat_entry); @@ -935,6 +940,8 @@ batadv_dat_cache_dump_entry(struct sk_buff *msg, u32 portid, struct netlink_callback *cb, struct batadv_dat_entry *dat_entry) { + u8 mac[ETH_ALEN]; + u64 u64_mac; int msecs; void *hdr; @@ -947,11 +954,12 @@ batadv_dat_cache_dump_entry(struct sk_buff *msg, u32 portid, genl_dump_check_consistent(cb, hdr); msecs = jiffies_to_msecs(jiffies - dat_entry->last_update); + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); if (nla_put_in_addr(msg, BATADV_ATTR_DAT_CACHE_IP4ADDRESS, dat_entry->ip) || - nla_put(msg, BATADV_ATTR_DAT_CACHE_HWADDRESS, ETH_ALEN, - dat_entry->mac_addr) || + nla_put(msg, BATADV_ATTR_DAT_CACHE_HWADDRESS, ETH_ALEN, mac) || nla_put_u16(msg, BATADV_ATTR_DAT_CACHE_VID, dat_entry->vid) || nla_put_u32(msg, BATADV_ATTR_LAST_SEEN_MSECS, msecs)) { genlmsg_cancel(msg, hdr); @@ -1231,10 +1239,12 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, struct sk_buff *skb_new; unsigned short vid; bool ret = false; + u8 mac[ETH_ALEN]; int hdr_size = 0; __be32 ip_dst; __be32 ip_src; u16 type = 0; + u64 u64_mac; u8 *hw_src; if (!READ_ONCE(bat_priv->distributed_arp_table)) @@ -1263,6 +1273,9 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_dst, vid); if (dat_entry) { + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + /* If the ARP request is destined for a local client the local * client will answer itself. DAT would only generate a * duplicate packet. @@ -1271,7 +1284,7 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, * additional DAT answer may trigger kernel warnings about * a packet coming from the wrong port. */ - if (batadv_is_my_client(bat_priv, dat_entry->mac_addr, vid)) { + if (batadv_is_my_client(bat_priv, mac, vid)) { ret = true; goto out; } @@ -1281,18 +1294,16 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, * the backbone gws belonging to our backbone has claimed the * destination. */ - if (!batadv_bla_check_claim(bat_priv, - dat_entry->mac_addr, vid)) { + if (!batadv_bla_check_claim(bat_priv, mac, vid)) { batadv_dbg(BATADV_DBG_DAT, bat_priv, "Device %pM claimed by another backbone gw. Don't send ARP reply!", - dat_entry->mac_addr); + mac); ret = true; goto out; } skb_new = batadv_dat_arp_create_reply(bat_priv, ip_dst, ip_src, - dat_entry->mac_addr, - hw_src, vid); + mac, hw_src, vid); if (!skb_new) goto out; @@ -1336,8 +1347,10 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv, struct sk_buff *skb_new; unsigned short vid; bool ret = false; + u8 mac[ETH_ALEN]; __be32 ip_src; __be32 ip_dst; + u64 u64_mac; u8 *hw_src; u16 type; int err; @@ -1367,8 +1380,11 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv, if (!dat_entry) goto out; + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + skb_new = batadv_dat_arp_create_reply(bat_priv, ip_dst, ip_src, - dat_entry->mac_addr, hw_src, vid); + mac, hw_src, vid); if (!skb_new) goto out; @@ -1460,8 +1476,10 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv, struct batadv_dat_entry *dat_entry = NULL; bool dropped = false; unsigned short vid; + u8 mac[ETH_ALEN]; __be32 ip_src; __be32 ip_dst; + u64 u64_mac; u8 *hw_src; u8 *hw_dst; u16 type; @@ -1493,11 +1511,17 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv, * this frame would lead to doubled receive of an ARP reply. */ dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_src, vid); - if (dat_entry && batadv_compare_eth(hw_src, dat_entry->mac_addr)) { - batadv_dbg(BATADV_DBG_DAT, bat_priv, "Doubled ARP reply removed: ARP MSG = [src: %pM-%pI4 dst: %pM-%pI4]; dat_entry: %pM-%pI4\n", - hw_src, &ip_src, hw_dst, &ip_dst, - dat_entry->mac_addr, &dat_entry->ip); - dropped = true; + if (dat_entry) { + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + + if (batadv_compare_eth(hw_src, mac)) { + batadv_dbg(BATADV_DBG_DAT, bat_priv, + "Doubled ARP reply removed: ARP MSG = [src: %pM-%pI4 dst: %pM-%pI4]; dat_entry: %pM-%pI4\n", + hw_src, &ip_src, hw_dst, &ip_dst, + mac, &dat_entry->ip); + dropped = true; + } } /* Update our internal cache with both the IP addresses the node got diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c index 77597171d637..d89d44706269 100644 --- a/net/batman-adv/main.c +++ b/net/batman-adv/main.c @@ -492,6 +492,10 @@ int batadv_batman_skb_recv(struct sk_buff *skb, struct net_device *dev, if (!skb) goto err_put; + /* Merged fragments re-enter here with reused skb metadata. */ + skb->dev = dev; + skb->skb_iif = dev->ifindex; + /* packet should hold at least type and version */ if (unlikely(!pskb_may_pull(skb, 2))) goto err_free; diff --git a/net/batman-adv/mesh-interface.c b/net/batman-adv/mesh-interface.c index 8e55b61dd2a6..e202088cf1fd 100644 --- a/net/batman-adv/mesh-interface.c +++ b/net/batman-adv/mesh-interface.c @@ -305,6 +305,8 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb, if (batadv_compare_eth(ethhdr->h_dest, ectp_addr)) goto dropped; + batadv_skb_set_priority(skb, 0); + gw_mode = READ_ONCE(bat_priv->gw.mode); if (is_multicast_ether_addr(ethhdr->h_dest)) { /* if gw mode is off, broadcast every packet */ @@ -338,6 +340,9 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb, send: if (do_bcast && !is_broadcast_ether_addr(ethhdr->h_dest)) { + /* WARNING batadv_mcast_forw_mode might add more headers + * in front of the skb. and might even reallocate the skb + */ forw_mode = batadv_mcast_forw_mode(bat_priv, skb, vid, &mcast_is_routable); switch (forw_mode) { @@ -355,8 +360,6 @@ send: } } - batadv_skb_set_priority(skb, 0); - /* ethernet packet should be broadcasted */ if (do_bcast) { primary_if = batadv_primary_if_get_selected(bat_priv); diff --git a/net/batman-adv/multicast_forw.c b/net/batman-adv/multicast_forw.c index bae2a8110976..ac97cd8b81dc 100644 --- a/net/batman-adv/multicast_forw.c +++ b/net/batman-adv/multicast_forw.c @@ -1105,8 +1105,11 @@ static int batadv_mcast_forw_expand_head(struct batadv_priv *bat_priv, return -EINVAL; } - if (skb_headroom(skb) < hdr_size && - pskb_expand_head(skb, hdr_size, 0, GFP_ATOMIC) < 0) + if (skb_cow(skb, hdr_size) < 0) + return -ENOMEM; + + /* batadv_mcast_forw_scrape() + batadv_mcast_forw_packet() require linearized skb */ + if (skb_linearize(skb) < 0) return -ENOMEM; return 0; diff --git a/net/batman-adv/tvlv.c b/net/batman-adv/tvlv.c index de907c07fa15..93492d393c34 100644 --- a/net/batman-adv/tvlv.c +++ b/net/batman-adv/tvlv.c @@ -438,8 +438,11 @@ static int batadv_tvlv_call_handler(struct batadv_priv *bat_priv, return NET_RX_SUCCESS; tvlv_offset = (unsigned char *)tvlv_value - skb->data; + if (!skb_set_transport_header_careful(skb, + tvlv_offset + tvlv_value_len)) + return -EINVAL; + skb_set_network_header(skb, tvlv_offset); - skb_set_transport_header(skb, tvlv_offset + tvlv_value_len); return tvlv_handler->mcast_handler(bat_priv, skb); } diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h index 42b631573512..9bdc5a3e799e 100644 --- a/net/batman-adv/types.h +++ b/net/batman-adv/types.h @@ -2176,7 +2176,7 @@ struct batadv_dat_entry { __be32 ip; /** @mac_addr: the MAC address associated to the stored IPv4 */ - u8 mac_addr[ETH_ALEN]; + atomic64_t mac_addr; /** @vid: the vlan ID associated to this entry */ unsigned short vid; diff --git a/net/bluetooth/eir.c b/net/bluetooth/eir.c index 1de5f9df6eec..a55696820b22 100644 --- a/net/bluetooth/eir.c +++ b/net/bluetooth/eir.c @@ -369,6 +369,7 @@ u8 eir_create_scan_rsp(struct hci_dev *hdev, u8 instance, u8 *ptr) void *eir_get_service_data(u8 *eir, size_t eir_len, u16 uuid, size_t *len) { + const u8 *eir_end = eir + eir_len; size_t dlen; while ((eir = eir_get_data(eir, eir_len, EIR_SERVICE_DATA, &dlen))) { @@ -381,7 +382,7 @@ void *eir_get_service_data(u8 *eir, size_t eir_len, u16 uuid, size_t *len) } eir += dlen; - eir_len -= dlen; + eir_len = eir_end - eir; } return NULL; diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 19b7629b1cc1..8de98af2fb58 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -1391,7 +1391,8 @@ static void hci_le_conn_failed(struct hci_conn *conn, u8 status) /* Enable advertising in case this was a failed connection * attempt as a peripheral. */ - hci_enable_advertising(hdev); + if (conn->role == HCI_ROLE_SLAVE) + hci_enable_advertising(hdev); } /* This function requires the caller holds hdev->lock */ diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index 509c820a693d..35a1be57e386 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -4093,7 +4093,7 @@ static int hci_send_cmd_sync(struct hci_dev *hdev, struct sk_buff *skb) if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND && !hci_dev_test_and_set_flag(hdev, HCI_CMD_PENDING)) { kfree_skb(hdev->req_skb); - hdev->req_skb = skb_clone(hdev->sent_cmd, GFP_KERNEL); + hdev->req_skb = skb_get(hdev->sent_cmd); } return err; diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 3eb1eaf6e6a0..2f5e21ff9752 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -5763,10 +5763,11 @@ static void le_conn_complete_evt(struct hci_dev *hdev, u8 status, hci_dev_lock(hdev); hci_store_wake_reason(hdev, bdaddr, bdaddr_type); - /* All controllers implicitly stop advertising in the event of a - * connection, so ensure that the state bit is cleared. + /* Advertising stops when a connection is created. On a failed + * connection it keeps running, so leave the state bit alone. */ - hci_dev_clear_flag(hdev, HCI_LE_ADV); + if (!status) + hci_dev_clear_flag(hdev, HCI_LE_ADV); /* Check for existing connection: * diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index b5897545d795..ffd7b37e7401 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -1287,6 +1287,7 @@ hci_set_ext_adv_params_sync(struct hci_dev *hdev, u8 instance, } static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { DEFINE_FLEX(struct hci_cp_le_set_ext_adv_data, pdu, data, length, HCI_MAX_EXT_AD_LENGTH); @@ -1375,6 +1376,7 @@ int hci_update_adv_data_sync(struct hci_dev *hdev, u8 instance) } int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { struct hci_cp_le_set_ext_adv_params cp; struct hci_rp_le_set_ext_adv_params rp; @@ -1535,6 +1537,7 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) } static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { DEFINE_FLEX(struct hci_cp_le_set_ext_scan_rsp_data, pdu, data, length, HCI_MAX_EXT_AD_LENGTH); @@ -1588,6 +1591,7 @@ static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) } static int __hci_set_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { struct hci_cp_le_set_scan_rsp_data cp; u8 len; @@ -1729,6 +1733,7 @@ static int hci_set_per_adv_params_sync(struct hci_dev *hdev, u8 instance, } static int hci_set_per_adv_data_sync(struct hci_dev *hdev, u8 instance) + __context_unsafe(/* conditional locking */) { DEFINE_FLEX(struct hci_cp_le_set_per_adv_data, pdu, data, length, HCI_MAX_PER_AD_LENGTH); @@ -5448,6 +5453,7 @@ int hci_dev_open_sync(struct hci_dev *hdev) if (hdev->req_skb) { kfree_skb(hdev->req_skb); hdev->req_skb = NULL; + hci_dev_clear_flag(hdev, HCI_CMD_PENDING); } clear_bit(HCI_RUNNING, &hdev->flags); @@ -5632,6 +5638,7 @@ int hci_dev_close_sync(struct hci_dev *hdev) if (hdev->req_skb) { kfree_skb(hdev->req_skb); hdev->req_skb = NULL; + hci_dev_clear_flag(hdev, HCI_CMD_PENDING); } clear_bit(HCI_RUNNING, &hdev->flags); @@ -7282,8 +7289,13 @@ static void create_le_conn_complete(struct hci_dev *hdev, void *data, int err) goto unlock; } - /* Check if connection is still pending */ - if (conn != hci_lookup_le_connect(hdev)) + /* Check if this connection is still pending. + * + * hci_lookup_le_connect() returns only the first LE connection + * in BT_CONNECT, which is not necessarily this one when two are + * pending at once, so ask the connection itself. + */ + if (conn->state != BT_CONNECT) goto unlock; /* Flush to make sure we send create conn cancel command if needed */ diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index aa2ce78f56a2..75bfd5938b2e 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -2277,6 +2277,14 @@ static void iso_conn_ready(struct iso_conn *conn) lock_sock(parent); + /* The listener may have been closed concurrently. */ + if (parent->sk_state != BT_LISTEN || + sock_flag(parent, SOCK_ZAPPED)) { + release_sock(parent); + sock_put(parent); + return; + } + sk = iso_sock_alloc(sock_net(parent), NULL, BTPROTO_ISO, GFP_ATOMIC, 0); if (!sk) { diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 735167f73f31..1194c37e466f 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -436,11 +436,26 @@ static int l2cap_get_mode(struct l2cap_chan *chan) return -EINVAL; } +static struct l2cap_conn *l2cap_chan_conn(struct l2cap_chan *chan) +{ + lockdep_assert_held(&chan->lock); + + /* l2cap_conn_del() sets FLAG_DEL while holding chan->lock before + * conn->hcon is deleted. If not set and conn is non-NULL, conn->hcon + * remains alive during this chan->lock critical section. + */ + if (test_bit(FLAG_DEL, &chan->flags)) + return NULL; + + return chan->conn; +} + static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, sockopt_t *sopt) { struct sock *sk = sock->sk; struct l2cap_chan *chan = l2cap_pi(sk)->chan; + struct l2cap_conn *conn; struct l2cap_options opts; struct l2cap_conninfo cinfo; int err = 0; @@ -451,6 +466,7 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, len = sopt->optlen; + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -537,9 +553,15 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, break; } + conn = l2cap_chan_conn(chan); + if (!conn) { + err = -ENOTCONN; + break; + } + memset(&cinfo, 0, sizeof(cinfo)); - cinfo.hci_handle = chan->conn->hcon->handle; - memcpy(cinfo.dev_class, chan->conn->hcon->dev_class, 3); + cinfo.hci_handle = conn->hcon->handle; + memcpy(cinfo.dev_class, conn->hcon->dev_class, 3); len = min(len, sizeof(cinfo)); if (copy_to_iter(&cinfo, len, &sopt->iter_out) != len) @@ -553,6 +575,8 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); + return err; } @@ -561,6 +585,7 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, { struct sock *sk = sock->sk; struct l2cap_chan *chan = l2cap_pi(sk)->chan; + struct l2cap_conn *conn; struct bt_security sec; struct bt_power pwr; int len, mode, err = 0; @@ -578,6 +603,7 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, len = sopt->optlen; + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -589,12 +615,14 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, break; } + conn = l2cap_chan_conn(chan); + memset(&sec, 0, sizeof(sec)); - if (chan->conn) { - sec.level = chan->conn->hcon->sec_level; + if (conn) { + sec.level = conn->hcon->sec_level; if (sk->sk_state == BT_CONNECTED) - sec.key_size = chan->conn->hcon->enc_key_size; + sec.key_size = conn->hcon->enc_key_size; } else { sec.level = chan->sec_level; } @@ -678,12 +706,14 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, break; case BT_PHY: - if (sk->sk_state != BT_CONNECTED) { + conn = l2cap_chan_conn(chan); + + if (sk->sk_state != BT_CONNECTED || !conn) { err = -ENOTCONN; break; } - opt = hci_conn_get_phy(chan->conn->hcon); + opt = hci_conn_get_phy(conn->hcon); if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) != sizeof(opt)) @@ -719,6 +749,7 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); return err; } @@ -749,6 +780,7 @@ static int l2cap_sock_setsockopt_old(struct socket *sock, int optname, BT_DBG("sk %p", sk); + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -850,6 +882,7 @@ static int l2cap_sock_setsockopt_old(struct socket *sock, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); return err; } @@ -913,6 +946,7 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, if (level != SOL_BLUETOOTH) return -ENOPROTOOPT; + l2cap_chan_lock(chan); lock_sock(sk); switch (optname) { @@ -938,11 +972,10 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, chan->sec_level = sec.level; - if (!chan->conn) + conn = l2cap_chan_conn(chan); + if (!conn) break; - conn = chan->conn; - /* change security for LE channels */ if (chan->scid == L2CAP_CID_ATT) { if (smp_conn_security(conn->hcon, sec.level)) { @@ -997,7 +1030,8 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, } if (opt == BT_FLUSHABLE_OFF) { - conn = chan->conn; + conn = l2cap_chan_conn(chan); + /* proceed further only when we have l2cap_conn and No Flush support in the LM */ if (!conn || !lmp_no_flush_capable(conn->hcon->hdev)) { @@ -1083,7 +1117,8 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, break; case BT_PHY: - if (sk->sk_state != BT_CONNECTED) { + conn = l2cap_chan_conn(chan); + if (sk->sk_state != BT_CONNECTED || !conn) { err = -ENOTCONN; break; } @@ -1093,10 +1128,6 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, if (err) break; - if (!chan->conn) - break; - - conn = chan->conn; err = hci_conn_set_phy(conn->hcon, phys); break; @@ -1139,6 +1170,7 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, } release_sock(sk); + l2cap_chan_unlock(chan); return err; } @@ -1312,7 +1344,12 @@ static void l2cap_sock_kill(struct sock *sk) BT_DBG("sk %p state %s", sk, state_to_string(sk->sk_state)); + /* Take lock to synchronize against access without owning sk->sk_socket, + * eg. in l2cap_sock_cleanup_listen(). proto_ops etc. don't need lock. + */ + lock_sock(sk); l2cap_sock_put_chan(sk); + release_sock(sk); /* Kill poor orphan */ sock_set_flag(sk, SOCK_DEAD); @@ -1516,14 +1553,10 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) * establish sk_lock -> conn->lock and invert the established * conn->lock -> chan->lock -> sk_lock order (lockdep deadlock). * - * Instead, briefly take the child sk lock to fetch and pin its chan. - * l2cap_conn_del() reaches the chan free only via - * l2cap_chan_del() -> l2cap_sock_teardown_cb(), which itself takes - * the child sk lock; holding it across l2cap_chan_hold_unless_zero() - * therefore guarantees the chan cannot be freed while we read and - * pin it (hold_unless_zero() additionally skips a chan already past - * its last reference). We then drop the sk lock before taking - * chan->lock, so sk and chan locks are never held together. + * Instead, briefly take the child sk lock to synchronize vs. + * l2cap_sock_kill that puts l2cap_pi(sk)->chan. We then drop the sk + * lock before taking chan->lock, so sk and chan locks are never held + * together. * * Since we cannot call l2cap_chan_close() without conn->lock, * schedule l2cap_chan_timeout to close the channel; it already @@ -1533,10 +1566,12 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) struct l2cap_chan *chan; lock_sock_nested(sk, L2CAP_NESTING_NORMAL); - chan = l2cap_chan_hold_unless_zero(l2cap_pi(sk)->chan); + chan = l2cap_pi(sk)->chan; + if (chan) + l2cap_chan_hold(chan); release_sock(sk); if (!chan) { - /* l2cap_conn_del() already tearing this child down */ + /* Already torn down */ sock_put(sk); continue; } @@ -1568,6 +1603,11 @@ static int l2cap_sock_new_connection_cb(struct l2cap_chan *chan, lock_sock(parent); + if (parent->sk_state != BT_LISTEN) { + release_sock(parent); + return -EINVAL; + } + /* Check for backlog size */ if (sk_acceptq_is_full(parent)) { BT_DBG("backlog full %d", parent->sk_ack_backlog); @@ -1731,10 +1771,14 @@ static void l2cap_sock_state_change_cb(struct l2cap_chan *chan, int state, if (!sk) return; + lock_sock(sk); + sk->sk_state = state; if (err) sk->sk_err = err; + + release_sock(sk); } static struct sk_buff *l2cap_sock_alloc_skb_cb(struct l2cap_chan *chan, @@ -1810,6 +1854,8 @@ static void l2cap_sock_resume_cb(struct l2cap_chan *chan) if (!sk) return; + lock_sock(sk); + if (test_and_clear_bit(FLAG_PENDING_SECURITY, &chan->flags)) { sk->sk_state = BT_CONNECTED; chan->state = BT_CONNECTED; @@ -1817,6 +1863,8 @@ static void l2cap_sock_resume_cb(struct l2cap_chan *chan) clear_bit(BT_SK_SUSPEND, &bt_sk(sk)->flags); sk->sk_state_change(sk); + + release_sock(sk); } static void l2cap_sock_set_shutdown_cb(struct l2cap_chan *chan) diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 860c086011b7..ac4864e56ec7 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -6171,6 +6171,7 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, struct mgmt_pending_cmd *cmd; const u16 max_uuid_count = ((U16_MAX - sizeof(*cp)) / 16); u16 uuid_count, expected_len; + u8 (*uuids)[16] = NULL; u8 status; int err; @@ -6247,12 +6248,10 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, hdev->discovery.result_filtering = true; hdev->discovery.type = cp->type; hdev->discovery.rssi = cp->rssi; - hdev->discovery.uuid_count = uuid_count; if (uuid_count > 0) { - hdev->discovery.uuids = kmemdup(cp->uuids, uuid_count * 16, - GFP_KERNEL); - if (!hdev->discovery.uuids) { + uuids = kmemdup(cp->uuids, uuid_count * sizeof(*uuids), GFP_KERNEL); + if (!uuids) { err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_START_SERVICE_DISCOVERY, MGMT_STATUS_FAILED, @@ -6262,6 +6261,11 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, } } + spin_lock(&hdev->discovery.lock); + hdev->discovery.uuids = uuids; + hdev->discovery.uuid_count = uuid_count; + spin_unlock(&hdev->discovery.lock); + err = hci_cmd_sync_queue(hdev, start_discovery_sync, cmd, start_discovery_complete); if (err < 0) { @@ -10505,6 +10509,7 @@ static bool is_filter_match(struct hci_dev *hdev, s8 rssi, u8 *eir, !hci_test_quirk(hdev, HCI_QUIRK_STRICT_DUPLICATE_FILTER)))) return false; + spin_lock(&hdev->discovery.lock); if (hdev->discovery.uuid_count != 0) { /* If a list of UUIDs is provided in filter, results with no * matching UUID should be dropped. @@ -10513,9 +10518,12 @@ static bool is_filter_match(struct hci_dev *hdev, s8 rssi, u8 *eir, hdev->discovery.uuids) && !eir_has_uuids(scan_rsp, scan_rsp_len, hdev->discovery.uuid_count, - hdev->discovery.uuids)) + hdev->discovery.uuids)) { + spin_unlock(&hdev->discovery.lock); return false; + } } + spin_unlock(&hdev->discovery.lock); /* If duplicate filtering does not report RSSI changes, then restart * scanning to ensure updated result with updated RSSI values. diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c index 9cdfea666a2c..f7463f092283 100644 --- a/net/bluetooth/rfcomm/core.c +++ b/net/bluetooth/rfcomm/core.c @@ -1455,6 +1455,10 @@ static int rfcomm_apply_pn(struct rfcomm_dlc *d, int cr, struct rfcomm_pn *pn) d->mtu = __le16_to_cpu(pn->mtu); + /* MTU 0 causes an infinite loop when fragmenting in sendmsg */ + if (!d->mtu) + d->mtu = RFCOMM_DEFAULT_MTU; + if (cr && d->mtu > s->mtu) d->mtu = s->mtu; @@ -2178,8 +2182,10 @@ static void rfcomm_kill_listener(void) BT_DBG(""); + rfcomm_lock(); list_for_each_entry_safe(s, n, &session_list, list) rfcomm_session_del(s); + rfcomm_unlock(); } static int rfcomm_run(void *unused) @@ -2213,9 +2219,13 @@ static void rfcomm_security_cfm(struct hci_conn *conn, u8 status, u8 encrypt) BT_DBG("conn %p status 0x%02x encrypt 0x%02x", conn, status, encrypt); + rfcomm_lock(); + s = rfcomm_session_get(&conn->hdev->bdaddr, &conn->dst); - if (!s) + if (!s) { + rfcomm_unlock(); return; + } list_for_each_entry_safe(d, n, &s->dlcs, list) { if (test_and_clear_bit(RFCOMM_SEC_PENDING, &d->flags)) { @@ -2247,6 +2257,8 @@ static void rfcomm_security_cfm(struct hci_conn *conn, u8 status, u8 encrypt) set_bit(RFCOMM_AUTH_REJECT, &d->flags); } + rfcomm_unlock(); + rfcomm_schedule(); } diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c index b6e5a86b6a92..87a40e2f1c50 100644 --- a/net/bridge/br_arp_nd_proxy.c +++ b/net/bridge/br_arp_nd_proxy.c @@ -196,13 +196,15 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, n = neigh_lookup(&arp_tbl, &tip, vlandev); if (n) { struct net_bridge_fdb_entry *f; + u8 ha[ETH_ALEN] __aligned(2); if (!(READ_ONCE(n->nud_state) & NUD_VALID)) { neigh_release(n); return; } - f = br_fdb_find_rcu(br, n->ha, vid); + neigh_ha_snapshot(ha, n, n->dev); + f = br_fdb_find_rcu(br, ha, vid); if (f) { const struct net_bridge_port *dst = READ_ONCE(f->dst); bool replied = false; @@ -212,10 +214,10 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, br_is_neigh_suppress_enabled(dst, vid)) { if (!vid) br_arp_send(br, p, skb->dev, sip, tip, - sha, n->ha, sha, 0, 0); + sha, ha, sha, 0, 0); else br_arp_send(br, p, skb->dev, sip, tip, - sha, n->ha, sha, + sha, ha, sha, skb->vlan_proto, skb_vlan_tag_get(skb)); replied = true; @@ -250,7 +252,7 @@ struct nd_msg *br_is_nd_neigh_msg(struct sk_buff *skb) } static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, - struct sk_buff *request, struct neighbour *n, + struct sk_buff *request, struct neighbour *n, u8 *ha, __be16 vlan_proto, u16 vlan_tci) { struct net_device *dev = request->dev; @@ -309,7 +311,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, ipv6_eth_mc_map(&in6addr_linklocal_allnodes, eth_hdr(reply)->h_dest); else ether_addr_copy(eth_hdr(reply)->h_dest, daddr); - ether_addr_copy(eth_hdr(reply)->h_source, n->ha); + ether_addr_copy(eth_hdr(reply)->h_source, ha); eth_hdr(reply)->h_proto = htons(ETH_P_IPV6); reply->protocol = htons(ETH_P_IPV6); @@ -339,7 +341,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, na->icmph.icmp6_override = 1; na->icmph.icmp6_solicited = dad ? 0 : 1; na->target = ns->target; - ether_addr_copy(&na->opt[2], n->ha); + ether_addr_copy(&na->opt[2], ha); na->opt[0] = ND_OPT_TARGET_LL_ADDR; na->opt[1] = na_olen >> 3; @@ -368,7 +370,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, __vlan_hwaccel_put_tag(reply, vlan_proto, vlan_tci); netdev_dbg(dev, "nd send dev %s dst %pI6 dst_hw %pM src %pI6 src_hw %pM\n", - dev->name, &pip6->daddr, daddr, &pip6->saddr, n->ha); + dev->name, &pip6->daddr, daddr, &pip6->saddr, ha); if (p) { dev_queue_xmit(reply); @@ -471,24 +473,26 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, n = neigh_lookup(&nd_tbl, &msg->target, vlandev); if (n) { struct net_bridge_fdb_entry *f; + u8 ha[ETH_ALEN] __aligned(2); if (!(READ_ONCE(n->nud_state) & NUD_VALID)) { neigh_release(n); return; } - f = br_fdb_find_rcu(br, n->ha, vid); + neigh_ha_snapshot(ha, n, n->dev); + f = br_fdb_find_rcu(br, ha, vid); if (f) { const struct net_bridge_port *dst = READ_ONCE(f->dst); bool replied = false; if (br_is_neigh_suppress_enabled(dst, vid)) { if (vid != 0) - br_nd_send(br, p, skb, n, + br_nd_send(br, p, skb, n, ha, skb->vlan_proto, skb_vlan_tag_get(skb)); else - br_nd_send(br, p, skb, n, 0, 0); + br_nd_send(br, p, skb, n, ha, 0, 0); replied = true; } diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index 75e1e2a8fc83..3ef5d8bbf552 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -4377,8 +4377,8 @@ void br_multicast_toggle_one_vlan(struct net_bridge_vlan *vlan, bool on) if (br_vlan_is_master(vlan)) { br = vlan->br; - if (!br_vlan_is_brentry(vlan) || - (on && + if (on && + (!br_vlan_is_brentry(vlan) || br_multicast_ctx_vlan_global_disabled(&vlan->br_mcast_ctx))) return; diff --git a/net/bridge/br_netlink_tunnel.c b/net/bridge/br_netlink_tunnel.c index a713668ea34f..e7eceab5b515 100644 --- a/net/bridge/br_netlink_tunnel.c +++ b/net/bridge/br_netlink_tunnel.c @@ -302,7 +302,8 @@ int br_process_vlan_tunnel_info(const struct net_bridge *br, if (!(tinfo_last->flags & BRIDGE_VLAN_INFO_RANGE_BEGIN)) return -EINVAL; - if ((tinfo_curr->vid - tinfo_last->vid) != + if (tinfo_curr->vid < tinfo_last->vid || + (tinfo_curr->vid - tinfo_last->vid) != (tinfo_curr->tunid - tinfo_last->tunid)) return -EINVAL; t = tinfo_last->tunid; diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index a91d2b58aadd..da5f881fbd3b 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -400,14 +400,8 @@ static __net_init void preinit_net_sysctl(struct net *net) } /* init code that must occur even if setup_net() is not called. */ -static __net_init int preinit_net(struct net *net, struct user_namespace *user_ns) +static __net_init void preinit_net(struct net *net, struct user_namespace *user_ns) { - int ret; - - ret = ns_common_init(net); - if (ret) - return ret; - refcount_set(&net->passive, 1); ref_tracker_dir_init(&net->refcnt_tracker, 128, "net_refcnt"); ref_tracker_dir_init(&net->notrefcnt_tracker, 128, "net_notrefcnt"); @@ -431,7 +425,6 @@ static __net_init int preinit_net(struct net *net, struct user_namespace *user_n INIT_LIST_HEAD(&net->ptype_all); INIT_LIST_HEAD(&net->ptype_specific); preinit_net_sysctl(net); - return 0; } /* @@ -536,8 +529,12 @@ void net_passive_dec(struct net *net) if (refcount_dec_and_test(&net->passive)) { kfree(rcu_access_pointer(net->gen)); +#ifdef CONFIG_REF_TRACKER /* There should not be any trackers left there. */ ref_tracker_dir_exit(&net->notrefcnt_tracker); + if (!net->refcnt_tracker.dead) + ref_tracker_dir_exit(&net->refcnt_tracker); +#endif /* Wait for an extra rcu_barrier() before final free. */ llist_add(&net->defer_free_list, &defer_free_list); @@ -570,12 +567,14 @@ struct net *copy_net_ns(u64 flags, goto dec_ucounts; } - rv = preinit_net(net, user_ns); - if (rv < 0) - goto dec_ucounts; + preinit_net(net, user_ns); net->ucounts = ucounts; get_user_ns(user_ns); + rv = ns_common_init(net); + if (rv) + goto put_userns_no_common; + rv = down_read_killable(&pernet_ops_rwsem); if (rv < 0) goto put_userns; @@ -587,6 +586,7 @@ struct net *copy_net_ns(u64 flags, if (rv < 0) { put_userns: ns_common_free(net); +put_userns_no_common: #ifdef CONFIG_KEYS key_remove_domain(net->key_domain); #endif @@ -1289,7 +1289,8 @@ void __init net_ns_init(void) * This currently cannot fail as the initial network namespace * has a static inode number. */ - if (preinit_net(&init_net, &init_user_ns)) + preinit_net(&init_net, &init_user_ns); + if (ns_common_init(&init_net)) panic("Could not preinitialize the initial network namespace"); down_write(&pernet_ops_rwsem); diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 50ee550fef73..8f8956fb061b 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -484,6 +484,13 @@ static int page_pool_register_dma_index(struct page_pool *pool, if (unlikely(!PP_DMA_INDEX_BITS)) goto out; + /* + * Drivers request GFP flags according to both the current context and + * the device constraints, but the XArray entry itself is by no mean + * used by the device, so remove zone/policy flags. + */ + gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_COMP); + if (in_softirq()) err = xa_alloc(&pool->dma_mapped, &id, netmem_to_page(netmem), PP_DMA_INDEX_LIMIT, gfp); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index c82a1472a5ea..966af3beed94 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1417,10 +1417,13 @@ EXPORT_SYMBOL(skb_dump); * * Report xmit error if a device callback is tracking this skb. * skb must be freed afterwards. + * + * Does nothing for a cloned skb: the zerocopy state lives in + * skb_shinfo(), which the clones share. */ void skb_tx_error(struct sk_buff *skb) { - if (skb) { + if (skb && !skb_cloned(skb)) { skb_zcopy_downgrade_managed(skb); skb_zcopy_clear(skb, true); } @@ -2001,12 +2004,12 @@ int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) int i, order, psize, new_frags; u32 d_off; - if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) - return -EINVAL; - if (!skb_frags_readable(skb)) return -EFAULT; + if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) + return -EINVAL; + if (!num_frags) goto release; @@ -3870,7 +3873,8 @@ EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); * Return value: * 0: everything is OK * -ENOMEM: couldn't orphan frags of @from due to lack of memory - * -EFAULT: skb_copy_bits() found some problem with skb geometry + * -EFAULT: skb_copy_bits() found some problem with skb geometry, or readable head + * payload would be mixed with unreadable frags. */ int skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) @@ -3905,10 +3909,16 @@ skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) } } + if (!skb_frags_readable(from) && j > 0 && len) { + put_page(page); + return -EFAULT; + } + skb_len_add(to, len + plen); if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { - skb_tx_error(from); + if (j > 0) + put_page(page); return -ENOMEM; } skb_zerocopy_clone(to, from, GFP_ATOMIC); @@ -3928,6 +3938,9 @@ skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) } skb_shinfo(to)->nr_frags = j; + if (i > 0 && from->unreadable) + to->unreadable = 1; + return 0; } EXPORT_SYMBOL_GPL(skb_zerocopy); @@ -4860,7 +4873,8 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, * doesn't fit into an MSS sized block, so take care of that * now. */ - partial_segs = len / mss; + DEBUG_NET_WARN_ON_ONCE(len / mss > GSO_MAX_SEGS); + partial_segs = min(len / mss, GSO_MAX_SEGS); if (partial_segs > 1) mss *= partial_segs; else diff --git a/net/core/xdp.c b/net/core/xdp.c index 0194e69da339..1d679e8fd649 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -585,7 +585,7 @@ struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) xdp->data - xdp->data_meta; totsize = xdp->data_end - xdp->data + metasize; - if (sizeof(*xdpf) + totsize > PAGE_SIZE) + if (sizeof(*xdpf) + totsize > SKB_WITH_OVERHEAD(PAGE_SIZE)) return NULL; page = dev_alloc_page(); @@ -602,7 +602,7 @@ struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) xdpf->data = addr + metasize; xdpf->len = totsize - metasize; - xdpf->headroom = 0; + xdpf->headroom = metasize; xdpf->metasize = metasize; xdpf->frame_sz = PAGE_SIZE; xdpf->mem_type = MEM_TYPE_PAGE_ORDER0; diff --git a/net/ife/ife.c b/net/ife/ife.c index 7a75947a31e3..2ddf725d3389 100644 --- a/net/ife/ife.c +++ b/net/ife/ife.c @@ -37,7 +37,7 @@ void *ife_encode(struct sk_buff *skb, u16 metalen) * where ORIGDATA = original ethernet header ... */ int hdrm = metalen + IFE_METAHDRLEN; - int total_push = hdrm + skb->dev->hard_header_len; + int total_push = hdrm + ETH_HLEN; struct ifeheadr *ifehdr; struct ethhdr *iethh; /* inner ether header */ int skboff = 0; @@ -50,9 +50,9 @@ void *ife_encode(struct sk_buff *skb, u16 metalen) iethh = (struct ethhdr *) skb->data; __skb_push(skb, total_push); - memcpy(skb->data, iethh, skb->dev->hard_header_len); + memcpy(skb->data, iethh, ETH_HLEN); skb_reset_mac_header(skb); - skboff += skb->dev->hard_header_len; + skboff += ETH_HLEN; /* total metadata length */ ifehdr = (struct ifeheadr *) (skb->data + skboff); @@ -69,12 +69,12 @@ void *ife_decode(struct sk_buff *skb, u16 *metalen) int total_pull; u16 ifehdrln; - if (!pskb_may_pull(skb, skb->dev->hard_header_len + IFE_METAHDRLEN)) + if (!pskb_may_pull(skb, ETH_HLEN + IFE_METAHDRLEN)) return NULL; - ifehdr = (struct ifeheadr *) (skb->data + skb->dev->hard_header_len); + ifehdr = (struct ifeheadr *)(skb->data + ETH_HLEN); ifehdrln = ntohs(ifehdr->metalen); - total_pull = skb->dev->hard_header_len + ifehdrln; + total_pull = ETH_HLEN + ifehdrln; if (unlikely(ifehdrln < 2)) return NULL; @@ -82,7 +82,7 @@ void *ife_decode(struct sk_buff *skb, u16 *metalen) if (unlikely(!pskb_may_pull(skb, total_pull + ETH_HLEN))) return NULL; - ifehdr = (struct ifeheadr *)(skb->data + skb->dev->hard_header_len); + ifehdr = (struct ifeheadr *)(skb->data + ETH_HLEN); skb_set_mac_header(skb, total_pull); __skb_pull(skb, total_pull); *metalen = ifehdrln - IFE_METAHDRLEN; diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c index fc0cb993959f..c17e57ec7d5c 100644 --- a/net/ipv4/inet_fragment.c +++ b/net/ipv4/inet_fragment.c @@ -435,6 +435,13 @@ int inet_frag_queue_insert(struct inet_frag_queue *q, struct sk_buff *skb, { struct sk_buff *last = q->fragments_tail; + /* An IP fragment is never a GSO packet, but an untrusted source + * (virtio_net_hdr) may have attached GSO metadata to it. Do not let + * that reach the reassembled skb, whose head keeps the first + * fragment's shinfo and whose frag_list is not GRO-shaped. + */ + skb_gso_reset(skb); + /* RFC5722, Section 4, amended by Errata ID : 3089 * When reassembling an IPv6 datagram, if * one or more its constituent fragments is determined to be an diff --git a/net/ipv4/inetpeer.c b/net/ipv4/inetpeer.c index 5b957a831e7c..adf6dc8a95b4 100644 --- a/net/ipv4/inetpeer.c +++ b/net/ipv4/inetpeer.c @@ -21,6 +21,7 @@ #include <net/ip.h> #include <net/inetpeer.h> #include <net/secure_seq.h> +#include <linux/siphash.h> /* * Theory of operations. @@ -52,6 +53,34 @@ */ static struct kmem_cache *peer_cachep __ro_after_init; +static siphash_aligned_key_t inetpeer_hash_key __read_mostly; + +static u64 inetpeer_addr_hash(const struct inetpeer_addr *a) +{ + net_get_random_once(&inetpeer_hash_key, sizeof(inetpeer_hash_key)); + + if (a->family == AF_INET) + return siphash_2u32((__force u32)a->a4.addr, a->a4.vif, + &inetpeer_hash_key); + + return siphash_4u32((__force u32)a->a6.s6_addr32[0], + (__force u32)a->a6.s6_addr32[1], + (__force u32)a->a6.s6_addr32[2], + (__force u32)a->a6.s6_addr32[3], + &inetpeer_hash_key); +} + +static int inetpeer_entry_cmp(u64 dhash, + const struct inetpeer_addr *daddr, + const struct inet_peer *p) +{ + if (dhash < p->hash) + return -1; + if (dhash > p->hash) + return 1; + + return inetpeer_addr_cmp(daddr, &p->daddr); +} void inet_peer_base_init(struct inet_peer_base *bp) { @@ -84,6 +113,7 @@ void __init inet_initpeers(void) /* Called with rcu_read_lock() or base->lock held */ static struct inet_peer *lookup(const struct inetpeer_addr *daddr, + u64 dhash, struct inet_peer_base *base, unsigned int seq, struct inet_peer *gc_stack[], @@ -105,7 +135,7 @@ static struct inet_peer *lookup(const struct inetpeer_addr *daddr, break; parent = next; p = rb_entry(parent, struct inet_peer, rb_node); - cmp = inetpeer_addr_cmp(daddr, &p->daddr); + cmp = inetpeer_entry_cmp(dhash, daddr, p); if (cmp == 0) { now = jiffies; if (READ_ONCE(p->dtime) != now) @@ -170,6 +200,7 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, const struct inetpeer_addr *daddr) { struct inet_peer *p, *gc_stack[PEER_MAX_GC]; + u64 dhash = inetpeer_addr_hash(daddr); struct rb_node **pp, *parent; unsigned int gc_cnt, seq; @@ -177,7 +208,7 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, * Because of a concurrent writer, we might not find an existing entry. */ seq = read_seqbegin(&base->lock); - p = lookup(daddr, base, seq, NULL, &gc_cnt, &parent, &pp); + p = lookup(daddr, dhash, base, seq, NULL, &gc_cnt, &parent, &pp); /* Make sure tree was not modified during our lookup. */ if (p && !read_seqretry(&base->lock, seq)) @@ -190,11 +221,12 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, write_seqlock_bh(&base->lock); gc_cnt = 0; - p = lookup(daddr, base, seq, gc_stack, &gc_cnt, &parent, &pp); + p = lookup(daddr, dhash, base, seq, gc_stack, &gc_cnt, &parent, &pp); if (!p) { p = kmem_cache_alloc(peer_cachep, GFP_ATOMIC); if (p) { p->daddr = *daddr; + p->hash = dhash; p->dtime = (__u32)jiffies; refcount_set(&p->refcnt, 1); atomic_set(&p->rid, 0); diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index 1630325c77d3..f684baf8e58f 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -248,7 +248,7 @@ static int ipip_tunnel_rcv(struct sk_buff *skb, u8 ipproto) tun_dst = ip_tun_rx_dst(skb, flags, 0, 0); if (!tun_dst) - return 0; + goto drop; ip_tunnel_md_udp_encap(skb, &tun_dst->u.tun_info); } skb_reset_mac_header(skb); diff --git a/net/ipv4/netfilter/ipt_ah.c b/net/ipv4/netfilter/ipt_ah.c index 161ba412cb08..7131f297ada2 100644 --- a/net/ipv4/netfilter/ipt_ah.c +++ b/net/ipv4/netfilter/ipt_ah.c @@ -19,12 +19,7 @@ MODULE_DESCRIPTION("Xtables: IPv4 IPsec-AH SPI match"); static inline bool spi_match(u_int32_t min, u_int32_t max, u_int32_t spi, bool invert) { - bool r; - pr_debug("spi_match:%c 0x%x <= 0x%x <= 0x%x\n", - invert ? '!' : ' ', min, spi, max); - r = (spi >= min && spi <= max) ^ invert; - pr_debug(" result %s\n", r ? "PASS" : "FAILED"); - return r; + return (spi >= min && spi <= max) ^ invert; } static bool ah_mt(const struct sk_buff *skb, struct xt_action_param *par) @@ -42,7 +37,6 @@ static bool ah_mt(const struct sk_buff *skb, struct xt_action_param *par) /* We've been asked to examine this packet, and we * can't. Hence, no choice but to drop. */ - pr_debug("Dropping evil AH tinygram.\n"); par->hotdrop = true; return false; } @@ -58,7 +52,7 @@ static int ah_mt_check(const struct xt_mtchk_param *par) /* Must specify no unknown invflags */ if (ahinfo->invflags & ~IPT_AH_INV_MASK) { - pr_debug("unknown flags %X\n", ahinfo->invflags); + pr_info_ratelimited("unknown flags %X\n", ahinfo->invflags); return -EINVAL; } return 0; diff --git a/net/ipv4/netfilter/nf_tproxy_ipv4.c b/net/ipv4/netfilter/nf_tproxy_ipv4.c index 041c3f37f237..5eab7a2dc8ef 100644 --- a/net/ipv4/netfilter/nf_tproxy_ipv4.c +++ b/net/ipv4/netfilter/nf_tproxy_ipv4.c @@ -137,7 +137,7 @@ nf_tproxy_get_sock_v4(struct net *net, struct sk_buff *skb, } break; default: - WARN_ON(1); + DEBUG_NET_WARN_ON_ONCE(1); sk = NULL; } diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 604cc51dfd9b..37674d76f90f 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -1363,8 +1363,8 @@ static unsigned int ipv4_default_advmss(const struct dst_entry *dst) rcu_read_lock(); net = dst_dev_net_rcu(dst); - advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size, - net->ipv4.ip_rt_min_advmss); + advmss = max_t(unsigned int, ip_dst_mtu_configured(dst) - header_size, + net->ipv4.ip_rt_min_advmss); rcu_read_unlock(); return min(advmss, IPV4_MAX_PMTU - header_size); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 455441f1b694..b4237d0e994d 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1240,7 +1240,8 @@ restart: trace_tcp_sendmsg_locked(sk, msg, skb, size_goal); - if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) { + if (copy <= 0 || !tcp_skb_can_collapse_to(skb) || + unlikely(skb_frags_readable(skb) != !binding)) { bool first_skb; new_segment: diff --git a/net/ipv4/tcp_ao.c b/net/ipv4/tcp_ao.c index e4ec60a33496..bb7bbc20ba3f 100644 --- a/net/ipv4/tcp_ao.c +++ b/net/ipv4/tcp_ao.c @@ -1160,6 +1160,15 @@ void tcp_ao_connect_init(struct sock *sk) l3index = l3mdev_master_ifindex_by_index(sock_net(sk), sk->sk_bound_dev_if); + hlist_for_each_entry(key, &ao_info->head, node) { + if (tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, + family, -1, -1)) { + /* pairs with tcp_inbound_ao_hash() */ + synchronize_rcu(); + break; + } + } + hlist_for_each_entry_safe(key, next, &ao_info->head, node) { if (!tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, family, -1, -1)) continue; @@ -1187,12 +1196,7 @@ void tcp_ao_connect_init(struct sock *sk) ao_info->lisn = htonl(tp->write_seq); ao_info->snd_sne = 0; } else { - /* Can't happen: tcp_connect() verifies that there's - * at least one tcp-ao key that matches the remote peer. - */ - WARN_ON_ONCE(1); - rcu_assign_pointer(tp->ao_info, NULL); - kfree(ao_info); + tcp_ao_destroy_sock(sk, false); } } @@ -1825,6 +1829,9 @@ static int tcp_ao_del_cmd(struct sock *sk, unsigned short int family, if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX)) return -EINVAL; + if (cmd.keyflags & TCP_AO_KEYF_IFINDEX) + l3index = cmd.ifindex; + ao_info = setsockopt_ao_info(sk); if (IS_ERR(ao_info)) return PTR_ERR(ao_info); diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 190c7af4cf92..9f053eb8b46e 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1736,7 +1736,7 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, tcp_ca_openreq_child(newsk, dst); tcp_sync_mss(newsk, dst4_mtu(dst)); - newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); + newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst)); tcp_initialize_rcv_mss(newsk); diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index 12254e6eb2f3..f3fa0b18eda0 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -440,7 +440,7 @@ void tcp_openreq_init_rwin(struct request_sock *req, u32 rcv_wnd; int mss; - mss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); + mss = tcp_mss_clamp(tp, tcp_dst_advmss(dst)); window_clamp = READ_ONCE(tp->window_clamp); /* Set this up on the first call only */ req->rsk_window_clamp = window_clamp ? : dst_metric(dst, RTAX_WINDOW); diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index fcaa04e65189..6f4dca4a4de9 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -143,7 +143,7 @@ static __u16 tcp_advertise_mss(struct sock *sk) int mss = tp->advmss; if (dst) { - unsigned int metric = dst_metric_advmss(dst); + unsigned int metric = tcp_dst_advmss(dst); if (metric < mss) { mss = metric; @@ -3603,7 +3603,7 @@ start: avail_wnd = cur_mss; } - len = cur_mss * segs; + len = cur_mss * (tcp_urg_mode(tp) ? 1 : segs); if (len > avail_wnd) { len = rounddown(avail_wnd, cur_mss); if (!len) @@ -3972,7 +3972,7 @@ struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, } skb_dst_set(skb, dst); - mss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); + mss = tcp_mss_clamp(tp, tcp_dst_advmss(dst)); memset(&opts, 0, sizeof(opts)); now = tcp_clock_ns(); @@ -4128,7 +4128,7 @@ static void tcp_connect_init(struct sock *sk) if (!tp->window_clamp) WRITE_ONCE(tp->window_clamp, dst_metric(dst, RTAX_WINDOW)); - tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); + tp->advmss = tcp_mss_clamp(tp, tcp_dst_advmss(dst)); tcp_initialize_rcv_mss(sk); diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c index 76f7a2de9108..c96f7e0d0a48 100644 --- a/net/ipv6/ah6.c +++ b/net/ipv6/ah6.c @@ -232,26 +232,28 @@ static void ipv6_rearrange_destopt(struct ipv6hdr *iph, struct ipv6_opt_hdr *des * Rearrange the destination address in @iph and the addresses in @rthdr * so that they appear in the order they will at the final destination. * See Appendix A2 of RFC 2402 for details. + * + * Return: 0 on success, -EINVAL if segments_left exceeds the number of + * addresses described by hdrlen. */ -static void ipv6_rearrange_rthdr(struct ipv6hdr *iph, struct ipv6_rt_hdr *rthdr) +static int ipv6_rearrange_rthdr(struct ipv6hdr *iph, struct ipv6_rt_hdr *rthdr) { - int segments, segments_left; + unsigned int segments, segments_left; struct in6_addr *addrs; struct in6_addr final_addr; segments_left = rthdr->segments_left; if (segments_left == 0) - return; - rthdr->segments_left = 0; + return 0; - /* The value of rthdr->hdrlen has been verified either by the system - * call if it is locally generated, or by ipv6_rthdr_rcv() for incoming - * packets. So we can assume that it is even and that segments is - * greater than or equal to segments_left. - * - * For the same reason we can assume that this option is of type 0. + /* Raw locally generated packets can reach AH6 without the invariant + * required by the rt0-style address rearrangement below. */ segments = rthdr->hdrlen >> 1; + if (segments_left > segments) + return -EINVAL; + + rthdr->segments_left = 0; addrs = ((struct rt0_hdr *)rthdr)->addr; final_addr = addrs[segments - 1]; @@ -261,6 +263,8 @@ static void ipv6_rearrange_rthdr(struct ipv6hdr *iph, struct ipv6_rt_hdr *rthdr) addrs[0] = iph->daddr; iph->daddr = final_addr; + + return 0; } static int ipv6_clear_mutable_options(struct ipv6hdr *iph, int len, int dir) @@ -273,6 +277,7 @@ static int ipv6_clear_mutable_options(struct ipv6hdr *iph, int len, int dir) } exthdr = { .iph = iph }; char *end = exthdr.raw + len; int nexthdr = iph->nexthdr; + int err; exthdr.iph++; @@ -292,7 +297,9 @@ static int ipv6_clear_mutable_options(struct ipv6hdr *iph, int len, int dir) break; case NEXTHDR_ROUTING: - ipv6_rearrange_rthdr(iph, exthdr.rth); + err = ipv6_rearrange_rthdr(iph, exthdr.rth); + if (err) + return err; break; default: diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c index 9c677eb1d1a6..51941ad656a3 100644 --- a/net/ipv6/exthdrs.c +++ b/net/ipv6/exthdrs.c @@ -368,23 +368,16 @@ static void seg6_update_csum(struct sk_buff *skb) (__be32 *)addr); } -static int ipv6_srh_rcv(struct sk_buff *skb) +static int ipv6_srh_rcv(struct sk_buff *skb, struct inet6_dev *idev) { struct inet6_skb_parm *opt = IP6CB(skb); struct net *net = dev_net(skb->dev); struct ipv6_sr_hdr *hdr; - struct inet6_dev *idev; struct in6_addr *addr; int accept_seg6; hdr = (struct ipv6_sr_hdr *)skb_transport_header(skb); - idev = __in6_dev_get(skb->dev); - if (!idev) { - kfree_skb(skb); - return -1; - } - accept_seg6 = min(READ_ONCE(net->ipv6.devconf_all->seg6_enabled), READ_ONCE(idev->cnf.seg6_enabled)); @@ -485,12 +478,11 @@ looped_back: return -1; } -static int ipv6_rpl_srh_rcv(struct sk_buff *skb) +static int ipv6_rpl_srh_rcv(struct sk_buff *skb, struct inet6_dev *idev) { struct ipv6_rpl_sr_hdr *hdr, *ohdr, *chdr; struct inet6_skb_parm *opt = IP6CB(skb); struct net *net = dev_net(skb->dev); - struct inet6_dev *idev; struct ipv6hdr *oldhdr; unsigned int chdr_len; unsigned char *buf; @@ -499,8 +491,6 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb) u64 n = 0; u32 r; - idev = __in6_dev_get(skb->dev); - accept_rpl_seg = min(READ_ONCE(net->ipv6.devconf_all->rpl_seg_enabled), READ_ONCE(idev->cnf.rpl_seg_enabled)); if (!accept_rpl_seg) { @@ -689,10 +679,14 @@ static int ipv6_rthdr_rcv(struct sk_buff *skb) switch (hdr->type) { case IPV6_SRCRT_TYPE_4: /* segment routing */ - return ipv6_srh_rcv(skb); + if (!idev) + goto disabled; + return ipv6_srh_rcv(skb, idev); case IPV6_SRCRT_TYPE_3: /* rpl segment routing */ - return ipv6_rpl_srh_rcv(skb); + if (!idev) + goto disabled; + return ipv6_rpl_srh_rcv(skb, idev); default: break; } @@ -837,6 +831,10 @@ unknown_rh: icmpv6_param_prob(skb, ICMPV6_HDR_FIELD, (&hdr->type) - skb_network_header(skb)); return -1; + +disabled: + kfree_skb_reason(skb, SKB_DROP_REASON_IPV6DISABLED); + return -1; } static const struct inet6_protocol rthdr_protocol = { diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index 604a58838901..3f2ed9b77deb 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -1162,10 +1162,10 @@ static int ip6mr_cache_report(const struct mr_table *mrt, struct sk_buff *pkt, msg->im6_src = ipv6_hdr(pkt)->saddr; msg->im6_dst = ipv6_hdr(pkt)->daddr; - skb_dst_set(skb, dst_clone(skb_dst(pkt))); skb->ip_summed = CHECKSUM_UNNECESSARY; } + skb_dst_drop(skb); mrt6msg_netlink_event(mrt, skb); /* Deliver to user space multicast routing algorithms */ @@ -1246,6 +1246,7 @@ static int ip6mr_cache_unresolved(struct mr_table *mrt, mifi_t mifi, skb->skb_iif = dev->ifindex; } + skb_dst_drop(skb); skb_queue_tail(&c->_c.mfc_un.unres.unresolved, skb); spin_unlock_bh(&mfc_unres_lock); diff --git a/net/ipv6/netfilter/ip6t_ah.c b/net/ipv6/netfilter/ip6t_ah.c index 1258783ed876..dab7dbc6a675 100644 --- a/net/ipv6/netfilter/ip6t_ah.c +++ b/net/ipv6/netfilter/ip6t_ah.c @@ -24,13 +24,7 @@ MODULE_AUTHOR("Andras Kis-Szabo <kisza@sch.bme.hu>"); static inline bool spi_match(u_int32_t min, u_int32_t max, u_int32_t spi, bool invert) { - bool r; - - pr_debug("spi_match:%c 0x%x <= 0x%x <= 0x%x\n", - invert ? '!' : ' ', min, spi, max); - r = (spi >= min && spi <= max) ^ invert; - pr_debug(" result %s\n", r ? "PASS" : "FAILED"); - return r; + return (spi >= min && spi <= max) ^ invert; } static bool ah_mt6(const struct sk_buff *skb, struct xt_action_param *par) @@ -62,23 +56,6 @@ static bool ah_mt6(const struct sk_buff *skb, struct xt_action_param *par) return false; } - pr_debug("IPv6 AH LEN %u %u ", hdrlen, ah->hdrlen); - pr_debug("RES %04X ", ah->reserved); - pr_debug("SPI %u %08X\n", ntohl(ah->spi), ntohl(ah->spi)); - - pr_debug("IPv6 AH spi %02X ", - spi_match(ahinfo->spis[0], ahinfo->spis[1], - ntohl(ah->spi), - !!(ahinfo->invflags & IP6T_AH_INV_SPI))); - pr_debug("len %02X %04X %02X ", - ahinfo->hdrlen, hdrlen, - (!ahinfo->hdrlen || - (ahinfo->hdrlen == hdrlen) ^ - !!(ahinfo->invflags & IP6T_AH_INV_LEN))); - pr_debug("res %02X %04X %02X\n", - ahinfo->hdrres, ah->reserved, - !(ahinfo->hdrres && ah->reserved)); - return spi_match(ahinfo->spis[0], ahinfo->spis[1], ntohl(ah->spi), !!(ahinfo->invflags & IP6T_AH_INV_SPI)) && @@ -93,7 +70,7 @@ static int ah_mt6_check(const struct xt_mtchk_param *par) const struct ip6t_ah *ahinfo = par->matchinfo; if (ahinfo->invflags & ~IP6T_AH_INV_MASK) { - pr_debug("unknown flags %X\n", ahinfo->invflags); + pr_info_ratelimited("unknown flags %X\n", ahinfo->invflags); return -EINVAL; } return 0; diff --git a/net/ipv6/netfilter/ip6t_frag.c b/net/ipv6/netfilter/ip6t_frag.c index 3aad6439386b..f5f3cfb8704c 100644 --- a/net/ipv6/netfilter/ip6t_frag.c +++ b/net/ipv6/netfilter/ip6t_frag.c @@ -23,12 +23,7 @@ MODULE_AUTHOR("Andras Kis-Szabo <kisza@sch.bme.hu>"); static inline bool id_match(u_int32_t min, u_int32_t max, u_int32_t id, bool invert) { - bool r; - pr_debug("id_match:%c 0x%x <= 0x%x <= 0x%x\n", invert ? '!' : ' ', - min, id, max); - r = (id >= min && id <= max) ^ invert; - pr_debug(" result %s\n", r ? "PASS" : "FAILED"); - return r; + return (id >= min && id <= max) ^ invert; } static bool @@ -53,38 +48,6 @@ frag_mt6(const struct sk_buff *skb, struct xt_action_param *par) return false; } - pr_debug("INFO %04X ", fh->frag_off); - pr_debug("OFFSET %04X ", ntohs(fh->frag_off) & ~0x7); - pr_debug("RES %02X %04X", fh->reserved, ntohs(fh->frag_off) & 0x6); - pr_debug("MF %04X ", fh->frag_off & htons(IP6_MF)); - pr_debug("ID %u %08X\n", ntohl(fh->identification), - ntohl(fh->identification)); - - pr_debug("IPv6 FRAG id %02X ", - id_match(fraginfo->ids[0], fraginfo->ids[1], - ntohl(fh->identification), - !!(fraginfo->invflags & IP6T_FRAG_INV_IDS))); - pr_debug("res %02X %02X%04X %02X ", - fraginfo->flags & IP6T_FRAG_RES, fh->reserved, - ntohs(fh->frag_off) & 0x6, - !((fraginfo->flags & IP6T_FRAG_RES) && - (fh->reserved || (ntohs(fh->frag_off) & 0x06)))); - pr_debug("first %02X %02X %02X ", - fraginfo->flags & IP6T_FRAG_FST, - ntohs(fh->frag_off) & ~0x7, - !((fraginfo->flags & IP6T_FRAG_FST) && - (ntohs(fh->frag_off) & ~0x7))); - pr_debug("mf %02X %02X %02X ", - fraginfo->flags & IP6T_FRAG_MF, - ntohs(fh->frag_off) & IP6_MF, - !((fraginfo->flags & IP6T_FRAG_MF) && - !((ntohs(fh->frag_off) & IP6_MF)))); - pr_debug("last %02X %02X %02X\n", - fraginfo->flags & IP6T_FRAG_NMF, - ntohs(fh->frag_off) & IP6_MF, - !((fraginfo->flags & IP6T_FRAG_NMF) && - (ntohs(fh->frag_off) & IP6_MF))); - return id_match(fraginfo->ids[0], fraginfo->ids[1], ntohl(fh->identification), !!(fraginfo->invflags & IP6T_FRAG_INV_IDS)) && @@ -103,7 +66,7 @@ static int frag_mt6_check(const struct xt_mtchk_param *par) const struct ip6t_frag *fraginfo = par->matchinfo; if (fraginfo->invflags & ~IP6T_FRAG_INV_MASK) { - pr_debug("unknown flags %X\n", fraginfo->invflags); + pr_info_ratelimited("unknown flags %X\n", fraginfo->invflags); return -EINVAL; } return 0; diff --git a/net/ipv6/netfilter/ip6t_hbh.c b/net/ipv6/netfilter/ip6t_hbh.c index 6d1a5d2026a6..6008dcff8488 100644 --- a/net/ipv6/netfilter/ip6t_hbh.c +++ b/net/ipv6/netfilter/ip6t_hbh.c @@ -79,14 +79,6 @@ hbh_mt6(const struct sk_buff *skb, struct xt_action_param *par) return false; } - pr_debug("IPv6 OPTS LEN %u %u ", hdrlen, oh->hdrlen); - - pr_debug("len %02X %04X %02X ", - optinfo->hdrlen, hdrlen, - (!(optinfo->flags & IP6T_OPTS_LEN) || - ((optinfo->hdrlen == hdrlen) ^ - !!(optinfo->invflags & IP6T_OPTS_INV_LEN)))); - ret = (!(optinfo->flags & IP6T_OPTS_LEN) || ((optinfo->hdrlen == hdrlen) ^ !!(optinfo->invflags & IP6T_OPTS_INV_LEN))); @@ -96,8 +88,6 @@ hbh_mt6(const struct sk_buff *skb, struct xt_action_param *par) if (!(optinfo->flags & IP6T_OPTS_OPTS)) { return ret; } else { - pr_debug("Strict "); - pr_debug("#%d ", optinfo->optsnr); for (temp = 0; temp < optinfo->optsnr; temp++) { /* type field exists ? */ if (hdrlen < 1) @@ -108,13 +98,9 @@ hbh_mt6(const struct sk_buff *skb, struct xt_action_param *par) break; /* Type check */ - if (*tp != (optinfo->opts[temp] & 0xFF00) >> 8) { - pr_debug("Tbad %02X %02X\n", *tp, - (optinfo->opts[temp] & 0xFF00) >> 8); + if (*tp != (optinfo->opts[temp] & 0xFF00) >> 8) return false; - } else { - pr_debug("Tok "); - } + /* Length check */ if (*tp) { u16 spec_len; @@ -129,26 +115,18 @@ hbh_mt6(const struct sk_buff *skb, struct xt_action_param *par) break; spec_len = optinfo->opts[temp] & 0x00FF; - if (spec_len != 0x00FF && spec_len != *lp) { - pr_debug("Lbad %02X %04X\n", *lp, - spec_len); + if (spec_len != 0x00FF && spec_len != *lp) return false; - } - pr_debug("Lok "); + optlen = *lp + 2; } else { - pr_debug("Pad1\n"); optlen = 1; } - /* Step to the next */ - pr_debug("len%04X\n", optlen); - if ((ptr > skb->len - optlen || hdrlen < optlen) && - temp < optinfo->optsnr - 1) { - pr_debug("new pointer is too large!\n"); + temp < optinfo->optsnr - 1) break; - } + ptr += optlen; hdrlen -= optlen; } @@ -166,16 +144,16 @@ static int hbh_mt6_check(const struct xt_mtchk_param *par) const struct ip6t_opts *optsinfo = par->matchinfo; if (optsinfo->invflags & ~IP6T_OPTS_INV_MASK) { - pr_debug("unknown flags %X\n", optsinfo->invflags); + pr_info_ratelimited("unknown flags %X\n", optsinfo->invflags); return -EINVAL; } if (optsinfo->optsnr > IP6T_OPTS_OPTSNR) { - pr_debug("too many supported opts specified\n"); + pr_info_ratelimited("too many supported opts specified\n"); return -EINVAL; } if (optsinfo->flags & IP6T_OPTS_NSTRICT) { - pr_debug("Not strict - not implemented"); + pr_info_ratelimited("Not strict - not implemented\n"); return -EINVAL; } diff --git a/net/ipv6/netfilter/ip6t_mh.c b/net/ipv6/netfilter/ip6t_mh.c index fd492b69acbc..ba6dcc7791a0 100644 --- a/net/ipv6/netfilter/ip6t_mh.c +++ b/net/ipv6/netfilter/ip6t_mh.c @@ -42,14 +42,11 @@ static bool mh_mt6(const struct sk_buff *skb, struct xt_action_param *par) if (mh == NULL) { /* We've been asked to examine this packet, and we can't. Hence, no choice but to drop. */ - pr_debug("Dropping evil MH tinygram.\n"); par->hotdrop = true; return false; } if (mh->ip6mh_proto != IPPROTO_NONE) { - pr_debug("Dropping invalid MH Payload Proto: %u\n", - mh->ip6mh_proto); par->hotdrop = true; return false; } diff --git a/net/ipv6/netfilter/ip6t_rt.c b/net/ipv6/netfilter/ip6t_rt.c index 278b52752f36..8051425213dd 100644 --- a/net/ipv6/netfilter/ip6t_rt.c +++ b/net/ipv6/netfilter/ip6t_rt.c @@ -155,18 +155,18 @@ static int rt_mt6_check(const struct xt_mtchk_param *par) const struct ip6t_rt *rtinfo = par->matchinfo; if (rtinfo->invflags & ~IP6T_RT_INV_MASK) { - pr_debug("unknown flags %X\n", rtinfo->invflags); + pr_info_ratelimited("unknown flags %X\n", rtinfo->invflags); return -EINVAL; } if (rtinfo->addrnr > IP6T_RT_HOPS) { - pr_debug("too many addresses specified\n"); + pr_info_ratelimited("too many addresses specified\n"); return -EINVAL; } if ((rtinfo->flags & (IP6T_RT_RES | IP6T_RT_FST_MASK)) && (!(rtinfo->flags & IP6T_RT_TYP) || (rtinfo->rt_type != 0) || (rtinfo->invflags & IP6T_RT_INV_TYP))) { - pr_debug("`--rt-type 0' required before `--rt-0-*'"); + pr_info_ratelimited("`--rt-type 0' required before `--rt-0-*'\n"); return -EINVAL; } diff --git a/net/ipv6/netfilter/nf_tproxy_ipv6.c b/net/ipv6/netfilter/nf_tproxy_ipv6.c index b2f59ed9d7cc..12ec36a6be2e 100644 --- a/net/ipv6/netfilter/nf_tproxy_ipv6.c +++ b/net/ipv6/netfilter/nf_tproxy_ipv6.c @@ -136,7 +136,7 @@ nf_tproxy_get_sock_v6(struct net *net, struct sk_buff *skb, int thoff, } break; default: - WARN_ON(1); + DEBUG_NET_WARN_ON_ONCE(1); sk = NULL; } diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 16dfac54a259..6a40c5074543 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -3261,7 +3261,7 @@ void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk) static unsigned int ip6_default_advmss(const struct dst_entry *dst) { - unsigned int mtu = dst6_mtu(dst); + unsigned int mtu = ip6_dst_mtu_configured(dst); struct net *net; mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr); @@ -4852,7 +4852,7 @@ static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total) { int upper_bound = -1; - if (!rt6_is_dead(rt)) { + if (total && !rt6_is_dead(rt)) { *weight += rt->fib6_nh->fib_nh_weight; upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31, total) - 1; @@ -6046,7 +6046,7 @@ static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg) return 0; for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { - hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { + hlist_for_each_entry_rcu(rt6_ex, &bucket->chain, hlist) { if (w->skip) { w->skip--; continue; diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c index 2b41e4c0dddd..7b5212220185 100644 --- a/net/ipv6/seg6_local.c +++ b/net/ipv6/seg6_local.c @@ -256,6 +256,22 @@ static bool decap_and_validate(struct sk_buff *skb, int proto) if (iptunnel_pull_offloads(skb)) return false; + if (proto == IPPROTO_IPIP) { + int iif = IP6CB(skb)->iif; + + memset(IPCB(skb), 0, sizeof(*IPCB(skb))); + IPCB(skb)->iif = iif; + } else if (proto == IPPROTO_IPV6) { + bool l3slave = ipv6_l3mdev_skb(IP6CB(skb)->flags); + int iif = IP6CB(skb)->iif; + + memset(IP6CB(skb), 0, sizeof(*IP6CB(skb))); + IP6CB(skb)->iif = iif; + IP6CB(skb)->nhoff = offsetof(struct ipv6hdr, nexthdr); + if (l3slave) + IP6CB(skb)->flags |= IP6SKB_L3SLAVE; + } + return true; } diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 9e9155b1b3aa..df9c29eb5c1f 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -1487,7 +1487,7 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * tcp_ca_openreq_child(newsk, dst); tcp_sync_mss(newsk, dst6_mtu(dst)); - newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); + newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst)); tcp_initialize_rcv_mss(newsk); diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c index 89d0443b5307..07edef258984 100644 --- a/net/ipv6/xfrm6_input.c +++ b/net/ipv6/xfrm6_input.c @@ -247,7 +247,7 @@ int xfrm6_input_addr(struct sk_buff *skb, xfrm_address_t *daddr, goto drop; } - if (1 + sp->len == XFRM_MAX_DEPTH) { + if (sp->len >= XFRM_MAX_DEPTH) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); goto drop; } diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c index ea047bab65e7..4e5cc9da6e06 100644 --- a/net/iucv/af_iucv.c +++ b/net/iucv/af_iucv.c @@ -2079,6 +2079,8 @@ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev, sk = NULL; read_lock(&iucv_sk_list.lock); sk_for_each(sk, &iucv_sk_list.head) { + if (iucv_sk(sk)->hs_dev != dev) + continue; if (trans_hdr->flags == AF_IUCV_FLAG_SYN) { if ((!memcmp(&iucv_sk(sk)->src_name, trans_hdr->destAppName, 8)) && diff --git a/net/l2tp/l2tp_netlink.c b/net/l2tp/l2tp_netlink.c index c0c4d1ebc7a3..38aac59d052c 100644 --- a/net/l2tp/l2tp_netlink.c +++ b/net/l2tp/l2tp_netlink.c @@ -251,8 +251,8 @@ static int l2tp_nl_cmd_tunnel_create(struct sk_buff *skb, struct genl_info *info kfree(tunnel); goto out; } - ret = l2tp_tunnel_notify(&l2tp_nl_family, info, tunnel, - L2TP_CMD_TUNNEL_CREATE); + l2tp_tunnel_notify(&l2tp_nl_family, info, tunnel, + L2TP_CMD_TUNNEL_CREATE); l2tp_tunnel_put(tunnel); out: @@ -308,8 +308,8 @@ static int l2tp_nl_cmd_tunnel_modify(struct sk_buff *skb, struct genl_info *info goto out; } - ret = l2tp_tunnel_notify(&l2tp_nl_family, info, - tunnel, L2TP_CMD_TUNNEL_MODIFY); + l2tp_tunnel_notify(&l2tp_nl_family, info, + tunnel, L2TP_CMD_TUNNEL_MODIFY); l2tp_tunnel_put(tunnel); @@ -648,8 +648,8 @@ static int l2tp_nl_cmd_session_create(struct sk_buff *skb, struct genl_info *inf session = l2tp_session_get(net, tunnel->sock, tunnel->version, tunnel_id, session_id); if (session) { - ret = l2tp_session_notify(&l2tp_nl_family, info, session, - L2TP_CMD_SESSION_CREATE); + l2tp_session_notify(&l2tp_nl_family, info, session, + L2TP_CMD_SESSION_CREATE); l2tp_session_put(session); } } @@ -713,8 +713,8 @@ static int l2tp_nl_cmd_session_modify(struct sk_buff *skb, struct genl_info *inf if (info->attrs[L2TP_ATTR_RECV_TIMEOUT]) session->reorder_timeout = nla_get_msecs(info->attrs[L2TP_ATTR_RECV_TIMEOUT]); - ret = l2tp_session_notify(&l2tp_nl_family, info, - session, L2TP_CMD_SESSION_MODIFY); + l2tp_session_notify(&l2tp_nl_family, info, + session, L2TP_CMD_SESSION_MODIFY); l2tp_session_put(session); diff --git a/net/mctp/route.c b/net/mctp/route.c index 1f3dccbb7aed..b19c63a5691a 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -998,14 +998,29 @@ int mctp_route_lookup(struct net *net, unsigned int dnet, mtu = mtu ?: rt->mtu; if (rt->dst_type == MCTP_ROUTE_DIRECT) { - mctp_eid_t saddr = mctp_dev_saddr(rt->dev); + mctp_eid_t saddr; + + /* rt->dev may be going away concurrently: its last + * reference is dropped in mctp_dev_put(), which frees + * mdev->addrs that mctp_dev_saddr() reads, and + * mctp_dst_from_route() takes a reference on it. Pin + * it before use, and skip a device that is already + * dead rather than resurrecting it. + */ + if (!refcount_inc_not_zero(&rt->dev->refs)) + break; + + saddr = mctp_dev_saddr(rt->dev); /* cannot do gateway-ed routes without a src */ - if (saddr == MCTP_ADDR_NULL && depth != 0) + if (saddr == MCTP_ADDR_NULL && depth != 0) { + mctp_dev_put(rt->dev); break; + } if (dst) mctp_dst_from_route(dst, daddr, saddr, mtu, rt); + mctp_dev_put(rt->dev); rc = 0; break; diff --git a/net/mptcp/syncookies.c b/net/mptcp/syncookies.c index 7f22526346a7..b5cac5701122 100644 --- a/net/mptcp/syncookies.c +++ b/net/mptcp/syncookies.c @@ -118,6 +118,7 @@ bool mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subfl subflow_req->local_nonce = e->local_nonce; subflow_req->backup = e->backup; subflow_req->remote_id = e->join_id; + subflow_req->local_id = e->local_id; subflow_req->token = e->token; subflow_req->msk = msk; spin_unlock_bh(&join_entry_locks[i]); diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index f00c82acd7f0..80ca523f304b 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -8,6 +8,7 @@ #include <linux/rcupdate_wait.h> #include <linux/jhash.h> #include <linux/types.h> +#include <linux/seqlock.h> #include <linux/netfilter/nfnetlink.h> #include <linux/netfilter/ipset/ip_set.h> @@ -98,14 +99,34 @@ struct htable { #define IPSET_NET_COUNT 1 #endif -/* Book-keeping of the prefixes added to the set */ +/** + * struct net_prefix - Representation of a network prefix. + * @cidr: The CIDR prefix length. + * @count: Number of occurrences. + */ struct net_prefix { - u8 cidr; /* the cidr value */ - u32 count; /* number of elements of this cidr */ + u32 cidr:8; + u32 count:24; }; +#define CIDR_MAX_COUNT ((1 << 24) - 1) + +/** + * struct net_prefixes - A collection of network prefixes. + * @rcu: RCU head + * @seq: Sequence counter guarding in-place reordering of @nets + * @len: Number of entries in the array. + * @nets: Array of net_prefix structures (sorted by CIDR descending). + * + * @nets entries are updated in place under @set's lock. A single entry's + * cidr/count pair is always updated atomically via READ_ONCE()/WRITE_ONCE(), + * but removing an entry also shifts every following entry down by one slot. + * Lockless readers that scan the whole array (i.e. more than a single + * indexed slot) must use @seq to detect and retry across such a shift. + */ struct net_prefixes { struct rcu_head rcu; + seqcount_spinlock_t seq; u8 len; struct net_prefix nets[] __counted_by(len); }; @@ -143,8 +164,11 @@ htable_size(u8 hbits) #endif #define INIT_CIDR(n, host_mask) ({ \ - const struct net_prefixes *__n = rcu_dereference(n); \ - DCIDR_PUT((__n)->len ? (__n)->nets[0].cidr : host_mask);\ + const struct net_prefixes *__n = rcu_dereference(n); \ + struct net_prefix __p = \ + __n->len ? READ_ONCE(__n->nets[0]) \ + : (struct net_prefix){}; \ + DCIDR_PUT(__p.count ? __p.cidr : host_mask); \ }) #endif /* IP_SET_HASH_WITH_NETS */ @@ -318,27 +342,43 @@ struct mtype_resize_ad { }; #ifdef IP_SET_HASH_WITH_NETS -/* Network cidr size book keeping when the hash stores different - * sized networks. cidr == real cidr + 1 to support /0. +/** + * mtype_add_cidr - Add a CIDR entry to hash table bookkeeping + * @set: Pointer to the ip_set + * @h: Pointer to the htype + * @cidr: The CIDR prefix length + * @n: The index of the net_prefix array to add @cidr to + * + * Performs an update if @cidr is found, otherwise performs COW-style + * allocation and replacement via RCU. + * + * Return: 0 on success, negative error code on failure. */ static int mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) { - struct net_prefixes *nets, *tmp; int i, j, found, len = 0, ret = 0; + struct net_prefixes *nets, *tmp; + struct net_prefix np; spin_lock_bh(&set->lock); nets = __ipset_dereference(h->rnets[n]); /* Add in increasing prefix order, so larger cidr first */ for (i = 0, found = -1; i < nets->len; i++) { - if (nets->nets[i].count) + np = READ_ONCE(nets->nets[i]); + if (np.count) len++; if (found != -1) { continue; - } else if (nets->nets[i].cidr < cidr) { + } else if (np.cidr < cidr) { found = i; - } else if (nets->nets[i].cidr == cidr) { - nets->nets[i].count++; + } else if (np.cidr == cidr) { + if (np.count < CIDR_MAX_COUNT) { + np.count++; + WRITE_ONCE(nets->nets[i], np); + } else { + ret = -EOVERFLOW; + } goto unlock; } } @@ -350,6 +390,7 @@ mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) } tmp->len = len; + seqcount_spinlock_init(&tmp->seq, &set->lock); for (i = 0, j = 0; i < nets->len; i++) { if (i == found) { tmp->nets[j].cidr = cidr; @@ -371,42 +412,60 @@ unlock: return ret; } +/** + * mtype_del_cidr - Remove CIDR entry and maintain array integrity. + * @set: Pointer to the ip_set. + * @h: Pointer to the htype. + * @cidr: The CIDR prefix length. + * @n: The index of the net_prefix array to remove @cidr from + * + * If CIDR entry count falls to 0, this function performs a "shift-left" + * operation on all following elements. This ensures that the array remains + * contiguous and maintains its descending order by CIDR. The vacated slot + * at the end of the array is zeroed out (cidr=0, count=0). + */ static void mtype_del_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) { - struct net_prefixes *nets, *tmp; - u8 i, j, len = 0; + struct net_prefixes *nets; + struct net_prefix np; int found; + u8 i, j; + + BUILD_BUG_ON(sizeof(struct net_prefix) != sizeof(u32)); spin_lock_bh(&set->lock); nets = __ipset_dereference(h->rnets[n]); for (i = 0, found = -1; i < nets->len; i++) { - if (nets->nets[i].count) - len++; - if (nets->nets[i].cidr == cidr) + np = READ_ONCE(nets->nets[i]); + if (np.count && np.cidr == cidr) { + np.count--; found = i; + break; + } } if (unlikely(found == -1)) goto unlock; - nets->nets[found].count--; - if (nets->nets[found].count) - goto unlock; - len--; - tmp = kzalloc_flex(*tmp, nets, len, GFP_ATOMIC); - if (!tmp) - /* Leave a hole */ + if (np.count) { + WRITE_ONCE(nets->nets[found], np); goto unlock; + } - tmp->len = len; + write_seqcount_begin(&nets->seq); for (i = 0, j = 0; i < nets->len; i++) { - if (!nets->nets[i].count || i == found) + if (i == found) continue; - tmp->nets[j].cidr = nets->nets[i].cidr; - tmp->nets[j++].count = nets->nets[i].count; + + np = READ_ONCE(nets->nets[i]); + if (i != j) + WRITE_ONCE(nets->nets[j], np); + j++; } - rcu_assign_pointer(h->rnets[n], tmp); - kfree_rcu(nets, rcu); + + while (j < nets->len) + WRITE_ONCE(nets->nets[j++], (struct net_prefix){}); + write_seqcount_end(&nets->seq); unlock: spin_unlock_bh(&set->lock); } @@ -451,7 +510,7 @@ mtype_flush(struct ip_set *set) { struct htype *h = set->data; #ifdef IP_SET_HASH_WITH_NETS - struct net_prefixes *nets, *tmp; + struct net_prefixes *nets; #endif struct htable *t; struct hbucket *n; @@ -477,17 +536,15 @@ mtype_flush(struct ip_set *set) } #ifdef IP_SET_HASH_WITH_NETS for (i = 0; i < IPSET_NET_COUNT; i++) { - nets = ipset_dereference_nfnl(h->rnets[i]); - tmp = kzalloc_obj(*tmp, GFP_ATOMIC); - if (!tmp) { - u8 j; + u8 j; - for (j = 0; j < nets->len; j++) - nets->nets[j].count = 0; - } else { - rcu_assign_pointer(h->rnets[i], tmp); - kfree_rcu(nets, rcu); - } + spin_lock_bh(&set->lock); + nets = ipset_dereference_nfnl(h->rnets[i]); + write_seqcount_begin(&nets->seq); + for (j = 0; j < nets->len; j++) + WRITE_ONCE(nets->nets[j], (struct net_prefix){}); + write_seqcount_end(&nets->seq); + spin_unlock_bh(&set->lock); } #endif } @@ -1253,31 +1310,41 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, #if IPSET_NET_COUNT == 2 struct net_prefixes *nets1; struct mtype_elem orig = *d; + unsigned int seq1; int ret, i, j, k; #else int ret, i, j; #endif - u32 key, multi = 0; + unsigned int seq0; + u32 key, multi; u8 pos; pr_debug("test by nets\n"); rcu_read_lock_bh(); +retry: + multi = 0; nets0 = rcu_dereference_bh(h->rnets[0]); + seq0 = read_seqcount_begin(&nets0->seq); #if IPSET_NET_COUNT == 2 nets1 = rcu_dereference_bh(h->rnets[1]); + seq1 = read_seqcount_begin(&nets1->seq); #endif for (j = 0; j < nets0->len && !multi; j++) { - if (!nets0->nets[j].count) + struct net_prefix p0 = READ_ONCE(nets0->nets[j]); + + if (!p0.count) continue; #if IPSET_NET_COUNT == 2 mtype_data_reset_elem(d, &orig); - mtype_data_netmask(d, nets0->nets[j].cidr, false); + mtype_data_netmask(d, p0.cidr, false); for (k = 0; k < nets1->len && !multi; k++) { - if (!nets1->nets[k].count) + struct net_prefix p1 = READ_ONCE(nets1->nets[k]); + + if (!p1.count) continue; - mtype_data_netmask(d, nets1->nets[k].cidr, true); + mtype_data_netmask(d, p1.cidr, true); #else - mtype_data_netmask(d, nets0->nets[j].cidr); + mtype_data_netmask(d, p0.cidr); #endif key = HKEY(d, h->initval, t->htable_bits); n = rcu_dereference_bh(hbucket(t, key)); @@ -1304,6 +1371,12 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, } ret = 0; unlock: + if (read_seqcount_retry(&nets0->seq, seq0)) + goto retry; +#if IPSET_NET_COUNT == 2 + if (read_seqcount_retry(&nets1->seq, seq1)) + goto retry; +#endif rcu_read_unlock_bh(); return ret; } @@ -1707,6 +1780,7 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set, kfree(rcu_dereference_raw(h->rnets[--i])); goto free_hregion; } + seqcount_spinlock_init(&nets->seq, &set->lock); RCU_INIT_POINTER(h->rnets[i], nets); } #endif diff --git a/net/netfilter/ipset/ip_set_hash_netiface.c b/net/netfilter/ipset/ip_set_hash_netiface.c index b44b95f766b7..b602cc43565d 100644 --- a/net/netfilter/ipset/ip_set_hash_netiface.c +++ b/net/netfilter/ipset/ip_set_hash_netiface.c @@ -38,7 +38,6 @@ MODULE_ALIAS("ip_set_hash:net,iface"); #define HTYPE hash_netiface #define IP_SET_HASH_WITH_NETS #define IP_SET_HASH_WITH_MULTI -#define IP_SET_HASH_WITH_NET0 #define STRSCPY(a, b) strscpy(a, b, IFNAMSIZ) diff --git a/net/netfilter/ipset/ip_set_hash_netportnet.c b/net/netfilter/ipset/ip_set_hash_netportnet.c index 6291532be7a5..61af1ce27127 100644 --- a/net/netfilter/ipset/ip_set_hash_netportnet.c +++ b/net/netfilter/ipset/ip_set_hash_netportnet.c @@ -36,7 +36,6 @@ MODULE_ALIAS("ip_set_hash:net,port,net"); #define IP_SET_HASH_WITH_PROTO #define IP_SET_HASH_WITH_NETS #define IPSET_NET_COUNT 2 -#define IP_SET_HASH_WITH_NET0 /* IPv4 variant */ diff --git a/net/netfilter/ipvs/ip_vs_ftp.c b/net/netfilter/ipvs/ip_vs_ftp.c index b315c608fda4..9e3e005a8263 100644 --- a/net/netfilter/ipvs/ip_vs_ftp.c +++ b/net/netfilter/ipvs/ip_vs_ftp.c @@ -102,7 +102,7 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, char *s, c; unsigned char p[6]; char edelim; - __u16 hport; + __u32 hport; int i = 0; if (data_limit - data < plen) { @@ -144,7 +144,11 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, return -1; c = *data; if (isdigit(c)) { - p[i] = p[i]*10 + c - '0'; + unsigned int val = p[i] * 10 + c - '0'; + + if (val > 255) + return -1; + p[i] = val; } else if (c == ',' && i < 5) { i++; p[i] = 0; @@ -222,6 +226,8 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, if (!isdigit(*s)) break; hport = hport * 10 + *s - '0'; + if (hport > 65535) + return -1; } if (s == data_limit || !hport || *s != edelim) return -1; diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c index 85487f92af50..4292c3d9addd 100644 --- a/net/netfilter/nf_conncount.c +++ b/net/netfilter/nf_conncount.c @@ -251,7 +251,8 @@ check_connections: list->last_gc_count = list->count; add_new_node: - if (WARN_ON_ONCE(list->count > INT_MAX)) { + if (unlikely(list->count > INT_MAX)) { + DEBUG_NET_WARN_ON_ONCE(1); err = -EOVERFLOW; goto out_put; } diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index f1f0c582db5d..06242c86e1dc 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -477,6 +477,11 @@ static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, lockdep_nfct_expect_lock_held(); + if (expect->flags & NF_CT_EXPECT_DEAD) { + DEBUG_NET_WARN_ON_ONCE(1); + return -EINVAL; + } + h = nf_ct_expect_dst_hash(net, &expect->tuple); hlist_for_each_entry_safe(i, next, &nf_ct_expect_hash[h], hnode) { if (nf_ct_exp_is_expired(i)) { @@ -528,12 +533,6 @@ int nf_ct_expect_related_report(struct nf_conntrack_expect *expect, int ret; spin_lock_bh(&nf_conntrack_expect_lock); - if (expect->flags & NF_CT_EXPECT_DEAD) { - DEBUG_NET_WARN_ON_ONCE(1); - ret = -EINVAL; - goto out; - } - master_help = nfct_help(expect->master); if (!master_help) { ret = -ESHUTDOWN; diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index fc3f60099af3..9b4e29557ec3 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -3042,7 +3042,7 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, #endif if (nla_put_be32(skb, CTA_EXPECT_TIMEOUT, htonl(timeout)) || nla_put_be32(skb, CTA_EXPECT_ID, nf_expect_get_id(exp)) || - nla_put_be32(skb, CTA_EXPECT_FLAGS, htonl(exp->flags)) || + nla_put_be32(skb, CTA_EXPECT_FLAGS, htonl(exp->flags & NF_CT_EXPECT_MASK)) || nla_put_be32(skb, CTA_EXPECT_CLASS, htonl(exp->class))) goto nla_put_failure; diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index af357f6c5070..765a92fa90d6 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -595,10 +595,15 @@ static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *tr static void nft_trans_commit_list_add_elem(struct net *net, struct nft_trans *trans) { struct nftables_pernet *nft_net = nft_pernet(net); + struct nft_trans_elem *te; WARN_ON_ONCE(trans->msg_type != NFT_MSG_NEWSETELEM && trans->msg_type != NFT_MSG_DELSETELEM); + te = nft_trans_container_elem(trans); + if (te->set->ops->commit && list_empty(&te->set->pending_update)) + list_add_tail(&te->set->pending_update, &nft_net->set_update_list); + if (nft_trans_try_collapse(nft_net, trans)) { kfree(trans); return; @@ -6916,39 +6921,11 @@ static void nft_trans_elems_destroy(const struct nft_ctx *ctx, nf_tables_set_elem_destroy(ctx, te->set, te->elems[i].priv); } -int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set, - struct nft_expr *expr_array[]) -{ - struct nft_expr *expr; - int err, i, k; - - for (i = 0; i < set->num_exprs; i++) { - expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT); - if (!expr) - goto err_expr; - - err = nft_expr_clone(expr, set->exprs[i], GFP_KERNEL_ACCOUNT); - if (err < 0) { - kfree(expr); - goto err_expr; - } - expr_array[i] = expr; - } - - return 0; - -err_expr: - for (k = i - 1; k >= 0; k--) - nft_expr_destroy(ctx, expr_array[k]); - - return -ENOMEM; -} - static int nft_set_elem_expr_setup(struct nft_ctx *ctx, const struct nft_set_ext_tmpl *tmpl, const struct nft_set_ext *ext, struct nft_expr *expr_array[], - u32 num_exprs) + u32 num_exprs, bool override_exprs) { struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext); u32 len = sizeof(struct nft_set_elem_expr); @@ -6971,7 +6948,8 @@ static int nft_set_elem_expr_setup(struct nft_ctx *ctx, goto err_elem_expr_setup; elem_expr->size += expr_array[i]->ops->size; - nft_expr_destroy(ctx, expr_array[i]); + if (override_exprs) + nft_expr_destroy(ctx, expr_array[i]); expr_array[i] = NULL; } @@ -6979,7 +6957,9 @@ static int nft_set_elem_expr_setup(struct nft_ctx *ctx, err_elem_expr_setup: for (; i < num_exprs; i++) { - nft_expr_destroy(ctx, expr_array[i]); + if (override_exprs) + nft_expr_destroy(ctx, expr_array[i]); + expr_array[i] = NULL; } @@ -7186,16 +7166,19 @@ static void nft_setelem_remove(const struct net *net, } static void nft_trans_elems_remove(const struct nft_ctx *ctx, - const struct nft_trans_elem *te) + const struct nft_trans_elem *te, + bool notify) { int i; for (i = 0; i < te->nelems; i++) { WARN_ON_ONCE(te->elems[i].update); - nf_tables_setelem_notify(ctx, te->set, - te->elems[i].priv, - te->nft_trans.msg_type); + if (notify) { + nf_tables_setelem_notify(ctx, te->set, + te->elems[i].priv, + te->nft_trans.msg_type); + } nft_setelem_remove(ctx->net, te->set, te->elems[i].priv); if (!nft_setelem_is_catchall(te->set, te->elems[i].priv)) { @@ -7205,6 +7188,20 @@ static void nft_trans_elems_remove(const struct nft_ctx *ctx, } } +static void nft_trans_elems_remove_notify(const struct nft_ctx *ctx, + const struct nft_trans_elem *te) +{ + int i; + + for (i = 0; i < te->nelems; i++) { + WARN_ON_ONCE(te->elems[i].update); + + nf_tables_setelem_notify(ctx, te->set, + te->elems[i].priv, + te->nft_trans.msg_type); + } +} + static bool nft_setelem_valid_key_end(const struct nft_set *set, struct nlattr **nla, u32 flags) { @@ -7258,6 +7255,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, struct nft_set_binding *binding; struct nft_elem_priv *elem_priv; struct nft_object *obj = NULL; + bool override_exprs = false; struct nft_userdata *udata; struct nft_data_desc desc; enum nft_registers dreg; @@ -7363,6 +7361,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, expr_array[0] = expr; num_exprs = 1; + override_exprs = true; if (set->num_exprs && set->exprs[0]->ops != expr->ops) { err = -EOPNOTSUPP; @@ -7391,6 +7390,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, } expr_array[i] = expr; num_exprs++; + override_exprs = true; if (set->num_exprs && expr->ops != set->exprs[i]->ops) { err = -EOPNOTSUPP; @@ -7404,9 +7404,8 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, } } else if (set->num_exprs > 0 && !(flags & NFT_SET_ELEM_INTERVAL_END)) { - err = nft_set_elem_expr_clone(ctx, set, expr_array); - if (err < 0) - goto err_set_elem_expr_clone; + for (i = 0; i < set->num_exprs; i++) + expr_array[i] = set->exprs[i]; num_exprs = set->num_exprs; } @@ -7545,7 +7544,8 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, udata->len = ulen - 1; nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen); } - err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs); + err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs, + override_exprs); if (err < 0) goto err_elem_free; @@ -7653,9 +7653,11 @@ err_parse_key_end: err_parse_key: nft_data_release(&elem.key.val, NFT_DATA_VALUE); err_set_elem_expr: - for (i = 0; i < num_exprs && expr_array[i]; i++) - nft_expr_destroy(ctx, expr_array[i]); -err_set_elem_expr_clone: + if (override_exprs) { + for (i = 0; i < num_exprs && expr_array[i]; i++) + nft_expr_destroy(ctx, expr_array[i]); + } + return err; } @@ -8715,18 +8717,17 @@ static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info, return nft_delobj(&ctx, obj); } -static void -__nft_obj_notify(struct net *net, const struct nft_table *table, - struct nft_object *obj, u32 portid, u32 seq, int event, - u16 flags, int family, int report, gfp_t gfp) +static struct sk_buff * +nft_obj_notify_alloc(struct net *net, const struct nft_table *table, + struct nft_object *obj, u32 portid, u32 seq, int event, + u16 flags, int family, int report, gfp_t gfp) { - struct nftables_pernet *nft_net = nft_pernet(net); struct sk_buff *skb; int err; if (!report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES)) - return; + return NULL; skb = nlmsg_new(NLMSG_GOODSIZE, gfp); if (skb == NULL) @@ -8740,10 +8741,10 @@ __nft_obj_notify(struct net *net, const struct nft_table *table, goto err; } - nft_notify_enqueue(skb, report, &nft_net->notify_list); - return; + return skb; err: nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS); + return NULL; } void nft_obj_notify(struct net *net, const struct nft_table *table, @@ -8752,6 +8753,7 @@ void nft_obj_notify(struct net *net, const struct nft_table *table, { char *buf = kasprintf(gfp, "%s:%u", table->name, nft_base_seq(net)); + struct sk_buff *skb; audit_log_nfcfg(buf, family, @@ -8762,17 +8764,27 @@ void nft_obj_notify(struct net *net, const struct nft_table *table, gfp); kfree(buf); - __nft_obj_notify(net, table, obj, portid, seq, event, - flags, family, report, gfp); + /* Called from the packet path, holding no mutex: notify_list is + * serialised by commit_mutex, so send this notification directly. + */ + skb = nft_obj_notify_alloc(net, table, obj, portid, seq, event, + flags, family, report, gfp); + if (skb) + nfnetlink_send(skb, net, portid, NFNLGRP_NFTABLES, report, gfp); } EXPORT_SYMBOL_GPL(nft_obj_notify); static void nf_tables_obj_notify(const struct nft_ctx *ctx, struct nft_object *obj, int event) { - __nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, - ctx->seq, event, ctx->flags, ctx->family, - ctx->report, GFP_KERNEL); + struct nftables_pernet *nft_net = nft_pernet(ctx->net); + struct sk_buff *skb; + + skb = nft_obj_notify_alloc(ctx->net, ctx->table, obj, ctx->portid, + ctx->seq, event, ctx->flags, ctx->family, + ctx->report, GFP_KERNEL); + if (skb) + nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list); } /* @@ -10848,11 +10860,35 @@ static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation) } } -static void nft_set_commit_update(struct list_head *set_update_list) +static void nft_set_commit_update(struct nft_ctx *ctx, + struct nftables_pernet *nft_net) { struct nft_set *set, *next; + struct nft_trans_elem *te; + struct nft_trans *trans; + + if (list_empty(&nft_net->set_update_list)) + return; - list_for_each_entry_safe(set, next, set_update_list, pending_update) { + list_for_each_entry(trans, &nft_net->commit_list, list) { + nft_ctx_update(ctx, trans); + + switch (trans->msg_type) { + case NFT_MSG_DELSET: + case NFT_MSG_DESTROYSET: + nft_trans_set(trans)->dead = 1; + break; + case NFT_MSG_DELSETELEM: + te = nft_trans_container_elem(trans); + if (!te->set->ops->commit) + break; + + nft_trans_elems_remove(ctx, te, false); + break; + } + } + + list_for_each_entry_safe(set, next, &nft_net->set_update_list, pending_update) { list_del_init(&set->pending_update); if (!set->ops->commit || set->dead) @@ -10885,7 +10921,6 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) struct nft_trans_binding *trans_binding; struct nft_trans *trans, *next; unsigned int base_seq, gc_seq; - LIST_HEAD(set_update_list); struct nft_trans_elem *te; struct nft_chain *chain; struct nft_table *table; @@ -10931,10 +10966,6 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) return -EAGAIN; } - err = nft_flow_rule_offload_commit(net); - if (err < 0) - return err; - /* 1. Allocate space for next generation rules_gen_X[] */ list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) { struct nft_table *table = trans->table; @@ -10959,7 +10990,19 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) } } + /* must be last, so audit and chain blob set up does not leave hardware + * in consistent state. + */ + err = nft_flow_rule_offload_commit(net); + if (err < 0) { + nf_tables_commit_chain_prepare_cancel(net); + nf_tables_commit_audit_free(&adl); + return err; + } + /* step 2. Make rules_gen_X visible to packet path */ + nft_set_commit_update(&ctx, nft_net); + list_for_each_entry(table, &nft_net->tables, list) { list_for_each_entry(chain, &table->chains, list) nf_tables_commit_chain(net, chain); @@ -11091,27 +11134,16 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) break; case NFT_MSG_NEWSETELEM: te = nft_trans_container_elem(trans); - nft_trans_elems_add(&ctx, te); - - if (te->set->ops->commit && - list_empty(&te->set->pending_update)) { - list_add_tail(&te->set->pending_update, - &set_update_list); - } nft_trans_destroy(trans); break; case NFT_MSG_DELSETELEM: case NFT_MSG_DESTROYSETELEM: te = nft_trans_container_elem(trans); - - nft_trans_elems_remove(&ctx, te); - - if (te->set->ops->commit && - list_empty(&te->set->pending_update)) { - list_add_tail(&te->set->pending_update, - &set_update_list); - } + if (te->set->ops->commit) + nft_trans_elems_remove_notify(&ctx, te); + else + nft_trans_elems_remove(&ctx, te, true); break; case NFT_MSG_NEWOBJ: if (nft_trans_obj_update(trans)) { @@ -11180,8 +11212,6 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) } } - nft_set_commit_update(&set_update_list); - nft_commit_notify(net, NETLINK_CB(skb).portid); nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN); nf_tables_commit_audit_log(&adl, nft_base_seq(net)); @@ -11247,11 +11277,11 @@ static void nf_tables_abort_release(struct nft_trans *trans) kfree(trans); } -static void nft_set_abort_update(struct list_head *set_update_list) +static void nft_set_abort_update(struct nftables_pernet *nft_net) { struct nft_set *set, *next; - list_for_each_entry_safe(set, next, set_update_list, pending_update) { + list_for_each_entry_safe(set, next, &nft_net->set_update_list, pending_update) { list_del_init(&set->pending_update); if (!set->ops->abort) @@ -11265,7 +11295,6 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action) { struct nftables_pernet *nft_net = nft_pernet(net); struct nft_trans *trans, *next; - LIST_HEAD(set_update_list); struct nft_trans_elem *te; struct nft_ctx ctx = { .net = net, @@ -11386,33 +11415,22 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action) nft_trans_destroy(trans); break; case NFT_MSG_NEWSETELEM: + te = nft_trans_container_elem(trans); if (nft_trans_elem_set_bound(trans)) { + list_del_init(&te->set->pending_update); nft_trans_destroy(trans); break; } - te = nft_trans_container_elem(trans); if (!nft_trans_elems_new_abort(&ctx, te)) { nft_trans_destroy(trans); break; } - - if (te->set->ops->abort && - list_empty(&te->set->pending_update)) { - list_add_tail(&te->set->pending_update, - &set_update_list); - } break; case NFT_MSG_DELSETELEM: case NFT_MSG_DESTROYSETELEM: te = nft_trans_container_elem(trans); nft_trans_elems_destroy_abort(&ctx, te); - - if (te->set->ops->abort && - list_empty(&te->set->pending_update)) { - list_add_tail(&te->set->pending_update, - &set_update_list); - } nft_trans_destroy(trans); break; case NFT_MSG_NEWOBJ: @@ -11458,7 +11476,7 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action) WARN_ON_ONCE(!list_empty(&nft_net->commit_set_list)); - nft_set_abort_update(&set_update_list); + nft_set_abort_update(nft_net); synchronize_rcu(); @@ -12142,6 +12160,7 @@ static int __net_init nf_tables_init_net(struct net *net) INIT_LIST_HEAD(&nft_net->binding_list); INIT_LIST_HEAD(&nft_net->module_list); INIT_LIST_HEAD(&nft_net->notify_list); + INIT_LIST_HEAD(&nft_net->set_update_list); mutex_init(&nft_net->commit_mutex); net->nft.base_seq = 1; nft_net->gc_seq = 0; @@ -12186,6 +12205,7 @@ static void __net_exit nf_tables_exit_net(struct net *net) WARN_ON_ONCE(!list_empty(&nft_net->module_list)); WARN_ON_ONCE(!list_empty(&nft_net->notify_list)); WARN_ON_ONCE(!list_empty(&nft_net->destroy_list)); + WARN_ON_ONCE(!list_empty(&nft_net->set_update_list)); } static void nf_tables_exit_batch(struct list_head *net_exit_list) diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index b8aaf39cb4d8..c727668b0c5b 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -28,10 +28,17 @@ #include <linux/netfilter/nfnetlink.h> #include <linux/netfilter/nfnetlink_queue.h> #include <linux/netfilter/nf_conntrack_common.h> +#include <linux/icmp.h> +#include <linux/icmpv6.h> +#include <linux/ip.h> #include <linux/list.h> +#include <linux/sctp.h> #include <linux/cgroup-defs.h> #include <linux/rhashtable.h> #include <linux/jhash.h> +#include <linux/tcp.h> +#include <linux/udp.h> +#include <net/gre.h> #include <net/gso.h> #include <net/sock.h> #include <net/tcp_states.h> @@ -1206,10 +1213,62 @@ static bool nfqnl_validate_ipopts(const struct iphdr *iph_new, return memcmp(iph_new + 1, ip_hdr(e->skb) + 1, ihl - sizeof(*iph_orig)) == 0; } +static bool nfqnl_validate_l4(const u8 *data, unsigned int data_len, + const struct nf_queue_entry *e, u8 proto, + bool fragment) +{ +#if IS_ENABLED(CONFIG_NF_CONNTRACK) + enum ip_conntrack_info ctinfo; + const struct nf_conn *ct; + + ct = nf_ct_get(e->skb, &ctinfo); + if (ct && !nf_ct_is_template(ct)) { + if (fragment || nf_ct_protonum(ct) != proto) + return false; + } +#endif + + if (fragment) + return true; + + switch (proto) { + case IPPROTO_TCP: { + const struct tcphdr *th = (const struct tcphdr *)data; + unsigned int thlen; + + if (data_len < sizeof(*th)) + return false; + + thlen = __tcp_hdrlen(th); + if (thlen < sizeof(*th) || data_len < thlen) + return false; + + return true; + } + case IPPROTO_UDP: + return data_len >= sizeof(struct udphdr); + case IPPROTO_ICMP: + return data_len >= sizeof(struct icmphdr); + case IPPROTO_ICMPV6: + return data_len >= sizeof(struct icmp6hdr); + case IPPROTO_SCTP: + return data_len >= sizeof(struct sctphdr); + case IPPROTO_GRE: + return data_len >= sizeof(struct gre_base_hdr); + case IPPROTO_AH: + return data_len >= sizeof(struct ip_auth_hdr); + case IPPROTO_ESP: + return data_len >= sizeof(struct ip_esp_hdr); + } + + return true; +} + static bool nfqnl_validate_ip4(const struct iphdr *iph, unsigned int data_len, const struct nf_queue_entry *e) { unsigned int ihl; + bool fragment; if (data_len < sizeof(*iph)) return false; @@ -1226,10 +1285,14 @@ static bool nfqnl_validate_ip4(const struct iphdr *iph, unsigned int data_len, if (ntohs(iph->tot_len) != data_len) return false; + fragment = iph->frag_off & htons(IP_MF | IP_OFFSET); + /* support for ipopts mangling would require * recompile + skb transport header update. */ - return nfqnl_validate_ipopts(iph, e); + return nfqnl_validate_ipopts(iph, e) && + nfqnl_validate_l4((const u8 *)iph + ihl, data_len - ihl, e, + iph->protocol, fragment); } static bool nfqnl_validate_one_exthdr(const u8 *data, @@ -1273,6 +1336,7 @@ static bool nfqnl_validate_exthdr(const struct ipv6hdr *ip6_new, const u8 *data = (const u8 *)ip6_new; u8 orig_nexthdr = ip6_orig->nexthdr; u8 new_nexthdr = ip6_new->nexthdr; + bool fragment = false; if (new_nexthdr != orig_nexthdr) return false; @@ -1286,7 +1350,8 @@ static bool nfqnl_validate_exthdr(const struct ipv6hdr *ip6_new, int hdrlen; if (orig_nexthdr == NEXTHDR_NONE) - return true; + return nfqnl_validate_l4(data, data_len, e, + new_nexthdr, fragment); if (unlikely(exthdr_cnt++ >= IP6_MAX_EXT_HDRS_CNT)) return false; @@ -1297,6 +1362,7 @@ static bool nfqnl_validate_exthdr(const struct ipv6hdr *ip6_new, switch (orig_nexthdr) { case NEXTHDR_FRAGMENT: + fragment = true; hdrlen = sizeof(struct frag_hdr); break; case NEXTHDR_AUTH: @@ -1323,7 +1389,7 @@ static bool nfqnl_validate_exthdr(const struct ipv6hdr *ip6_new, data += hdrlen; } - return true; + return nfqnl_validate_l4(data, data_len, e, new_nexthdr, fragment); } static bool nfqnl_validate_ip6(const struct ipv6hdr *ip6, unsigned int data_len, diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c index ee9d3e7b1ecf..fa4da694e92d 100644 --- a/net/netfilter/nft_dynset.c +++ b/net/netfilter/nft_dynset.c @@ -19,7 +19,8 @@ struct nft_dynset { u8 sreg_key; u8 sreg_data; bool invert; - bool expr; + bool expr:1, + override_exprs:1; u8 num_exprs; u64 timeout; struct nft_expr *expr_array[NFT_SET_EXPR_MAX]; @@ -257,6 +258,7 @@ static int nft_dynset_init(const struct nft_ctx *ctx, priv->num_exprs++; priv->expr_array[0] = dynset_expr; + priv->override_exprs = true; if (set->num_exprs > 1 || (set->num_exprs == 1 && @@ -289,6 +291,7 @@ static int nft_dynset_init(const struct nft_ctx *ctx, } priv->expr_array[i] = dynset_expr; priv->num_exprs++; + priv->override_exprs = true; if (set->num_exprs) { if (i >= set->num_exprs) { @@ -307,9 +310,8 @@ static int nft_dynset_init(const struct nft_ctx *ctx, goto err_expr_free; } } else if (set->num_exprs > 0) { - err = nft_set_elem_expr_clone(ctx, set, priv->expr_array); - if (err < 0) - return err; + for (i = 0; i < set->num_exprs; i++) + priv->expr_array[i] = set->exprs[i]; priv->num_exprs = set->num_exprs; } @@ -339,8 +341,10 @@ static int nft_dynset_init(const struct nft_ctx *ctx, return 0; err_expr_free: - for (i = 0; i < priv->num_exprs; i++) - nft_expr_destroy(ctx, priv->expr_array[i]); + if (priv->override_exprs) { + for (i = 0; i < priv->num_exprs; i++) + nft_expr_destroy(ctx, priv->expr_array[i]); + } return err; } @@ -367,9 +371,10 @@ static void nft_dynset_destroy(const struct nft_ctx *ctx, struct nft_dynset *priv = nft_expr_priv(expr); int i; - for (i = 0; i < priv->num_exprs; i++) - nft_expr_destroy(ctx, priv->expr_array[i]); - + if (priv->override_exprs) { + for (i = 0; i < priv->num_exprs; i++) + nft_expr_destroy(ctx, priv->expr_array[i]); + } nf_tables_destroy_set(ctx, priv->set); } @@ -393,7 +398,7 @@ static int nft_dynset_dump(struct sk_buff *skb, nf_jiffies64_to_msecs(priv->timeout), NFTA_DYNSET_PAD)) goto nla_put_failure; - if (priv->set->num_exprs == 0) { + if (priv->set->num_exprs == 0 || priv->override_exprs) { if (priv->num_exprs == 1) { if (nft_expr_dump(skb, NFTA_DYNSET_EXPR, priv->expr_array[0], reset)) diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c index 8a4472fd77d9..e315d35f73d4 100644 --- a/net/netfilter/nft_payload.c +++ b/net/netfilter/nft_payload.c @@ -1067,6 +1067,17 @@ static bool nft_payload_csum_write_ok(const struct nft_pktinfo *pkt, return false; } +static bool nft_th_write_ok(const struct nft_pktinfo *pkt, + const struct nft_payload_set *priv) +{ + unsigned int doff = offsetof(struct tcphdr, ack_seq) + sizeof(__be32); + + if (pkt->tprot != IPPROTO_TCP) + return true; + + return priv->offset > doff || priv->offset + priv->len <= doff; +} + static void nft_payload_set_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt) @@ -1105,6 +1116,8 @@ static void nft_payload_set_eval(const struct nft_expr *expr, case NFT_PAYLOAD_TRANSPORT_HEADER: if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff) goto err; + if (!nft_th_write_ok(pkt, priv)) + goto err; offset = nft_thoff(pkt); break; case NFT_PAYLOAD_INNER_HEADER: diff --git a/net/netfilter/nft_set_pipapo_avx2.c b/net/netfilter/nft_set_pipapo_avx2.c index b3f105520a85..21f5be68c703 100644 --- a/net/netfilter/nft_set_pipapo_avx2.c +++ b/net/netfilter/nft_set_pipapo_avx2.c @@ -1134,6 +1134,7 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m, struct nft_pipapo_scratch *scratch; const struct nft_pipapo_field *f; unsigned long *res, *fill, *map; + struct nft_pipapo_elem *e; bool map_index; int ret = 0; int i; @@ -1207,14 +1208,11 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m, next_match: if (ret < 0) { scratch->map_index = map_index; - kernel_fpu_end(); - __local_unlock_nested_bh(&scratch->bh_lock); - return NULL; + e = NULL; + goto out; } if (last) { - struct nft_pipapo_elem *e; - e = f->mt[ret].e; if (unlikely(__nft_set_elem_expired(&e->ext, tstamp) || !nft_set_elem_active(&e->ext, genmask))) { @@ -1224,9 +1222,7 @@ next_match: } scratch->map_index = map_index; - kernel_fpu_end(); - __local_unlock_nested_bh(&scratch->bh_lock); - return e; + goto out; } map_index = !map_index; @@ -1234,9 +1230,12 @@ next_match: data += NFT_PIPAPO_GROUPS_PADDED_SIZE(f); } + e = NULL; +out: + asm volatile("vzeroupper"); kernel_fpu_end(); __local_unlock_nested_bh(&scratch->bh_lock); - return NULL; + return e; } /** diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c index bfcf2d44e93d..fe7d8d19629b 100644 --- a/net/netfilter/xt_IDLETIMER.c +++ b/net/netfilter/xt_IDLETIMER.c @@ -102,8 +102,6 @@ static void idletimer_tg_expired(struct timer_list *t) { struct idletimer_tg *timer = timer_container_of(timer, t, timer); - pr_debug("timer %s expired\n", timer->attr.attr.name); - schedule_work(&timer->work); } @@ -111,7 +109,6 @@ static void idletimer_tg_alarmproc(struct alarm *alarm, ktime_t now) { struct idletimer_tg *timer = alarm->data; - pr_debug("alarm %s expired\n", timer->attr.attr.name); schedule_work(&timer->work); } @@ -171,7 +168,7 @@ static int idletimer_tg_create(struct idletimer_tg_info *info) ret = sysfs_create_file(idletimer_tg_kobj, &info->timer->attr.attr); if (ret < 0) { - pr_debug("couldn't add file to sysfs"); + pr_info_ratelimited("couldn't add file to sysfs\n"); goto out_free_attr; } @@ -220,7 +217,7 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info) ret = sysfs_create_file(idletimer_tg_kobj, &info->timer->attr.attr); if (ret < 0) { - pr_debug("couldn't add file to sysfs"); + pr_info_ratelimited("couldn't add file to sysfs\n"); goto out_free_attr; } @@ -228,7 +225,6 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info) kobject_uevent(idletimer_tg_kobj,KOBJ_ADD); list_add(&info->timer->entry, &idletimer_tg_list); - pr_debug("timer type value is %u", info->timer_type); info->timer->timer_type = info->timer_type; info->timer->refcnt = 1; @@ -263,9 +259,6 @@ static unsigned int idletimer_tg_target(struct sk_buff *skb, { const struct idletimer_tg_info *info = par->targinfo; - pr_debug("resetting timer %s, timeout period %u\n", - info->label, info->timeout); - mod_timer(&info->timer->timer, secs_to_jiffies(info->timeout) + jiffies); @@ -280,9 +273,6 @@ static unsigned int idletimer_tg_target_v1(struct sk_buff *skb, { const struct idletimer_tg_info_v1 *info = par->targinfo; - pr_debug("resetting timer %s, timeout period %u\n", - info->label, info->timeout); - if (info->timer->timer_type & XT_IDLETIMER_ALARM) { idletimer_start_alarm_sec(info->timer, info->timeout); } else { @@ -296,17 +286,17 @@ static unsigned int idletimer_tg_target_v1(struct sk_buff *skb, static int idletimer_tg_helper(struct idletimer_tg_info *info) { if (info->timeout == 0) { - pr_debug("timeout value is zero\n"); + pr_info_ratelimited("timeout value is zero\n"); return -EINVAL; } if (info->timeout >= INT_MAX / 1000) { - pr_debug("timeout value is too big\n"); + pr_info_ratelimited("timeout value is too big\n"); return -EINVAL; } if (info->label[0] == '\0' || strnlen(info->label, MAX_IDLETIMER_LABEL_SIZE) == MAX_IDLETIMER_LABEL_SIZE) { - pr_debug("label is empty or not nul-terminated\n"); + pr_info_ratelimited("label is empty or not nul-terminated\n"); return -EINVAL; } return 0; @@ -318,34 +308,25 @@ static int idletimer_tg_checkentry(const struct xt_tgchk_param *par) struct idletimer_tg_info *info = par->targinfo; int ret; - pr_debug("checkentry targinfo%s\n", info->label); - ret = idletimer_tg_helper(info); if(ret < 0) - { - pr_debug("checkentry helper return invalid\n"); return -EINVAL; - } mutex_lock(&list_mutex); info->timer = __idletimer_tg_find_by_label(info->label); if (info->timer) { if (info->timer->timer_type & XT_IDLETIMER_ALARM) { - pr_debug("Adding/Replacing rule with same label and different timer type is not allowed\n"); mutex_unlock(&list_mutex); + pr_info_ratelimited("Adding/Replacing rule with same label and different timer type is not allowed\n"); return -EINVAL; } info->timer->refcnt++; mod_timer(&info->timer->timer, secs_to_jiffies(info->timeout) + jiffies); - - pr_debug("increased refcnt of timer %s to %u\n", - info->label, info->timer->refcnt); } else { ret = idletimer_tg_create(info); if (ret < 0) { - pr_debug("failed to create timer\n"); mutex_unlock(&list_mutex); return ret; } @@ -360,30 +341,23 @@ static int idletimer_tg_checkentry_v1(const struct xt_tgchk_param *par) struct idletimer_tg_info_v1 *info = par->targinfo; int ret; - pr_debug("checkentry targinfo%s\n", info->label); - if (info->send_nl_msg) return -EOPNOTSUPP; ret = idletimer_tg_helper((struct idletimer_tg_info *)info); if(ret < 0) - { - pr_debug("checkentry helper return invalid\n"); return -EINVAL; - } - if (info->timer_type > XT_IDLETIMER_ALARM) { - pr_debug("invalid value for timer type\n"); + if (info->timer_type > XT_IDLETIMER_ALARM) return -EINVAL; - } mutex_lock(&list_mutex); info->timer = __idletimer_tg_find_by_label(info->label); if (info->timer) { if (info->timer->timer_type != info->timer_type) { - pr_debug("Adding/Replacing rule with same label and different timer type is not allowed\n"); mutex_unlock(&list_mutex); + pr_info_ratelimited("Adding/Replacing rule with same label and different timer type is not allowed\n"); return -EINVAL; } @@ -393,21 +367,15 @@ static int idletimer_tg_checkentry_v1(const struct xt_tgchk_param *par) ktime_t tout = alarm_expires_remaining(&info->timer->alarm); struct timespec64 ktimespec = ktime_to_timespec64(tout); - if (ktimespec.tv_sec > 0) { - pr_debug("time_expiry_remaining %lld\n", - ktimespec.tv_sec); + if (ktimespec.tv_sec > 0) idletimer_start_alarm_ktime(info->timer, tout); - } } else { mod_timer(&info->timer->timer, secs_to_jiffies(info->timeout) + jiffies); } - pr_debug("increased refcnt of timer %s to %u\n", - info->label, info->timer->refcnt); } else { ret = idletimer_tg_create_v1(info); if (ret < 0) { - pr_debug("failed to create timer\n"); mutex_unlock(&list_mutex); return ret; } @@ -421,19 +389,13 @@ static void idletimer_tg_destroy(const struct xt_tgdtor_param *par) { const struct idletimer_tg_info *info = par->targinfo; - pr_debug("destroy targinfo %s\n", info->label); - mutex_lock(&list_mutex); if (--info->timer->refcnt > 0) { - pr_debug("decreased refcnt of timer %s to %u\n", - info->label, info->timer->refcnt); mutex_unlock(&list_mutex); return; } - pr_debug("deleting timer %s\n", info->label); - list_del(&info->timer->entry); mutex_unlock(&list_mutex); @@ -448,19 +410,13 @@ static void idletimer_tg_destroy_v1(const struct xt_tgdtor_param *par) { const struct idletimer_tg_info_v1 *info = par->targinfo; - pr_debug("destroy targinfo %s\n", info->label); - mutex_lock(&list_mutex); if (--info->timer->refcnt > 0) { - pr_debug("decreased refcnt of timer %s to %u\n", - info->label, info->timer->refcnt); mutex_unlock(&list_mutex); return; } - pr_debug("deleting timer %s\n", info->label); - list_del(&info->timer->entry); mutex_unlock(&list_mutex); @@ -534,7 +490,7 @@ static int __init idletimer_tg_init(void) idletimer_tg_class = class_create("xt_idletimer"); err = PTR_ERR(idletimer_tg_class); if (IS_ERR(idletimer_tg_class)) { - pr_debug("couldn't register device class\n"); + pr_err("couldn't register device class\n"); goto out; } @@ -542,7 +498,7 @@ static int __init idletimer_tg_init(void) MKDEV(0, 0), NULL, "timers"); err = PTR_ERR(idletimer_tg_device); if (IS_ERR(idletimer_tg_device)) { - pr_debug("couldn't register system device\n"); + pr_err("couldn't register system device\n"); goto out_class; } @@ -551,7 +507,7 @@ static int __init idletimer_tg_init(void) err = xt_register_targets(idletimer_tg, ARRAY_SIZE(idletimer_tg)); if (err < 0) { - pr_debug("couldn't register xt target\n"); + pr_err("couldn't register xt target\n"); goto out_dev; } diff --git a/net/netfilter/xt_LOG.c b/net/netfilter/xt_LOG.c index f39244f9c0ed..de3f176792a0 100644 --- a/net/netfilter/xt_LOG.c +++ b/net/netfilter/xt_LOG.c @@ -50,12 +50,12 @@ static int log_tg_check(const struct xt_tgchk_param *par) return -EINVAL; if (loginfo->level >= 8) { - pr_debug("level %u >= 8\n", loginfo->level); + pr_info_ratelimited("level %u >= 8\n", loginfo->level); return -EINVAL; } if (loginfo->prefix[sizeof(loginfo->prefix)-1] != '\0') { - pr_debug("prefix is not null-terminated\n"); + pr_info_ratelimited("prefix is not null-terminated\n"); return -EINVAL; } diff --git a/net/netfilter/xt_MASQUERADE.c b/net/netfilter/xt_MASQUERADE.c index eae05c178336..cea488cec544 100644 --- a/net/netfilter/xt_MASQUERADE.c +++ b/net/netfilter/xt_MASQUERADE.c @@ -21,11 +21,11 @@ static int masquerade_tg_check(const struct xt_tgchk_param *par) const struct nf_nat_ipv4_multi_range_compat *mr = par->targinfo; if (mr->range[0].flags & NF_NAT_RANGE_MAP_IPS) { - pr_debug("bad MAP_IPS.\n"); + pr_info_ratelimited("bad MAP_IPS.\n"); return -EINVAL; } if (mr->rangesize != 1) { - pr_debug("bad rangesize %u\n", mr->rangesize); + pr_info_ratelimited("bad rangesize %u\n", mr->rangesize); return -EINVAL; } return nf_ct_netns_get(par->net, par->family); diff --git a/net/netfilter/xt_NETMAP.c b/net/netfilter/xt_NETMAP.c index cb2ee80d84fa..7da065a472e0 100644 --- a/net/netfilter/xt_NETMAP.c +++ b/net/netfilter/xt_NETMAP.c @@ -4,6 +4,8 @@ * Copyright (c) 2011 Patrick McHardy <kaber@trash.net> */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/ip.h> #include <linux/kernel.h> #include <linux/module.h> @@ -106,11 +108,11 @@ static int netmap_tg4_check(const struct xt_tgchk_param *par) const struct nf_nat_ipv4_multi_range_compat *mr = par->targinfo; if (!(mr->range[0].flags & NF_NAT_RANGE_MAP_IPS)) { - pr_debug("bad MAP_IPS.\n"); + pr_info_ratelimited("bad MAP_IPS.\n"); return -EINVAL; } if (mr->rangesize != 1) { - pr_debug("bad rangesize %u.\n", mr->rangesize); + pr_info_ratelimited("bad rangesize %u.\n", mr->rangesize); return -EINVAL; } return nf_ct_netns_get(par->net, par->family); diff --git a/net/netfilter/xt_REDIRECT.c b/net/netfilter/xt_REDIRECT.c index ff66b56a3f97..dd050947257b 100644 --- a/net/netfilter/xt_REDIRECT.c +++ b/net/netfilter/xt_REDIRECT.c @@ -8,6 +8,8 @@ * NAT funded by Astaro. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/if.h> #include <linux/inetdevice.h> #include <linux/ip.h> @@ -51,11 +53,11 @@ static int redirect_tg4_check(const struct xt_tgchk_param *par) const struct nf_nat_ipv4_multi_range_compat *mr = par->targinfo; if (mr->range[0].flags & NF_NAT_RANGE_MAP_IPS) { - pr_debug("bad MAP_IPS.\n"); + pr_info_ratelimited("bad MAP_IPS.\n"); return -EINVAL; } if (mr->rangesize != 1) { - pr_debug("bad rangesize %u.\n", mr->rangesize); + pr_info_ratelimited("bad rangesize %u.\n", mr->rangesize); return -EINVAL; } return nf_ct_netns_get(par->net, par->family); diff --git a/net/netfilter/xt_cgroup.c b/net/netfilter/xt_cgroup.c index 43d2ae2be628..28e6cd51b2fb 100644 --- a/net/netfilter/xt_cgroup.c +++ b/net/netfilter/xt_cgroup.c @@ -23,7 +23,7 @@ MODULE_DESCRIPTION("Xtables: process control group matching"); MODULE_ALIAS("ipt_cgroup"); MODULE_ALIAS("ip6t_cgroup"); -#define NET_CLS_CLASSID_INVALID_MSG "xt_cgroup: classid invalid without net_cls cgroups\n" +#define NET_CLS_CLASSID_INVALID_MSG "classid invalid without net_cls cgroups\n" static int cgroup_mt_check_v0(const struct xt_mtchk_param *par) { @@ -33,7 +33,7 @@ static int cgroup_mt_check_v0(const struct xt_mtchk_param *par) return -EINVAL; if (!IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } @@ -49,7 +49,7 @@ static int cgroup_mt_check_v1(const struct xt_mtchk_param *par) return -EINVAL; if (!info->has_path && !info->has_classid) { - pr_info("xt_cgroup: no path or classid specified\n"); + pr_info_ratelimited("no path or classid specified\n"); return -EINVAL; } @@ -59,7 +59,7 @@ static int cgroup_mt_check_v1(const struct xt_mtchk_param *par) } if (info->has_classid && !IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } @@ -89,7 +89,7 @@ static int cgroup_mt_check_v2(const struct xt_mtchk_param *par) return -EINVAL; if (!info->has_path && !info->has_classid) { - pr_info("xt_cgroup: no path or classid specified\n"); + pr_info_ratelimited("no path or classid specified\n"); return -EINVAL; } @@ -99,7 +99,7 @@ static int cgroup_mt_check_v2(const struct xt_mtchk_param *par) } if (info->has_classid && !IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } diff --git a/net/netfilter/xt_esp.c b/net/netfilter/xt_esp.c index 2a1c0ad0ff07..68fd75884268 100644 --- a/net/netfilter/xt_esp.c +++ b/net/netfilter/xt_esp.c @@ -25,12 +25,7 @@ MODULE_ALIAS("ip6t_esp"); static inline bool spi_match(u_int32_t min, u_int32_t max, u_int32_t spi, bool invert) { - bool r; - pr_debug("spi_match:%c 0x%x <= 0x%x <= 0x%x\n", - invert ? '!' : ' ', min, spi, max); - r = (spi >= min && spi <= max) ^ invert; - pr_debug(" result %s\n", r ? "PASS" : "FAILED"); - return r; + return (spi >= min && spi <= max) ^ invert; } static bool esp_mt(const struct sk_buff *skb, struct xt_action_param *par) @@ -48,7 +43,6 @@ static bool esp_mt(const struct sk_buff *skb, struct xt_action_param *par) /* We've been asked to examine this packet, and we * can't. Hence, no choice but to drop. */ - pr_debug("Dropping evil ESP tinygram.\n"); par->hotdrop = true; return false; } @@ -62,7 +56,7 @@ static int esp_mt_check(const struct xt_mtchk_param *par) const struct xt_esp *espinfo = par->matchinfo; if (espinfo->invflags & ~XT_ESP_INV_MASK) { - pr_debug("unknown flags %X\n", espinfo->invflags); + pr_info_ratelimited("unknown flags %X\n", espinfo->invflags); return -EINVAL; } diff --git a/net/netfilter/xt_hl.c b/net/netfilter/xt_hl.c index 4a12a757ecbf..59e93d97b507 100644 --- a/net/netfilter/xt_hl.c +++ b/net/netfilter/xt_hl.c @@ -28,7 +28,7 @@ static int ttl_mt_check(const struct xt_mtchk_param *par) const struct ipt_ttl_info *info = par->matchinfo; if (info->mode > IPT_TTL_GT) { - pr_err("Unknown TTL match mode: %d\n", info->mode); + pr_info_ratelimited("Unknown TTL match mode: %d\n", info->mode); return -EINVAL; } @@ -59,7 +59,7 @@ static int hl_mt6_check(const struct xt_mtchk_param *par) const struct ip6t_hl_info *info = par->matchinfo; if (info->mode > IP6T_HL_GT) { - pr_err("Unknown Hop Limit match mode: %d\n", info->mode); + pr_info_ratelimited("Unknown Hop Limit match mode: %d\n", info->mode); return -EINVAL; } diff --git a/net/netfilter/xt_ipcomp.c b/net/netfilter/xt_ipcomp.c index 472da639a32e..3299c1ea60f9 100644 --- a/net/netfilter/xt_ipcomp.c +++ b/net/netfilter/xt_ipcomp.c @@ -29,12 +29,7 @@ MODULE_ALIAS("ip6t_ipcomp"); static inline bool spi_match(u_int32_t min, u_int32_t max, u_int32_t spi, bool invert) { - bool r; - pr_debug("spi_match:%c 0x%x <= 0x%x <= 0x%x\n", - invert ? '!' : ' ', min, spi, max); - r = (spi >= min && spi <= max) ^ invert; - pr_debug(" result %s\n", r ? "PASS" : "FAILED"); - return r; + return (spi >= min && spi <= max) ^ invert; } static bool comp_mt(const struct sk_buff *skb, struct xt_action_param *par) @@ -52,7 +47,6 @@ static bool comp_mt(const struct sk_buff *skb, struct xt_action_param *par) /* We've been asked to examine this packet, and we * can't. Hence, no choice but to drop. */ - pr_debug("Dropping evil IPComp tinygram.\n"); par->hotdrop = true; return false; } diff --git a/net/netfilter/xt_iprange.c b/net/netfilter/xt_iprange.c index 0c9e014e30b4..bf61141fb785 100644 --- a/net/netfilter/xt_iprange.c +++ b/net/netfilter/xt_iprange.c @@ -24,27 +24,15 @@ iprange_mt4(const struct sk_buff *skb, struct xt_action_param *par) m = ntohl(iph->saddr) < ntohl(info->src_min.ip); m |= ntohl(iph->saddr) > ntohl(info->src_max.ip); m ^= !!(info->flags & IPRANGE_SRC_INV); - if (m) { - pr_debug("src IP %pI4 NOT in range %s%pI4-%pI4\n", - &iph->saddr, - (info->flags & IPRANGE_SRC_INV) ? "(INV) " : "", - &info->src_min.ip, - &info->src_max.ip); + if (m) return false; - } } if (info->flags & IPRANGE_DST) { m = ntohl(iph->daddr) < ntohl(info->dst_min.ip); m |= ntohl(iph->daddr) > ntohl(info->dst_max.ip); m ^= !!(info->flags & IPRANGE_DST_INV); - if (m) { - pr_debug("dst IP %pI4 NOT in range %s%pI4-%pI4\n", - &iph->daddr, - (info->flags & IPRANGE_DST_INV) ? "(INV) " : "", - &info->dst_min.ip, - &info->dst_max.ip); + if (m) return false; - } } return true; } @@ -73,27 +61,15 @@ iprange_mt6(const struct sk_buff *skb, struct xt_action_param *par) m = iprange_ipv6_lt(&iph->saddr, &info->src_min.in6); m |= iprange_ipv6_lt(&info->src_max.in6, &iph->saddr); m ^= !!(info->flags & IPRANGE_SRC_INV); - if (m) { - pr_debug("src IP %pI6 NOT in range %s%pI6-%pI6\n", - &iph->saddr, - (info->flags & IPRANGE_SRC_INV) ? "(INV) " : "", - &info->src_min.in6, - &info->src_max.in6); + if (m) return false; - } } if (info->flags & IPRANGE_DST) { m = iprange_ipv6_lt(&iph->daddr, &info->dst_min.in6); m |= iprange_ipv6_lt(&info->dst_max.in6, &iph->daddr); m ^= !!(info->flags & IPRANGE_DST_INV); - if (m) { - pr_debug("dst IP %pI6 NOT in range %s%pI6-%pI6\n", - &iph->daddr, - (info->flags & IPRANGE_DST_INV) ? "(INV) " : "", - &info->dst_min.in6, - &info->dst_max.in6); + if (m) return false; - } } return true; } diff --git a/net/netfilter/xt_ipvs.c b/net/netfilter/xt_ipvs.c index 253c71cc9a63..e13c0ffb73a9 100644 --- a/net/netfilter/xt_ipvs.c +++ b/net/netfilter/xt_ipvs.c @@ -148,7 +148,6 @@ ipvs_mt(const struct sk_buff *skb, struct xt_action_param *par) out_put_cp: __ip_vs_conn_put(cp); out: - pr_debug("match=%d\n", match); return match; } diff --git a/net/netfilter/xt_multiport.c b/net/netfilter/xt_multiport.c index a1691ff405d3..bff5f53a9bef 100644 --- a/net/netfilter/xt_multiport.c +++ b/net/netfilter/xt_multiport.c @@ -37,7 +37,6 @@ ports_match_v1(const struct xt_multiport_v1 *minfo, if (minfo->pflags[i]) { /* range port matching */ e = minfo->ports[++i]; - pr_debug("src or dst matches with %d-%d?\n", s, e); switch (minfo->flags) { case XT_MULTIPORT_SOURCE: @@ -58,8 +57,6 @@ ports_match_v1(const struct xt_multiport_v1 *minfo, } } else { /* exact port matching */ - pr_debug("src or dst matches with %d?\n", s); - switch (minfo->flags) { case XT_MULTIPORT_SOURCE: if (src == s) @@ -97,7 +94,6 @@ multiport_mt(const struct sk_buff *skb, struct xt_action_param *par) /* We've been asked to examine this packet, and we * can't. Hence, no choice but to drop. */ - pr_debug("Dropping evil offset=0 tinygram.\n"); par->hotdrop = true; return false; } diff --git a/net/netfilter/xt_sctp.c b/net/netfilter/xt_sctp.c index b46a6a512058..d35c21d9651b 100644 --- a/net/netfilter/xt_sctp.c +++ b/net/netfilter/xt_sctp.c @@ -48,30 +48,17 @@ match_packet(const struct sk_buff *skb, const struct xt_sctp_flag_info *flag_info = info->flag_info; int flag_count = info->flag_count; -#ifdef DEBUG - int i = 0; -#endif - if (chunk_match_type == SCTP_CHUNK_MATCH_ALL) SCTP_CHUNKMAP_COPY(chunkmapcopy, info->chunkmap); do { sch = skb_header_pointer(skb, offset, sizeof(_sch), &_sch); if (sch == NULL || sch->length == 0) { - pr_debug("Dropping invalid SCTP packet.\n"); *hotdrop = true; return false; } -#ifdef DEBUG - pr_debug("Chunk num: %d\toffset: %d\ttype: %d\tlength: %d" - "\tflags: %x\n", - ++i, offset, sch->type, htons(sch->length), - sch->flags); -#endif offset += SCTP_PAD4(ntohs(sch->length)); - pr_debug("skb->len: %d\toffset: %d\n", skb->len, offset); - if (SCTP_CHUNKMAP_IS_SET(info->chunkmap, sch->type)) { switch (chunk_match_type) { case SCTP_CHUNK_MATCH_ANY: @@ -121,18 +108,14 @@ sctp_mt(const struct sk_buff *skb, struct xt_action_param *par) const struct sctphdr *sh; struct sctphdr _sh; - if (par->fragoff != 0) { - pr_debug("Dropping non-first fragment.. FIXME\n"); + if (par->fragoff != 0) return false; - } sh = skb_header_pointer(skb, par->thoff, sizeof(_sh), &_sh); if (sh == NULL) { - pr_debug("Dropping evil TCP offset=0 tinygram.\n"); par->hotdrop = true; return false; } - pr_debug("spt: %d\tdpt: %d\n", ntohs(sh->source), ntohs(sh->dest)); return SCCHECK(ntohs(sh->source) >= info->spts[0] && ntohs(sh->source) <= info->spts[1], diff --git a/net/netfilter/xt_tcpudp.c b/net/netfilter/xt_tcpudp.c index f76cf18f1a24..70608b8d06ab 100644 --- a/net/netfilter/xt_tcpudp.c +++ b/net/netfilter/xt_tcpudp.c @@ -44,8 +44,6 @@ tcp_find_option(u_int8_t option, u_int8_t _opt[60 - sizeof(struct tcphdr)]; unsigned int i; - pr_debug("finding option\n"); - if (!optlen) return invert; @@ -81,10 +79,8 @@ static bool tcp_mt(const struct sk_buff *skb, struct xt_action_param *par) causes this. Its a cracker trying to break in by doing a flag overwrite to pass the direction checks. */ - if (par->fragoff == 1) { - pr_debug("Dropping evil TCP offset=1 frag.\n"); + if (par->fragoff == 1) par->hotdrop = true; - } /* Must not be a fragment. */ return false; } @@ -93,7 +89,6 @@ static bool tcp_mt(const struct sk_buff *skb, struct xt_action_param *par) if (th == NULL) { /* We've been asked to examine this packet, and we can't. Hence, no choice but to drop. */ - pr_debug("Dropping evil TCP offset=0 tinygram.\n"); par->hotdrop = true; return false; } @@ -145,7 +140,6 @@ static bool udp_mt(const struct sk_buff *skb, struct xt_action_param *par) if (uh == NULL) { /* We've been asked to examine this packet, and we can't. Hence, no choice but to drop. */ - pr_debug("Dropping evil UDP tinygram.\n"); par->hotdrop = true; return false; } diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index 49eb2b0d234d..27115967e5d9 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -932,10 +932,14 @@ static int ovs_ct_check_limit(struct net *net, const struct ovs_conntrack_info *info) { struct ovs_net *ovs_net = net_generic(net, ovs_net_id); - const struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; + const struct ovs_ct_limit_info *ct_limit_info; u32 per_zone_limit, connections; u32 conncount_key; + ct_limit_info = rcu_dereference(ovs_net->ct_limit_info); + if (!ct_limit_info) + return 0; + conncount_key = info->zone.id; per_zone_limit = ct_limit_get(ct_limit_info, info->zone.id); @@ -1579,40 +1583,55 @@ static void __ovs_ct_free_action(struct ovs_conntrack_info *ct_info) #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) static int ovs_ct_limit_init(struct net *net, struct ovs_net *ovs_net) { + struct ovs_ct_limit_info *info; int i, err; - ovs_net->ct_limit_info = kmalloc_obj(*ovs_net->ct_limit_info); - if (!ovs_net->ct_limit_info) + info = kmalloc_obj(*info); + if (!info) return -ENOMEM; - ovs_net->ct_limit_info->default_limit = OVS_CT_LIMIT_DEFAULT; - ovs_net->ct_limit_info->limits = + info->default_limit = OVS_CT_LIMIT_DEFAULT; + info->limits = kmalloc_objs(struct hlist_head, CT_LIMIT_HASH_BUCKETS); - if (!ovs_net->ct_limit_info->limits) { - kfree(ovs_net->ct_limit_info); + if (!info->limits) { + kfree(info); return -ENOMEM; } for (i = 0; i < CT_LIMIT_HASH_BUCKETS; i++) - INIT_HLIST_HEAD(&ovs_net->ct_limit_info->limits[i]); + INIT_HLIST_HEAD(&info->limits[i]); - ovs_net->ct_limit_info->data = nf_conncount_init(net, sizeof(u32)); + info->data = nf_conncount_init(net, sizeof(u32)); - if (IS_ERR(ovs_net->ct_limit_info->data)) { - err = PTR_ERR(ovs_net->ct_limit_info->data); - kfree(ovs_net->ct_limit_info->limits); - kfree(ovs_net->ct_limit_info); + if (IS_ERR(info->data)) { + err = PTR_ERR(info->data); + kfree(info->limits); + kfree(info); pr_err("openvswitch: failed to init nf_conncount %d\n", err); return err; } + rcu_assign_pointer(ovs_net->ct_limit_info, info); return 0; } -static void ovs_ct_limit_exit(struct net *net, struct ovs_net *ovs_net) +static void *ovs_ct_limit_exit_start(struct ovs_net *ovs_net) +{ + return rcu_replace_pointer(ovs_net->ct_limit_info, NULL, + lockdep_ovsl_is_held()); +} + +/* The CT limit state must be detached by ovs_ct_limit_exit_start() and an + * RCU grace period must elapse before this function runs. The pernet core + * guarantees the grace period between the .pre_exit and .exit callbacks. + */ +static void ovs_ct_limit_exit_finish(struct net *net, void *data) { - const struct ovs_ct_limit_info *info = ovs_net->ct_limit_info; + const struct ovs_ct_limit_info *info = data; int i; + if (!info) + return; + nf_conncount_destroy(net, info->data); for (i = 0; i < CT_LIMIT_HASH_BUCKETS; ++i) { struct hlist_head *head = &info->limits[i]; @@ -1620,7 +1639,7 @@ static void ovs_ct_limit_exit(struct net *net, struct ovs_net *ovs_net) struct hlist_node *next; hlist_for_each_entry_safe(ct_limit, next, head, hlist_node) - kfree_rcu(ct_limit, rcu); + kfree(ct_limit); } kfree(info->limits); kfree(info); @@ -1659,12 +1678,13 @@ static bool check_zone_id(int zone_id, u16 *pzone) return false; } -static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, - struct ovs_ct_limit_info *info) +static int ovs_ct_limit_set_zone_limit(struct ovs_net *ovs_net, + struct nlattr *nla_zone_limit) { struct ovs_zone_limit *zone_limit; - int rem; + struct ovs_ct_limit_info *info; u16 zone; + int rem; rem = NLA_ALIGN(nla_len(nla_zone_limit)); zone_limit = (struct ovs_zone_limit *)nla_data(nla_zone_limit); @@ -1673,6 +1693,7 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, if (unlikely(zone_limit->zone_id == OVS_ZONE_LIMIT_DEFAULT_ZONE)) { ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); info->default_limit = zone_limit->limit; ovs_unlock(); } else if (unlikely(!check_zone_id( @@ -1689,6 +1710,7 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, ct_limit->limit = zone_limit->limit; ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); ct_limit_set(info, ct_limit); ovs_unlock(); } @@ -1703,12 +1725,13 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, return 0; } -static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit, - struct ovs_ct_limit_info *info) +static int ovs_ct_limit_del_zone_limit(struct ovs_net *ovs_net, + struct nlattr *nla_zone_limit) { struct ovs_zone_limit *zone_limit; - int rem; + struct ovs_ct_limit_info *info; u16 zone; + int rem; rem = NLA_ALIGN(nla_len(nla_zone_limit)); zone_limit = (struct ovs_zone_limit *)nla_data(nla_zone_limit); @@ -1717,6 +1740,7 @@ static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit, if (unlikely(zone_limit->zone_id == OVS_ZONE_LIMIT_DEFAULT_ZONE)) { ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); info->default_limit = OVS_CT_LIMIT_DEFAULT; ovs_unlock(); } else if (unlikely(!check_zone_id( @@ -1724,6 +1748,7 @@ static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit, OVS_NLERR(true, "zone id is out of range"); } else { ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); ct_limit_del(info, zone); ovs_unlock(); } @@ -1767,6 +1792,7 @@ static int __ovs_ct_limit_get_zone_limit(struct net *net, return nla_put_nohdr(reply, sizeof(zone_limit), &zone_limit); } +/* Called with RCU read lock held. */ static int ovs_ct_limit_get_zone_limit(struct net *net, struct nlattr *nla_zone_limit, struct ovs_ct_limit_info *info, @@ -1790,12 +1816,10 @@ static int ovs_ct_limit_get_zone_limit(struct net *net, &zone))) { OVS_NLERR(true, "zone id is out of range"); } else { - rcu_read_lock(); limit = ct_limit_get(info, zone); err = __ovs_ct_limit_get_zone_limit( net, info->data, zone, limit, reply); - rcu_read_unlock(); if (err) return err; } @@ -1810,6 +1834,7 @@ static int ovs_ct_limit_get_zone_limit(struct net *net, return 0; } +/* Called with RCU read lock held. */ static int ovs_ct_limit_get_all_zone_limit(struct net *net, struct ovs_ct_limit_info *info, struct sk_buff *reply) @@ -1822,19 +1847,16 @@ static int ovs_ct_limit_get_all_zone_limit(struct net *net, if (err) return err; - rcu_read_lock(); for (i = 0; i < CT_LIMIT_HASH_BUCKETS; ++i) { head = &info->limits[i]; hlist_for_each_entry_rcu(ct_limit, head, hlist_node) { err = __ovs_ct_limit_get_zone_limit(net, info->data, ct_limit->zone, ct_limit->limit, reply); if (err) - goto exit_err; + return err; } } -exit_err: - rcu_read_unlock(); return err; } @@ -1844,7 +1866,6 @@ static int ovs_ct_limit_cmd_set(struct sk_buff *skb, struct genl_info *info) struct sk_buff *reply; struct ovs_header *ovs_reply_header; struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id); - struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_SET, @@ -1857,8 +1878,8 @@ static int ovs_ct_limit_cmd_set(struct sk_buff *skb, struct genl_info *info) goto exit_err; } - err = ovs_ct_limit_set_zone_limit(a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT], - ct_limit_info); + err = ovs_ct_limit_set_zone_limit(ovs_net, + a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]); if (err) goto exit_err; @@ -1878,7 +1899,6 @@ static int ovs_ct_limit_cmd_del(struct sk_buff *skb, struct genl_info *info) struct sk_buff *reply; struct ovs_header *ovs_reply_header; struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id); - struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_DEL, @@ -1891,8 +1911,8 @@ static int ovs_ct_limit_cmd_del(struct sk_buff *skb, struct genl_info *info) goto exit_err; } - err = ovs_ct_limit_del_zone_limit(a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT], - ct_limit_info); + err = ovs_ct_limit_del_zone_limit(ovs_net, + a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]); if (err) goto exit_err; @@ -1912,7 +1932,7 @@ static int ovs_ct_limit_cmd_get(struct sk_buff *skb, struct genl_info *info) struct ovs_header *ovs_reply_header; struct net *net = sock_net(skb->sk); struct ovs_net *ovs_net = net_generic(net, ovs_net_id); - struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; + struct ovs_ct_limit_info *ct_limit_info; int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_GET, @@ -1926,18 +1946,19 @@ static int ovs_ct_limit_cmd_get(struct sk_buff *skb, struct genl_info *info) goto exit_err; } + rcu_read_lock(); + ct_limit_info = rcu_dereference(ovs_net->ct_limit_info); if (a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]) { err = ovs_ct_limit_get_zone_limit( net, a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT], ct_limit_info, reply); - if (err) - goto exit_err; } else { err = ovs_ct_limit_get_all_zone_limit(net, ct_limit_info, reply); - if (err) - goto exit_err; } + rcu_read_unlock(); + if (err) + goto exit_err; nla_nest_end(reply, nla_reply); genlmsg_end(reply, ovs_reply_header); @@ -2012,12 +2033,29 @@ int ovs_ct_init(struct net *net) return err; } -void ovs_ct_exit(struct net *net) +/* Must be called with ovs_mutex held. Detaches the RCU-protected + * ct_limit_info and stores it in ovs_net->ct_limit_exit_data for + * ovs_ct_exit_finish() to complete the teardown after an RCU grace period. + */ +void ovs_ct_exit_start(struct net *net __maybe_unused) +{ +#if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) + struct ovs_net *ovs_net = net_generic(net, ovs_net_id); + + ovs_net->ct_limit_exit_data = ovs_ct_limit_exit_start(ovs_net); +#endif +} + +/* Completes the CT limit teardown. The pernet core guarantees an RCU + * grace period between detaching the state in ovs_ct_exit_start() and + * this call, so no RCU readers remain. + */ +void ovs_ct_exit_finish(struct net *net) { struct ovs_net *ovs_net = net_generic(net, ovs_net_id); #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) - ovs_ct_limit_exit(net, ovs_net); + ovs_ct_limit_exit_finish(net, ovs_net->ct_limit_exit_data); #endif if (ovs_net->xt_label) diff --git a/net/openvswitch/conntrack.h b/net/openvswitch/conntrack.h index 317e525c8a11..ab21d032fde3 100644 --- a/net/openvswitch/conntrack.h +++ b/net/openvswitch/conntrack.h @@ -14,7 +14,8 @@ enum ovs_key_attr; #if IS_ENABLED(CONFIG_NF_CONNTRACK) int ovs_ct_init(struct net *); -void ovs_ct_exit(struct net *); +void ovs_ct_exit_start(struct net *net); +void ovs_ct_exit_finish(struct net *net); bool ovs_ct_verify(struct net *, enum ovs_key_attr attr); int ovs_ct_copy_action(struct net *, const struct nlattr *, const struct sw_flow_key *, struct sw_flow_actions **, @@ -40,7 +41,8 @@ void ovs_ct_free_action(const struct nlattr *a); static inline int ovs_ct_init(struct net *net) { return 0; } -static inline void ovs_ct_exit(struct net *net) { } +static inline void ovs_ct_exit_start(struct net *net) { } +static inline void ovs_ct_exit_finish(struct net *net) { } static inline bool ovs_ct_verify(struct net *net, int attr) { diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index 69999f9cc44c..631a03136fa1 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -285,6 +285,7 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key) consume_skb(skb); break; default: + skb_tx_error(skb); kfree_skb(skb); break; } @@ -467,6 +468,9 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, if (!dp_ifindex) return -ENODEV; + if (!skb_frags_readable(skb)) + return -EFAULT; + if (skb_vlan_tag_present(skb)) { nskb = skb_clone(skb, GFP_ATOMIC); if (!nskb) @@ -601,8 +605,6 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid); user_skb = NULL; out: - if (err) - skb_tx_error(skb); consume_skb(user_skb); consume_skb(nskb); @@ -2739,6 +2741,13 @@ static void __net_exit list_vports_from_net(struct net *net, struct net *dnet, } } +static void __net_exit ovs_pre_exit_net(struct net *dnet) +{ + ovs_lock(); + ovs_ct_exit_start(dnet); + ovs_unlock(); +} + static void __net_exit ovs_exit_net(struct net *dnet) { struct datapath *dp, *dp_next; @@ -2749,7 +2758,7 @@ static void __net_exit ovs_exit_net(struct net *dnet) ovs_lock(); - ovs_ct_exit(dnet); + ovs_ct_exit_finish(dnet); list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node) __dp_destroy(dp); @@ -2773,6 +2782,7 @@ static void __net_exit ovs_exit_net(struct net *dnet) static struct pernet_operations ovs_net_ops = { .init = ovs_init_net, + .pre_exit = ovs_pre_exit_net, .exit = ovs_exit_net, .id = &ovs_net_id, .size = sizeof(struct ovs_net), diff --git a/net/openvswitch/datapath.h b/net/openvswitch/datapath.h index 696640e88fa7..b2c2b8da12d4 100644 --- a/net/openvswitch/datapath.h +++ b/net/openvswitch/datapath.h @@ -164,7 +164,10 @@ struct dp_upcall_info { * Protected by genl_mutex. * @dp_notify_work: A work notifier to handle port unregistering. * @masks_rebalance: A work to periodically optimize flow table caches. - * @ct_limit_info: A hash table of conntrack zone connection limits. + * @ct_limit_info: Hash table of conntrack zone connection limits. Protected + * by RCU; updates and teardown are serialized by ovs_mutex. May be NULL during + * netns teardown. + * @ct_limit_exit_data: CT limit state detached at .pre_exit, freed at .exit. * @xt_label: Whether connlables are configured for the network or not. */ struct ovs_net { @@ -172,7 +175,8 @@ struct ovs_net { struct work_struct dp_notify_work; struct delayed_work masks_rebalance; #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) - struct ovs_ct_limit_info *ct_limit_info; + struct ovs_ct_limit_info __rcu *ct_limit_info; + struct ovs_ct_limit_info *ct_limit_exit_data; #endif bool xt_label; }; diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 1168bd6b09cd..b22cda322136 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -88,6 +88,7 @@ #include <linux/errqueue.h> #include <linux/net_tstamp.h> #include <linux/percpu.h> +#include <linux/workqueue.h> #ifdef CONFIG_INET #include <net/inet_common.h> #endif @@ -1341,6 +1342,8 @@ static void packet_sock_destruct(struct sock *sk) WARN_ON(atomic_read(&sk->sk_rmem_alloc)); WARN_ON(refcount_read(&sk->sk_wmem_alloc)); + packet_free_pending(pkt_sk(sk)); + if (!sock_flag(sk, SOCK_DEAD)) { pr_err("Attempt to release alive packet socket: %p\n", sk); return; @@ -2534,11 +2537,11 @@ static void tpacket_destruct_skb(struct sk_buff *skb) __u32 ts; ph = skb_zcopy_get_nouarg(skb); - packet_dec_pending(&po->tx_ring); ts = __packet_set_timestamp(po, ph, skb); __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); + packet_dec_pending(&po->tx_ring); complete(&po->skb_completion); } @@ -3207,7 +3210,6 @@ static int packet_release(struct socket *sock) /* Purge queues */ skb_queue_purge(&sk->sk_receive_queue); - packet_free_pending(po); sock_put(sk); return 0; @@ -4370,11 +4372,26 @@ static const struct vm_operations_struct packet_mmap_ops = { .close = packet_mm_close, }; +struct packet_pg_vec { + struct packet_pg_vec_free *deferred; + unsigned int order; + unsigned int len; + struct pgv pg_vec[] __counted_by(len); +}; + +struct packet_pg_vec_free { + struct delayed_work work; + struct sock *sk; + struct packet_pg_vec *vec; +}; + static void free_pg_vec(struct pgv *pg_vec, unsigned int order, unsigned int len) { + struct packet_pg_vec *vec; int i; + vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]); for (i = 0; i < len; i++) { if (likely(pg_vec[i].buffer)) { if (is_vmalloc_addr(pg_vec[i].buffer)) @@ -4385,7 +4402,46 @@ static void free_pg_vec(struct pgv *pg_vec, unsigned int order, pg_vec[i].buffer = NULL; } } - kfree(pg_vec); + kfree(vec->deferred); + kfree(vec); +} + +static void packet_free_pg_vec_work(struct work_struct *work) +{ + struct packet_pg_vec_free *deferred; + struct packet_pg_vec *vec; + struct sock *sk; + + deferred = container_of_const(to_delayed_work(work), + struct packet_pg_vec_free, work); + vec = deferred->vec; + sk = deferred->sk; + if (sk_wmem_alloc_get(sk)) { + queue_delayed_work(system_long_wq, &deferred->work, 1); + return; + } + + free_pg_vec(vec->pg_vec, vec->order, vec->len); + sock_put(sk); +} + +static void packet_free_tx_ring(struct sock *sk, struct pgv *pg_vec, + unsigned int order, unsigned int len) +{ + struct packet_pg_vec_free *deferred; + struct packet_pg_vec *vec; + + vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]); + deferred = vec->deferred; + if (!deferred || !sk_wmem_alloc_get(sk)) { + free_pg_vec(pg_vec, order, len); + return; + } + + /* A detached ring's pending count can miss late skb destructors. */ + deferred->sk = sk; + sock_hold(sk); + queue_delayed_work(system_long_wq, &deferred->work, 0); } static char *alloc_one_pg_vec_page(unsigned long order) @@ -4413,20 +4469,35 @@ static char *alloc_one_pg_vec_page(unsigned long order) return NULL; } -static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order) +static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order, bool tx_ring) { unsigned int block_nr = req->tp_block_nr; + struct packet_pg_vec *vec; struct pgv *pg_vec; int i; - pg_vec = kzalloc_objs(struct pgv, block_nr, GFP_KERNEL | __GFP_NOWARN); - if (unlikely(!pg_vec)) - goto out; + vec = kzalloc_flex(*vec, pg_vec, block_nr, GFP_KERNEL | __GFP_NOWARN); + if (unlikely(!vec)) + return NULL; + vec->order = order; + vec->len = block_nr; + pg_vec = vec->pg_vec; for (i = 0; i < block_nr; i++) { pg_vec[i].buffer = alloc_one_pg_vec_page(order); if (unlikely(!pg_vec[i].buffer)) goto out_free_pgvec; + + if (tx_ring && !vec->deferred && + is_vmalloc_addr(pg_vec[i].buffer)) { + vec->deferred = kzalloc_obj(*vec->deferred, + GFP_KERNEL | __GFP_NOWARN); + if (!vec->deferred) + goto out_free_pgvec; + vec->deferred->vec = vec; + INIT_DELAYED_WORK(&vec->deferred->work, + packet_free_pg_vec_work); + } } out: @@ -4509,7 +4580,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, err = -ENOMEM; order = get_order(req->tp_block_size); - pg_vec = alloc_pg_vec(req, order); + pg_vec = alloc_pg_vec(req, order, tx_ring); if (unlikely(!pg_vec)) goto out; switch (po->tp_version) { @@ -4561,6 +4632,9 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, err = -EBUSY; mutex_lock(&po->pg_vec_lock); if (closing || atomic_long_read(&po->mapped) == 0) { + if (tx_ring && !closing && packet_read_pending(rb)) + goto out_unlock; + err = 0; spin_lock_bh(&rb_queue->lock); swap(rb->pg_vec, pg_vec); @@ -4582,6 +4656,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, pr_err("packet_mmap: vma is busy: %ld\n", atomic_long_read(&po->mapped)); } +out_unlock: mutex_unlock(&po->pg_vec_lock); spin_lock(&po->bind_lock); @@ -4603,7 +4678,10 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, out_free_pg_vec: if (pg_vec) { bitmap_free(rx_owner_map); - free_pg_vec(pg_vec, order, req->tp_block_nr); + if (tx_ring && closing) + packet_free_tx_ring(sk, pg_vec, order, req->tp_block_nr); + else + free_pg_vec(pg_vec, order, req->tp_block_nr); } out: return err; diff --git a/net/rds/cong.c b/net/rds/cong.c index 3133b91f9e69..f7634ce3ffc1 100644 --- a/net/rds/cong.c +++ b/net/rds/cong.c @@ -256,9 +256,9 @@ void rds_cong_map_updated(struct rds_cong_map *map, uint64_t portmask) map, &map->m_addr); rds_stats_inc(s_cong_update_received); atomic_inc(&rds_cong_generation); - if (waitqueue_active(&map->m_waitq)) + if (wq_has_sleeper(&map->m_waitq)) wake_up(&map->m_waitq); - if (waitqueue_active(&rds_poll_waitq)) + if (wq_has_sleeper(&rds_poll_waitq)) wake_up_all(&rds_poll_waitq); if (portmask && !list_empty(&rds_cong_monitor)) { diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 600b7804befd..b4415d358c91 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -452,7 +452,10 @@ static size_t tcf_action_shared_attrs_size(const struct tc_action *act) /* TCA_STATS_QUEUE */ + nla_total_size_64bit(sizeof(struct gnet_stats_queue)) + nla_total_size(0) /* TCA_ACT_OPTIONS nested */ - + nla_total_size(sizeof(struct tcf_t)); /* TCA_GACT_TM */ + /* TCA_GACT_TM; actions dump their tcf_t with nla_put_64bit(), + * which may emit an extra NLA_PAD attribute. + */ + + nla_total_size_64bit(sizeof(struct tcf_t)); } static size_t tcf_action_full_attrs_size(size_t sz) diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index 09d46e195e33..06d8f78b7368 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -389,6 +389,31 @@ static void tcf_bpf_cleanup(struct tc_action *act) tcf_bpf_cfg_cleanup(&tmp); } +static size_t tcf_bpf_get_fill_size(const struct tc_action *act) +{ + struct tcf_bpf *prog = to_bpf(act); + size_t size = nla_total_size(sizeof(struct tc_act_bpf)); + + /* bpf_ops and bpf_num_ops are published as separate stores under + * tcf_lock, so take it here as tcf_bpf_dump() does. + */ + spin_lock_bh(&prog->tcf_lock); + if (tcf_bpf_is_ebpf(prog)) { + /* TCA_ACT_BPF_NAME */ + size += nla_total_size(ACT_BPF_NAME_LEN + 1); + size += nla_total_size(sizeof(u32)); /* TCA_ACT_BPF_ID */ + size += nla_total_size(BPF_TAG_SIZE); /* TCA_ACT_BPF_TAG */ + } else { + size += nla_total_size(sizeof(u16)); /* TCA_ACT_BPF_OPS_LEN */ + /* TCA_ACT_BPF_OPS */ + size += nla_total_size(prog->bpf_num_ops * + sizeof(struct sock_filter)); + } + spin_unlock_bh(&prog->tcf_lock); + + return size; +} + static struct tc_action_ops act_bpf_ops __read_mostly = { .kind = "bpf", .id = TCA_ID_BPF, @@ -397,6 +422,7 @@ static struct tc_action_ops act_bpf_ops __read_mostly = { .dump = tcf_bpf_dump, .cleanup = tcf_bpf_cleanup, .init = tcf_bpf_init, + .get_fill_size = tcf_bpf_get_fill_size, .size = sizeof(struct tcf_bpf), }; MODULE_ALIAS_NET_ACT("bpf"); diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index dc0b5724785d..9080cb386c16 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1657,6 +1657,51 @@ static int tcf_ct_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_ct_get_fill_size(const struct tc_action *act) +{ + const struct tcf_ct_params *p; + size_t size; + + size = nla_total_size(sizeof(struct tc_ct)) /* TCA_CT_PARMS */ + + nla_total_size(sizeof(u16)); /* TCA_CT_ACTION */ + + rcu_read_lock(); + p = rcu_dereference(to_ct(act)->params); + + if (p->ct_action & TCA_CT_ACT_CLEAR) + goto out; + + /* TCA_CT_MARK, TCA_CT_MARK_MASK */ + if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) + size += nla_total_size(sizeof(p->mark)) + + nla_total_size(sizeof(p->mark_mask)); + + /* TCA_CT_LABELS, TCA_CT_LABELS_MASK */ + if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) + size += nla_total_size(sizeof(p->labels)) + + nla_total_size(sizeof(p->labels_mask)); + + if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) + size += nla_total_size(sizeof(p->zone)); /* TCA_CT_ZONE */ + + if (p->ct_action & TCA_CT_ACT_NAT) + /* TCA_CT_NAT_IPV6_{MIN,MAX}, the larger of the two address + * variants, plus TCA_CT_NAT_PORT_{MIN,MAX}. + */ + size += 2 * nla_total_size(sizeof(struct in6_addr)) + + 2 * nla_total_size(sizeof(__be16)); + + /* TCA_CT_HELPER_{NAME,FAMILY,PROTO} */ + if (p->helper) + size += nla_total_size(NF_CT_HELPER_NAME_LEN) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); +out: + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_ct_ops = { .kind = "ct", .id = TCA_ID_CT, @@ -1666,6 +1711,7 @@ static struct tc_action_ops act_ct_ops = { .init = tcf_ct_init, .cleanup = tcf_ct_cleanup, .stats_update = tcf_stats_update, + .get_fill_size = tcf_ct_get_fill_size, .offload_act_setup = tcf_ct_offload_act_setup, .size = sizeof(struct tcf_ct), }; diff --git a/net/sched/act_ctinfo.c b/net/sched/act_ctinfo.c index 1886ffd2ca95..fced4b1094af 100644 --- a/net/sched/act_ctinfo.c +++ b/net/sched/act_ctinfo.c @@ -356,6 +356,16 @@ static void tcf_ctinfo_cleanup(struct tc_action *a) kfree_rcu(cp, rcu); } +static size_t tcf_ctinfo_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_ctinfo)) /* TCA_CTINFO_ACT */ + + nla_total_size(sizeof(u16)) /* TCA_CTINFO_ZONE */ + /* TCA_CTINFO_PARMS_{DSCP_MASK,DSCP_STATEMASK,CPMARK_MASK} */ + + 3 * nla_total_size(sizeof(u32)) + /* TCA_CTINFO_STATS_{DSCP_SET,DSCP_ERROR,CPMARK_SET} */ + + 3 * nla_total_size_64bit(sizeof(u64)); +} + static struct tc_action_ops act_ctinfo_ops = { .kind = "ctinfo", .id = TCA_ID_CTINFO, @@ -364,6 +374,7 @@ static struct tc_action_ops act_ctinfo_ops = { .dump = tcf_ctinfo_dump, .init = tcf_ctinfo_init, .cleanup= tcf_ctinfo_cleanup, + .get_fill_size = tcf_ctinfo_get_fill_size, .size = sizeof(struct tcf_ctinfo), }; MODULE_ALIAS_NET_ACT("ctinfo"); diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 065228026c58..9cea71fc1db3 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -28,6 +28,7 @@ #include <uapi/linux/tc_act/tc_ife.h> #include <net/tc_act/tc_ife.h> #include <linux/etherdevice.h> +#include <linux/if_arp.h> #include <net/ife.h> #include <net/tc_wrapper.h> @@ -723,7 +724,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, tcf_lastuse_update(&ife->tcf_tm); if (skb_at_tc_ingress(skb)) - skb_push(skb, skb->dev->hard_header_len); + skb_push(skb, ETH_HLEN); tlv_data = ife_decode(skb, &metalen); if (unlikely(!tlv_data)) { @@ -795,7 +796,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, where ORIGDATA = original ethernet header ... */ u16 metalen = ife_get_sz(skb, p); - int hdrm = metalen + skb->dev->hard_header_len + IFE_METAHDRLEN; + int hdrm = metalen + ETH_HLEN + IFE_METAHDRLEN; unsigned int skboff = 0; int new_len = skb->len + hdrm; bool exceed_mtu = false; @@ -826,7 +827,7 @@ drop: } if (skb_at_tc_ingress(skb)) - skb_push(skb, skb->dev->hard_header_len); + skb_push(skb, ETH_HLEN); ife_meta = ife_encode(skb, metalen); if (!ife_meta) @@ -856,11 +857,27 @@ drop: oethh->h_proto = htons(p->eth_type); if (skb_at_tc_ingress(skb)) - skb_pull(skb, skb->dev->hard_header_len); + skb_pull(skb, ETH_HLEN); return action; } +/* IFE encapsulates the original Ethernet header and, on decode, expects to + * find one, so it can only ever work on skbs that carry one. Loopback carries + * Ethernet header as well, so it qualifies here. + * At ingress, also verify that the L2 header about to be pushed back really + * is an Ethernet header because the skb could've been redirected with mirred + * from a non-Ethernet device. + */ +static bool tcf_ife_is_eth_skb(const struct sk_buff *skb) +{ + if (skb->dev->type != ARPHRD_ETHER && + skb->dev->type != ARPHRD_LOOPBACK) + return false; + + return !skb_at_tc_ingress(skb) || skb->mac_len == ETH_HLEN; +} + TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) @@ -869,6 +886,13 @@ TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, struct tcf_ife_params *p; int ret; + if (unlikely(!tcf_ife_is_eth_skb(skb))) { + bstats_update(this_cpu_ptr(ife->common.cpu_bstats), skb); + tcf_lastuse_update(&ife->tcf_tm); + qstats_cpu_drop_inc(ife->common.cpu_qstats); + return TC_ACT_SHOT; + } + p = rcu_dereference_bh(ife->params); if (p->flags & IFE_ENCODE) { ret = tcf_ife_encode(skb, a, res, p); @@ -878,6 +902,28 @@ TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, return tcf_ife_decode(skb, a, res); } +static size_t tcf_ife_get_fill_size(const struct tc_action *act) +{ + struct tcf_ife_info *ife = to_ife(act); + const struct tcf_ife_params *p; + struct tcf_meta_info *e; + size_t size = nla_total_size(sizeof(struct tc_ife)) /* TCA_IFE_PARMS */ + + nla_total_size(ETH_ALEN) /* TCA_IFE_DMAC */ + + nla_total_size(ETH_ALEN) /* TCA_IFE_SMAC */ + + nla_total_size(2) /* TCA_IFE_TYPE */ + + nla_total_size(0); /* TCA_IFE_METALST */ + + rcu_read_lock(); + p = rcu_dereference(ife->params); + if (p) { + list_for_each_entry_rcu(e, &p->metalist, metalist) + size += nla_total_size(sizeof(u32)); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_ife_ops = { .kind = "ife", .id = TCA_ID_IFE, @@ -886,6 +932,7 @@ static struct tc_action_ops act_ife_ops = { .dump = tcf_ife_dump, .cleanup = tcf_ife_cleanup, .init = tcf_ife_init, + .get_fill_size = tcf_ife_get_fill_size, .size = sizeof(struct tcf_ife_info), }; MODULE_ALIAS_NET_ACT("ife"); diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index d4d47a9921f4..99d7e36510bd 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -626,6 +626,29 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_pedit_get_fill_size(const struct tc_action *act) +{ + const struct tcf_pedit_parms *parms; + size_t size; + + rcu_read_lock(); + parms = rcu_dereference(to_pedit(act)->parms); + size = nla_total_size(struct_size_t(struct tc_pedit, keys, + parms->tcfp_nkeys)); + if (parms->tcfp_keys_ex) { + /* TCA_PEDIT_KEYS_EX, holding one TCA_PEDIT_KEY_EX nest with a + * HTYPE and a CMD attribute per key. + */ + size += nla_total_size(0) + + parms->tcfp_nkeys * (nla_total_size(0) + + nla_total_size(sizeof(u16)) + + nla_total_size(sizeof(u16))); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_pedit_ops = { .kind = "pedit", .id = TCA_ID_PEDIT, @@ -635,6 +658,7 @@ static struct tc_action_ops act_pedit_ops = { .dump = tcf_pedit_dump, .cleanup = tcf_pedit_cleanup, .init = tcf_pedit_init, + .get_fill_size = tcf_pedit_get_fill_size, .offload_act_setup = tcf_pedit_offload_act_setup, .size = sizeof(struct tcf_pedit), }; diff --git a/net/sched/act_police.c b/net/sched/act_police.c index ce08f6840ef7..3f8147f37549 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -490,6 +490,17 @@ static int tcf_police_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_police_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_police)) /* TCA_POLICE_TBF */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_RATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PEAKRATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PKTRATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PKTBURST64 */ + + nla_total_size(sizeof(u32)) /* TCA_POLICE_RESULT */ + + nla_total_size(sizeof(u32)); /* TCA_POLICE_AVRATE */ +} + MODULE_AUTHOR("Alexey Kuznetsov"); MODULE_DESCRIPTION("Policing actions"); MODULE_LICENSE("GPL"); @@ -503,6 +514,7 @@ static struct tc_action_ops act_police_ops = { .dump = tcf_police_dump, .init = tcf_police_init, .cleanup = tcf_police_cleanup, + .get_fill_size = tcf_police_get_fill_size, .offload_act_setup = tcf_police_offload_act_setup, .size = sizeof(struct tcf_police), }; diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c index 2ceb4d141b71..44319a159b55 100644 --- a/net/sched/act_sample.c +++ b/net/sched/act_sample.c @@ -315,6 +315,14 @@ static int tcf_sample_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_sample_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_sample)) /* TCA_SAMPLE_PARMS */ + + nla_total_size(sizeof(u32)) /* TCA_SAMPLE_RATE */ + + nla_total_size(sizeof(u32)) /* TCA_SAMPLE_TRUNC_SIZE */ + + nla_total_size(sizeof(u32)); /* TCA_SAMPLE_PSAMPLE_GROUP */ +} + static struct tc_action_ops act_sample_ops = { .kind = "sample", .id = TCA_ID_SAMPLE, @@ -324,6 +332,7 @@ static struct tc_action_ops act_sample_ops = { .dump = tcf_sample_dump, .init = tcf_sample_init, .cleanup = tcf_sample_cleanup, + .get_fill_size = tcf_sample_get_fill_size, .get_psample_group = tcf_sample_get_group, .offload_act_setup = tcf_sample_offload_act_setup, .size = sizeof(struct tcf_sample), diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c index a464b0a3c1b8..7579cf1e0ff3 100644 --- a/net/sched/act_skbmod.c +++ b/net/sched/act_skbmod.c @@ -38,7 +38,6 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, if (unlikely(p->action == TC_ACT_SHOT)) goto drop; - max_edit_len = skb_mac_header_len(skb); flags = p->flags; /* tcf_skbmod_init() guarantees "flags" to be one of the following: @@ -51,14 +50,19 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, if (flags == SKBMOD_F_ECN) { switch (skb_protocol(skb, true)) { case cpu_to_be16(ETH_P_IP): + max_edit_len = sizeof(struct iphdr); + break; case cpu_to_be16(ETH_P_IPV6): - max_edit_len += skb_network_header_len(skb); + max_edit_len = sizeof(struct ipv6hdr); break; default: goto out; } - } else if (!skb->dev || skb->dev->type != ARPHRD_ETHER) { - goto out; + max_edit_len += skb_network_offset(skb); + } else { + if (!skb->dev || skb->dev->type != ARPHRD_ETHER) + goto out; + max_edit_len = ETH_HLEN; } err = skb_ensure_writable(skb, max_edit_len); diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index b14807761d82..ff401ace4f3d 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -835,6 +835,85 @@ static int tcf_tunnel_key_offload_act_setup(struct tc_action *act, return 0; } +static size_t +tunnel_key_geneve_opts_fill_size(const struct ip_tunnel_info *info) +{ + const u8 *src = ip_tunnel_info_opts(info); + int len = info->options_len; + size_t size = 0; + + while (len > 0) { + const struct geneve_opt *opt = (const struct geneve_opt *)src; + + /* TCA_TUNNEL_KEY_ENC_OPT_GENEVE_{CLASS,TYPE,DATA} */ + size += nla_total_size(2) + + nla_total_size(1) + + nla_total_size(opt->length * 4); + + len -= sizeof(struct geneve_opt) + opt->length * 4; + src += sizeof(struct geneve_opt) + opt->length * 4; + } + + return size; +} + +static size_t tunnel_key_opts_fill_size(const struct ip_tunnel_info *info) +{ + size_t size; + + if (!info->options_len) + return 0; + + /* TCA_TUNNEL_KEY_ENC_OPTS and the per-protocol nest inside it */ + size = nla_total_size(0) + nla_total_size(0); + + if (test_bit(IP_TUNNEL_GENEVE_OPT_BIT, info->key.tun_flags)) { + size += tunnel_key_geneve_opts_fill_size(info); + } else if (test_bit(IP_TUNNEL_VXLAN_OPT_BIT, info->key.tun_flags)) { + /* TCA_TUNNEL_KEY_ENC_OPT_VXLAN_GBP */ + size += nla_total_size(sizeof(u32)); + } else if (test_bit(IP_TUNNEL_ERSPAN_OPT_BIT, info->key.tun_flags)) { + /* TCA_TUNNEL_KEY_ENC_OPT_ERSPAN_{VER,INDEX,DIR,HWID} */ + size += nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(__be32)) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); + } + + return size; +} + +static size_t tunnel_key_get_fill_size(const struct tc_action *act) +{ + struct tcf_tunnel_key *t = to_tunnel_key(act); + const struct tcf_tunnel_key_params *params; + /* TCA_TUNNEL_KEY_PARMS */ + size_t size = nla_total_size(sizeof(struct tc_tunnel_key)); + + rcu_read_lock(); + params = rcu_dereference(t->params); + if (params->tcft_action == TCA_TUNNEL_KEY_ACT_SET) { + const struct ip_tunnel_info *info = + ¶ms->tcft_enc_metadata->u.tun_info; + + /* In dump order: TCA_TUNNEL_KEY_ENC_KEY_ID, the IPv6 address + * pair (larger than the IPv4 one), ..._ENC_DST_PORT, + * ..._NO_CSUM, ..._NO_FRAG, the options and ..._ENC_{TOS,TTL}. + */ + size += nla_total_size(sizeof(__be32)) + + 2 * nla_total_size(sizeof(struct in6_addr)) + + nla_total_size(sizeof(__be16)) + + nla_total_size(sizeof(u8)) + + nla_total_size(0) + + tunnel_key_opts_fill_size(info) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_tunnel_key_ops = { .kind = "tunnel_key", .id = TCA_ID_TUNNEL_KEY, @@ -843,6 +922,7 @@ static struct tc_action_ops act_tunnel_key_ops = { .dump = tunnel_key_dump, .init = tunnel_key_init, .cleanup = tunnel_key_release, + .get_fill_size = tunnel_key_get_fill_size, .offload_act_setup = tcf_tunnel_key_offload_act_setup, .size = sizeof(struct tcf_tunnel_key), }; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 3271963c945d..9966766661d5 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -3372,7 +3372,8 @@ int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action, * This reference might be taken later from tcf_exts_get_net(). */ exts->net = net; - exts->actions = kzalloc_objs(struct tc_action *, TCA_ACT_MAX_PRIO); + exts->actions = kzalloc_objs(struct tc_action *, TCA_ACT_MAX_PRIO, + GFP_KERNEL_ACCOUNT); if (!exts->actions) return -ENOMEM; #endif diff --git a/net/sched/cls_basic.c b/net/sched/cls_basic.c index 492cd9ce8d46..e2a94ba9fba7 100644 --- a/net/sched/cls_basic.c +++ b/net/sched/cls_basic.c @@ -193,7 +193,7 @@ static int basic_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; @@ -212,9 +212,11 @@ static int basic_change(struct net *net, struct sk_buff *in_skb, if (err) goto errout; fnew->handle = handle; - fnew->pf = alloc_percpu(struct tc_basic_pcnt); + fnew->pf = alloc_percpu_gfp(struct tc_basic_pcnt, GFP_KERNEL_ACCOUNT); if (!fnew->pf) { err = -ENOMEM; + if (!fold) + idr_remove(&head->handle_idr, fnew->handle); goto errout; } diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index 6d19155becc8..188cf0f949dd 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -352,7 +352,7 @@ static int cls_bpf_prog_from_ops(struct nlattr **tb, struct cls_bpf_prog *prog) if (bpf_size != nla_len(tb[TCA_BPF_OPS])) return -EINVAL; - bpf_ops = kmemdup(nla_data(tb[TCA_BPF_OPS]), bpf_size, GFP_KERNEL); + bpf_ops = kmemdup(nla_data(tb[TCA_BPF_OPS]), bpf_size, GFP_KERNEL_ACCOUNT); if (bpf_ops == NULL) return -ENOMEM; @@ -403,7 +403,7 @@ static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog, } if (tb[TCA_BPF_NAME]) { - name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL); + name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL_ACCOUNT); if (!name) { bpf_prog_put(fp); return -ENOMEM; @@ -443,7 +443,7 @@ static int cls_bpf_change(struct net *net, struct sk_buff *in_skb, if (ret < 0) return ret; - prog = kzalloc_obj(*prog); + prog = kzalloc_obj(*prog, GFP_KERNEL_ACCOUNT); if (!prog) return -ENOBUFS; diff --git a/net/sched/cls_cgroup.c b/net/sched/cls_cgroup.c index 680a5c308094..210fd9fd26d8 100644 --- a/net/sched/cls_cgroup.c +++ b/net/sched/cls_cgroup.c @@ -95,7 +95,7 @@ static int cls_cgroup_change(struct net *net, struct sk_buff *in_skb, if (head && handle != head->handle) return -ENOENT; - new = kzalloc_obj(*head); + new = kzalloc_obj(*head, GFP_KERNEL_ACCOUNT); if (!new) return -ENOBUFS; diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index 356c68ebc389..a9ac3acf6eda 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -438,7 +438,7 @@ static int flow_change(struct net *net, struct sk_buff *in_skb, return -EOPNOTSUPP; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index 88f8a32fab2b..0e275b58151c 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -2233,7 +2233,7 @@ static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head, struct fl_flow_mask *newmask; int err; - newmask = kzalloc_obj(*newmask); + newmask = kzalloc_obj(*newmask, GFP_KERNEL_ACCOUNT); if (!newmask) return ERR_PTR(-ENOMEM); @@ -2394,7 +2394,7 @@ static int fl_change(struct net *net, struct sk_buff *in_skb, goto errout_tb; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) { err = -ENOBUFS; goto errout_tb; diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c index 646a730dca93..a462b262719c 100644 --- a/net/sched/cls_fw.c +++ b/net/sched/cls_fw.c @@ -276,7 +276,7 @@ static int fw_change(struct net *net, struct sk_buff *in_skb, if (f->id != handle && handle) return -EINVAL; - fnew = kzalloc_obj(struct fw_filter); + fnew = kzalloc_obj(struct fw_filter, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; @@ -330,7 +330,7 @@ static int fw_change(struct net *net, struct sk_buff *in_skb, rcu_assign_pointer(tp->root, head); } - f = kzalloc_obj(struct fw_filter); + f = kzalloc_obj(struct fw_filter, GFP_KERNEL_ACCOUNT); if (f == NULL) return -ENOBUFS; diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c index 6f126872c14a..c14899b935bf 100644 --- a/net/sched/cls_matchall.c +++ b/net/sched/cls_matchall.c @@ -189,7 +189,7 @@ static int mall_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; } - new = kzalloc_obj(*new); + new = kzalloc_obj(*new, GFP_KERNEL_ACCOUNT); if (!new) return -ENOBUFS; @@ -201,7 +201,7 @@ static int mall_change(struct net *net, struct sk_buff *in_skb, handle = 1; new->handle = handle; new->flags = userflags; - new->pf = alloc_percpu(struct tc_matchall_pcnt); + new->pf = alloc_percpu_gfp(struct tc_matchall_pcnt, GFP_KERNEL_ACCOUNT); if (!new->pf) { err = -ENOMEM; goto err_alloc_percpu; diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c index eded7aacd3f7..0d1324c90583 100644 --- a/net/sched/cls_route.c +++ b/net/sched/cls_route.c @@ -455,7 +455,7 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, h1 = to_hash(nhandle); b = rtnl_dereference(head->table[h1]); if (!b) { - b = kzalloc_obj(struct route4_bucket); + b = kzalloc_obj(struct route4_bucket, GFP_KERNEL_ACCOUNT); if (b == NULL) return -ENOBUFS; @@ -524,7 +524,7 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; err = -ENOBUFS; - f = kzalloc_obj(struct route4_filter); + f = kzalloc_obj(struct route4_filter, GFP_KERNEL_ACCOUNT); if (!f) goto errout; diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index c297d7dbcf91..ac6d0fa5a40e 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -825,7 +825,7 @@ static struct tc_u_knode *u32_init_knode(struct net *net, struct tcf_proto *tp, struct tc_u32_sel *s = &n->sel; struct tc_u_knode *new; - new = kzalloc_flex(*new, sel.keys, s->nkeys); + new = kzalloc_flex(*new, sel.keys, s->nkeys, GFP_KERNEL_ACCOUNT); if (!new) return NULL; @@ -1114,15 +1114,16 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, goto erridr; } - n = kzalloc_flex(*n, sel.keys, s->nkeys); + n = kzalloc_flex(*n, sel.keys, s->nkeys, GFP_KERNEL_ACCOUNT); if (n == NULL) { err = -ENOBUFS; goto erridr; } #ifdef CONFIG_CLS_U32_PERF - n->pf = __alloc_percpu(struct_size(n->pf, kcnts, s->nkeys), - __alignof__(struct tc_u32_pcnt)); + n->pf = __alloc_percpu_gfp(struct_size(n->pf, kcnts, s->nkeys), + __alignof__(struct tc_u32_pcnt), + GFP_KERNEL_ACCOUNT); if (!n->pf) { err = -ENOBUFS; goto errfree; @@ -1144,7 +1145,7 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, goto errout; #ifdef CONFIG_CLS_U32_MARK - n->pcpu_success = alloc_percpu(u32); + n->pcpu_success = alloc_percpu_gfp(u32, GFP_KERNEL_ACCOUNT); if (!n->pcpu_success) { err = -ENOMEM; goto errout; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 65b35528d125..90503e59e6e3 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -610,8 +610,11 @@ void __qdisc_calculate_pkt_len(struct sk_buff *skb, pkt_len <<= stab->szopts.size_log; out: - if (unlikely(pkt_len < 1)) - pkt_len = 1; + /* A size table can inflate qdisc_pkt_len() beyond any real packet + * (via overhead, the data table, or size_log); cap it so deficit + * schedulers such as DRR/ETS terminate their refill loops. + */ + pkt_len = clamp_t(int, pkt_len, 1, QDISC_PKT_LEN_MAX); qdisc_skb_cb(skb)->pkt_len = pkt_len; } diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index f25f60978631..dc93267029e7 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1907,6 +1907,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, ktime_add_ms(q->last_reconfig_time, 250))) { q->config->rate_bps = (q->avg_peak_bandwidth * 15) >> 4; + q->last_reconfig_time = now; cake_reconfigure(sch); } } diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index cacf5244958e..6aa5829d6961 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -205,7 +205,7 @@ static int codel_init(struct Qdisc *sch, struct nlattr *opt, codel_params_init(&q->params); codel_vars_init(&q->vars); codel_stats_init(&q->stats); - q->params.mtu = psched_mtu(qdisc_dev(sch)); + q->params.mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, 1 << 20); if (opt) { int err = codel_change(sch, opt, extack); diff --git a/net/sched/sch_fifo.c b/net/sched/sch_fifo.c index e6bfd39ff339..1b6388d50967 100644 --- a/net/sched/sch_fifo.c +++ b/net/sched/sch_fifo.c @@ -19,7 +19,7 @@ static int bfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) { - if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= + if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <= READ_ONCE(sch->limit))) return qdisc_enqueue_tail(skb, sch); diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index 7cae082a9847..6144b5686f13 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -828,8 +828,12 @@ begin: * f->time_next_packet was set when prior packet was sent, * and current time (@now) can be too late by tens of us. */ - if (f->time_next_packet) - len -= min(len/2, now - f->time_next_packet); + if (f->time_next_packet) { + s64 drift = now - f->time_next_packet; + + if (drift > 0) + len -= min_t(u64, len / 2, drift); + } f->time_next_packet = now + len; } out: @@ -1222,12 +1226,14 @@ static int fq_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct fq_sched_data *q = qdisc_priv(sch); + u32 mtu; int i, err; sch->limit = 10000; q->flow_plimit = 100; - q->quantum = 2 * psched_mtu(qdisc_dev(sch)); - q->initial_quantum = 10 * psched_mtu(qdisc_dev(sch)); + mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); + q->quantum = min_t(u32, 2 * mtu, 1 << 20); + q->initial_quantum = min_t(u32, 10 * mtu, 1 << 20); q->flow_refill_delay = msecs_to_jiffies(40); q->flow_max_rate = ~0UL; q->time_next_delayed_flow = ~0ULL; diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 6cce86ba383c..969b2510b0b8 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -509,6 +509,7 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct fq_codel_sched_data *q = qdisc_priv(sch); + u32 mtu; int i; int err; @@ -516,13 +517,14 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, q->flows_cnt = 1024; q->memory_limit = 32 << 20; /* 32 MBytes */ q->drop_batch_size = 64; - q->quantum = psched_mtu(qdisc_dev(sch)); + mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, FQ_CODEL_QUANTUM_MAX); + q->quantum = mtu; INIT_LIST_HEAD(&q->new_flows); INIT_LIST_HEAD(&q->old_flows); codel_params_init(&q->cparams); codel_stats_init(&q->cstats); q->cparams.ecn = true; - q->cparams.mtu = psched_mtu(qdisc_dev(sch)); + q->cparams.mtu = mtu; if (opt) { err = fq_codel_change(sch, opt, extack); diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 069e1facd413..b27d95418707 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -427,7 +427,8 @@ static int fq_pie_init(struct Qdisc *sch, struct nlattr *opt, pie_params_init(&q->p_params); sch->limit = 10 * 1024; q->p_params.limit = sch->limit; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); q->sch = sch; q->ecn_prob = 10; q->flows_cnt = 1024; diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index ef2b4bf51564..4539dc2c6d38 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -1278,7 +1278,7 @@ static void transition_one_qdisc(struct net_device *dev, rcu_assign_pointer(dev_queue->qdisc, new_qdisc); if (need_watchdog_p) { - WRITE_ONCE(dev_queue->trans_start, 0); + WRITE_ONCE(dev_queue->trans_start, jiffies); *need_watchdog_p = 1; } } diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c index fcc1a4c03636..f04f425c6c44 100644 --- a/net/sched/sch_gred.c +++ b/net/sched/sch_gred.c @@ -179,7 +179,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch, * if no default DP has been configured. This * allows for DP flows to be left untouched. */ - if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= + if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <= sch->limit)) return qdisc_enqueue_tail(skb, sch); else @@ -244,7 +244,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch, break; } - if (gred_backlog(t, q, sch) + qdisc_pkt_len(skb) <= q->limit) { + if ((u64)gred_backlog(t, q, sch) + qdisc_pkt_len(skb) <= q->limit) { q->backlog += qdisc_pkt_len(skb); return qdisc_enqueue_tail(skb, sch); } diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index d85cb0263b67..96acab6a8da0 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -624,6 +624,10 @@ static int hhf_init(struct Qdisc *sch, struct nlattr *opt, q->hhf_evict_timeout = HZ; /* 1 sec */ q->hhf_non_hh_weight = 2; + if ((int)q->quantum <= 0 || + (u64)q->quantum * q->hhf_non_hh_weight > INT_MAX) + q->quantum = 256; + if (opt) { int err = hhf_change(sch, opt, extack); diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index fdac0dc8f35a..1ba67b121de4 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -224,6 +224,7 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, struct htb_class *cl; struct tcf_result res; struct tcf_proto *tcf; + unsigned int hops = 0; int result; /* allow to select class by setting skb->priority to valid classid; @@ -266,6 +267,10 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, if (!cl->level) return cl; /* we hit leaf; return it */ + if (++hops > TC_HTB_MAXDEPTH) { + pr_warn_ratelimited("htb: classify loop detected, dropping packet\n"); + return NULL; + } /* we have got inner class; apply inner filter chain */ tcf = rcu_dereference_bh(cl->filter_list); } @@ -633,13 +638,11 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } else { return qdisc_drop(skb, sch, to_free); } -#ifdef CONFIG_NET_CLS_ACT } else if (!cl) { if (ret & __NET_XMIT_BYPASS) qdisc_qstats_drop(sch); __qdisc_drop(skb, to_free); return ret; -#endif } else if ((ret = qdisc_enqueue(skb, cl->leaf.q, to_free)) != NET_XMIT_SUCCESS) { if (net_xmit_drop_count(ret)) { diff --git a/net/sched/sch_plug.c b/net/sched/sch_plug.c index cefb65201e17..b60ddfee6a68 100644 --- a/net/sched/sch_plug.c +++ b/net/sched/sch_plug.c @@ -89,7 +89,7 @@ static int plug_enqueue(struct sk_buff *skb, struct Qdisc *sch, { struct plug_sched_data *q = qdisc_priv(sch); - if (likely(sch->qstats.backlog + skb->len <= q->limit)) { + if (likely((u64)sch->qstats.backlog + skb->len <= q->limit)) { if (!q->unplug_indefinite) q->pkts_current_epoch++; return qdisc_enqueue_tail(skb, sch); diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index 77675f9a4c46..187d3ed578f2 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -799,7 +799,8 @@ static int sfq_init(struct Qdisc *sch, struct nlattr *opt, q->tail = NULL; q->divisor = SFQ_DEFAULT_HASH_DIVISOR; q->maxflows = SFQ_DEFAULT_FLOWS; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); q->perturb_period = 0; get_random_bytes(&q->perturbation, sizeof(q->perturbation)); diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 5c42a29a981c..9e52afc2d980 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -377,6 +377,7 @@ restart: nores = 1; break; } + skb->dev = dev; __skb_pull(skb, skb_network_offset(skb)); } while ((q = rcu_dereference(NEXT_SLAVE(q))) != start); diff --git a/net/sctp/inqueue.c b/net/sctp/inqueue.c index a024c0843247..5f988b3a8814 100644 --- a/net/sctp/inqueue.c +++ b/net/sctp/inqueue.c @@ -71,8 +71,11 @@ void sctp_inq_free(struct sctp_inq *queue) */ void sctp_inq_push(struct sctp_inq *q, struct sctp_chunk *chunk) { - /* Directly call the packet handling routine. */ - if (chunk->rcvr->dead) { + /* Directly call the packet handling routine. Drop the chunk if the + * receiver or the transport it was looked up on is gone. + */ + if (chunk->rcvr->dead || + (chunk->transport && chunk->transport->dead)) { sctp_chunk_free(chunk); return; } diff --git a/net/sctp/stream.c b/net/sctp/stream.c index 34ffe6c945a4..e4354d58f20b 100644 --- a/net/sctp/stream.c +++ b/net/sctp/stream.c @@ -22,6 +22,15 @@ #include <net/sctp/sm.h> #include <net/sctp/stream_sched.h> +#define SCTP_STRRESET_MASK(type) \ + BIT(ntohs(type) - ntohs(SCTP_PARAM_RESET_OUT_REQUEST)) +#define SCTP_STRRESET_TEST(asoc, type) \ + ((asoc)->strreset_outstanding & SCTP_STRRESET_MASK(type)) +#define SCTP_STRRESET_SET(asoc, type) \ + ((asoc)->strreset_outstanding |= SCTP_STRRESET_MASK(type)) +#define SCTP_STRRESET_CLEAR(asoc, type) \ + ((asoc)->strreset_outstanding &= ~SCTP_STRRESET_MASK(type)) + static void sctp_stream_shrink_out(struct sctp_stream *stream, __u16 outcnt) { struct sctp_association *asoc; @@ -372,7 +381,10 @@ int sctp_send_reset_streams(struct sctp_association *asoc, goto out; } - asoc->strreset_outstanding = out + in; + if (out) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_OUT_REQUEST); + if (in) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_IN_REQUEST); out: return retval; @@ -417,7 +429,7 @@ int sctp_send_reset_assoc(struct sctp_association *asoc) return retval; } - asoc->strreset_outstanding = 1; + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_TSN_REQUEST); return 0; } @@ -474,7 +486,10 @@ int sctp_send_add_streams(struct sctp_association *asoc, goto out; } - asoc->strreset_outstanding = !!out + !!in; + if (out) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_ADD_OUT_STREAMS); + if (in) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_ADD_IN_STREAMS); out: return retval; @@ -482,13 +497,13 @@ out: static struct sctp_paramhdr *sctp_chunk_lookup_strreset_param( struct sctp_association *asoc, __be32 resp_seq, - __be16 type) + __be16 type, bool match_seq) { struct sctp_chunk *chunk = asoc->strreset_chunk; struct sctp_reconf_chunk *hdr; union sctp_params param; - if (!chunk) + if (!chunk || !chunk->transport) return NULL; hdr = (struct sctp_reconf_chunk *)chunk->chunk_hdr; @@ -499,7 +514,7 @@ static struct sctp_paramhdr *sctp_chunk_lookup_strreset_param( */ struct sctp_strreset_tsnreq *req = param.v; - if ((!resp_seq || req->request_seq == resp_seq) && + if ((!match_seq || req->request_seq == resp_seq) && (!type || type == req->param_hdr.type)) return param.v; } @@ -564,13 +579,14 @@ struct sctp_chunk *sctp_process_strreset_outreq( if (asoc->strreset_chunk) { if (!sctp_chunk_lookup_strreset_param( asoc, outreq->response_seq, - SCTP_PARAM_RESET_IN_REQUEST)) { + SCTP_PARAM_RESET_IN_REQUEST, true) || + !SCTP_STRRESET_TEST(asoc, SCTP_PARAM_RESET_IN_REQUEST)) { /* same process with outstanding isn't 0 */ result = SCTP_STRRESET_ERR_IN_PROGRESS; goto out; } - asoc->strreset_outstanding--; + SCTP_STRRESET_CLEAR(asoc, SCTP_PARAM_RESET_IN_REQUEST); asoc->strreset_outseq++; if (!asoc->strreset_outstanding) { @@ -669,7 +685,7 @@ struct sctp_chunk *sctp_process_strreset_inreq( SCTP_SO(stream, i)->state = SCTP_STREAM_CLOSED; asoc->strreset_chunk = chunk; - asoc->strreset_outstanding = 1; + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_OUT_REQUEST); sctp_chunk_hold(asoc->strreset_chunk); result = SCTP_STRRESET_PERFORMED; @@ -816,13 +832,14 @@ struct sctp_chunk *sctp_process_strreset_addstrm_out( if (asoc->strreset_chunk) { if (!sctp_chunk_lookup_strreset_param( - asoc, 0, SCTP_PARAM_RESET_ADD_IN_STREAMS)) { + asoc, 0, SCTP_PARAM_RESET_ADD_IN_STREAMS, false) || + !SCTP_STRRESET_TEST(asoc, SCTP_PARAM_RESET_ADD_IN_STREAMS)) { /* same process with outstanding isn't 0 */ result = SCTP_STRRESET_ERR_IN_PROGRESS; goto out; } - asoc->strreset_outstanding--; + SCTP_STRRESET_CLEAR(asoc, SCTP_PARAM_RESET_ADD_IN_STREAMS); asoc->strreset_outseq++; if (!asoc->strreset_outstanding) { @@ -899,7 +916,7 @@ struct sctp_chunk *sctp_process_strreset_addstrm_in( goto out; asoc->strreset_chunk = chunk; - asoc->strreset_outstanding = 1; + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_ADD_OUT_STREAMS); sctp_chunk_hold(asoc->strreset_chunk); stream->outcnt = outcnt; @@ -927,8 +944,9 @@ struct sctp_chunk *sctp_process_strreset_resp( struct sctp_paramhdr *req; __u32 result; - req = sctp_chunk_lookup_strreset_param(asoc, resp->response_seq, 0); - if (!req) + req = sctp_chunk_lookup_strreset_param(asoc, resp->response_seq, 0, + true); + if (!req || !SCTP_STRRESET_TEST(asoc, req->type)) return NULL; result = ntohl(resp->result); @@ -1078,7 +1096,7 @@ struct sctp_chunk *sctp_process_strreset_resp( nums, 0, GFP_ATOMIC); } - asoc->strreset_outstanding--; + SCTP_STRRESET_CLEAR(asoc, req->type); asoc->strreset_outseq++; /* remove everything for this reconf request */ diff --git a/net/smc/smc.h b/net/smc/smc.h index 52145df83f6e..427b6d63b993 100644 --- a/net/smc/smc.h +++ b/net/smc/smc.h @@ -277,9 +277,9 @@ struct smc_connection { * 0 for SMC-R, 32 for SMC-D */ u64 peer_token; /* SMC-D token of peer */ - u8 killed : 1; /* abnormal termination */ - u8 freed : 1; /* normal termination */ - u8 out_of_sync : 1; /* out of sync with peer */ + u8 killed; /* abnormal termination */ + u8 freed; /* normal termination */ + u8 out_of_sync; /* out of sync with peer */ }; struct smc_sock { /* smc sock container */ diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c index 181647982490..04aedd957543 100644 --- a/net/smc/smc_core.c +++ b/net/smc/smc_core.c @@ -1148,13 +1148,13 @@ again: read_unlock_bh(&lgr->conns_lock); /* pre-fetch buffer outside of send_lock, might sleep */ rc = smc_cdc_get_free_slot(conn, to_lnk, &wr_buf, NULL, &pend); - if (rc) - goto err_out; - /* avoid race with smcr_tx_sndbuf_nonempty() */ - spin_lock_bh(&conn->send_lock); - smc_switch_link_and_count(conn, to_lnk); - rc = smc_switch_cursor(smc, pend, wr_buf); - spin_unlock_bh(&conn->send_lock); + if (!rc) { + /* avoid race with smcr_tx_sndbuf_nonempty() */ + spin_lock_bh(&conn->send_lock); + smc_switch_link_and_count(conn, to_lnk); + rc = smc_switch_cursor(smc, pend, wr_buf); + spin_unlock_bh(&conn->send_lock); + } sock_put(&smc->sk); if (rc) goto err_out; diff --git a/net/smc/smc_inet.c b/net/smc/smc_inet.c index a94084b4a498..520b666fdd8f 100644 --- a/net/smc/smc_inet.c +++ b/net/smc/smc_inet.c @@ -15,13 +15,16 @@ #include "smc_inet.h" #include "smc.h" +#include "smc_close.h" static int smc_inet_init_sock(struct sock *sk); +static void smc_inet_destroy_sock(struct sock *sk); static struct proto smc_inet_prot = { .name = "INET_SMC", .owner = THIS_MODULE, .init = smc_inet_init_sock, + .destroy = smc_inet_destroy_sock, .hash = smc_hash_sk, .unhash = smc_unhash_sk, .release_cb = smc_release_cb, @@ -68,6 +71,7 @@ static struct proto smc_inet6_prot = { .name = "INET6_SMC", .owner = THIS_MODULE, .init = smc_inet_init_sock, + .destroy = smc_inet_destroy_sock, .hash = smc_hash_sk, .unhash = smc_unhash_sk, .release_cb = smc_release_cb, @@ -116,6 +120,18 @@ static int smc_inet_init_sock(struct sock *sk) return smc_create_clcsk(net, sk, sk->sk_family); } +static void smc_inet_destroy_sock(struct sock *sk) +{ + /* The sock is hashed and smc_diag dumps dereference smc->clcsock + * without clcsock_release_lock, while sk_common_release() calls + * .destroy before .unhash. Unhash first, as __smc_release() does, + * so no dump can observe the clcsock being released; the second + * unhash is a no-op. + */ + sk->sk_prot->unhash(sk); + smc_clcsock_release(smc_sk(sk)); +} + int __init smc_inet_init(void) { int rc; diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c index aa6d83af55ed..c9ccd8480369 100644 --- a/net/smc/smc_llc.c +++ b/net/smc/smc_llc.c @@ -157,6 +157,7 @@ struct smc_llc_msg_confirm_rkey { /* type 0x06 */ }; #define SMC_LLC_DEL_RKEY_MAX 8 +#define SMC_LLC_DEL_RKEY_V2_INLINE 9 #define SMC_LLC_FLAG_RKEY_RETRY 0x10 #define SMC_LLC_FLAG_RKEY_NEG 0x20 @@ -177,6 +178,15 @@ struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */ __be32 rkey[]; }; +/* the leading rkeys of a DELETE_RKEY_V2 fit into union smc_llc_msg */ +struct smc_llc_msg_delete_rkey_v2_inline { /* type 0x29 */ + struct smc_llc_hdr hd; + u8 num_rkeys; + u8 num_inval_rkeys; + u8 reserved[2]; + __be32 rkey[SMC_LLC_DEL_RKEY_V2_INLINE]; +}; + union smc_llc_msg { struct smc_llc_msg_confirm_link confirm_link; struct smc_llc_msg_add_link add_link; @@ -186,6 +196,7 @@ union smc_llc_msg { struct smc_llc_msg_confirm_rkey confirm_rkey; struct smc_llc_msg_delete_rkey delete_rkey; + struct smc_llc_msg_delete_rkey_v2_inline delete_rkey_v2; struct smc_llc_msg_test_link test_link; struct { @@ -194,15 +205,25 @@ union smc_llc_msg { } raw; }; +static_assert(SMC_LLC_DEL_RKEY_V2_INLINE == + (sizeof(union smc_llc_msg) - + offsetof(struct smc_llc_msg_delete_rkey_v2, rkey)) / + sizeof(__be32)); +static_assert(offsetof(struct smc_llc_msg_delete_rkey_v2_inline, rkey) == + offsetof(struct smc_llc_msg_delete_rkey_v2, rkey)); + #define SMC_LLC_FLAG_RESP 0x80 struct smc_llc_qentry { struct list_head list; struct smc_link *link; + u16 body_len; union smc_llc_msg msg; + u8 body[] __counted_by(body_len); }; -static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); +static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc, + u32 byte_len); struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) { @@ -320,6 +341,7 @@ again: void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) { spin_lock_bh(&lgr->llc_flow_lock); + smc_llc_flow_qentry_del(flow); memset(flow, 0, sizeof(*flow)); flow->type = SMC_LLC_FLOW_NONE; spin_unlock_bh(&lgr->llc_flow_lock); @@ -998,15 +1020,20 @@ static int smc_llc_cli_conf_link(struct smc_link *link, static void smc_llc_save_add_link_rkeys(struct smc_link *link, struct smc_link *link_new, - u8 *llc_msg) + struct smc_llc_qentry *qentry) { + const u32 rt_off = offsetof(struct smc_llc_msg_add_link_v2_ext, rt); struct smc_llc_msg_add_link_v2_ext *ext; struct smc_link_group *lgr = link->lgr; int max, i; - ext = (struct smc_llc_msg_add_link_v2_ext *)(llc_msg + - SMC_WR_TX_SIZE); + /* the rkey count itself is only there if enough bytes arrived */ + if (qentry->body_len < rt_off) + return; + ext = (struct smc_llc_msg_add_link_v2_ext *)qentry->body; max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); + max = min_t(u32, max, (qentry->body_len - rt_off) / + sizeof(ext->rt[0])); down_write(&lgr->rmbs_lock); for (i = 0; i < max; i++) { smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, @@ -1099,9 +1126,7 @@ int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) if (rc) goto out_clear_lnk; if (lgr->smc_version == SMC_V2) { - u8 *llc_msg = smc_link_shared_v2_rxbuf(link) ? - (u8 *)lgr->wr_rx_buf_v2 : (u8 *)llc; - smc_llc_save_add_link_rkeys(link, lnk_new, llc_msg); + smc_llc_save_add_link_rkeys(link, lnk_new, qentry); } else { rc = smc_llc_cli_rkey_exchange(link, lnk_new); if (rc) { @@ -1481,7 +1506,7 @@ int smc_llc_srv_add_link(struct smc_link *link, } add_llc = &qentry->msg.add_link; if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { - smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); + smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); rc = -ENOLINK; goto out_err; } @@ -1492,7 +1517,8 @@ int smc_llc_srv_add_link(struct smc_link *link, lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; } smc_llc_save_add_link_info(link_new, add_llc); - smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); + /* add_llc still points into qentry, so only detach it here */ + smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); rc = smc_ib_ready_link(link_new); if (rc) @@ -1501,9 +1527,7 @@ int smc_llc_srv_add_link(struct smc_link *link, if (rc) goto out_err; if (lgr->smc_version == SMC_V2) { - u8 *llc_msg = smc_link_shared_v2_rxbuf(link) ? - (u8 *)lgr->wr_rx_buf_v2 : (u8 *)add_llc; - smc_llc_save_add_link_rkeys(link, link_new, llc_msg); + smc_llc_save_add_link_rkeys(link, link_new, qentry); } else { rc = smc_llc_srv_rkey_exchange(link, link_new); if (rc) @@ -1512,14 +1536,14 @@ int smc_llc_srv_add_link(struct smc_link *link, rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); if (rc) goto out_err; - kfree(ini); - return 0; + goto out; out_err: if (link_new) { link_new->state = SMC_LNK_INACTIVE; smcr_link_clear(link_new, false); } out: + kfree(qentry); kfree(ini); if (send_req_add_link_resp) smc_llc_send_req_add_link_response(req_qentry); @@ -1552,7 +1576,8 @@ void smc_llc_add_link_local(struct smc_link *link) add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK; smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc)); /* no dev and port needed */ - smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); + smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc, + sizeof(union smc_llc_msg)); } /* worker to process an add link message */ @@ -1588,7 +1613,8 @@ void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) del_llc.link_num = del_link_id; del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; - smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); + smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc, + sizeof(union smc_llc_msg)); } static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) @@ -1810,19 +1836,28 @@ static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) link = qentry->link; if (lgr->smc_version == SMC_V2) { - struct smc_llc_msg_delete_rkey_v2 *llcv2; + struct smc_llc_msg_delete_rkey_v2_inline *llcv2; - if (smc_link_shared_v2_rxbuf(link)) { - memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc)); - llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2; - } else { - llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)llc; - } + /* The leading SMC_LLC_DEL_RKEY_V2_INLINE rkeys are declared in + * the message itself, any further ones were received into + * qentry->body. + */ + llcv2 = &qentry->msg.delete_rkey_v2; llcv2->num_inval_rkeys = 0; max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); + max = min_t(u32, max, SMC_LLC_DEL_RKEY_V2_INLINE + + qentry->body_len / sizeof(__be32)); for (i = 0; i < max; i++) { - if (smc_rtoken_delete(link, llcv2->rkey[i])) + __be32 rkey; + + if (i < SMC_LLC_DEL_RKEY_V2_INLINE) + rkey = llcv2->rkey[i]; + else + memcpy(&rkey, qentry->body + + (i - SMC_LLC_DEL_RKEY_V2_INLINE) * + sizeof(rkey), sizeof(rkey)); + if (smc_rtoken_delete(link, rkey)) llcv2->num_inval_rkeys++; } memset(&llc->rkey[0], 0, sizeof(llc->rkey)); @@ -1911,6 +1946,8 @@ static void smc_llc_event_handler(struct smc_llc_qentry *qentry) if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK) { /* server started add_link processing */ + /* free any qentry stashed in REQ_ADD_LINK state */ + smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK; smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); @@ -2064,18 +2101,52 @@ assign: wake_up(&link->lgr->llc_msg_waiter); } -static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) +/* the longest tail either reader of qentry->body can use */ +static u32 smc_llc_max_body_len(union smc_llc_msg *llc) +{ + switch (llc->raw.hdr.common.llc_type) { + case SMC_LLC_ADD_LINK: + return offsetof(struct smc_llc_msg_add_link_v2_ext, rt) + + SMC_LLC_RKEYS_PER_MSG_V2 * + sizeof(struct smc_llc_msg_add_link_cont_rt); + case SMC_LLC_DELETE_RKEY: + return (SMC_LLC_RKEYS_PER_MSG_V2 - + SMC_LLC_DEL_RKEY_V2_INLINE) * sizeof(__be32); + default: + return 0; + } +} + +static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc, + u32 byte_len) { struct smc_link_group *lgr = link->lgr; struct smc_llc_qentry *qentry; unsigned long flags; + u16 body_len = 0; + + /* V2 messages can be longer than the inline union smc_llc_msg. Carry + * the remainder in the qentry itself, so that its lifetime and its + * length match the message the peer actually sent. + */ + if (lgr->smc_version == SMC_V2 && byte_len > SMC_WR_TX_SIZE) + body_len = min_t(u32, byte_len, SMC_WR_BUF_V2_SIZE) - + SMC_WR_TX_SIZE; + body_len = min_t(u32, body_len, smc_llc_max_body_len(llc)); - qentry = kmalloc_obj(*qentry, GFP_ATOMIC); + qentry = kmalloc_flex(*qentry, body, body_len, GFP_ATOMIC); if (!qentry) return; + qentry->body_len = body_len; qentry->link = link; INIT_LIST_HEAD(&qentry->list); memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); + if (body_len) { + u8 *src = smc_link_shared_v2_rxbuf(link) ? + (u8 *)lgr->wr_rx_buf_v2 : (u8 *)llc; + + memcpy(qentry->body, src + SMC_WR_TX_SIZE, body_len); + } /* process responses immediately */ if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) && @@ -2107,7 +2178,7 @@ static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) return; /* invalid message */ } - smc_llc_enqueue(link, llc); + smc_llc_enqueue(link, llc, wc->byte_len); } /***************************** worker, utils *********************************/ diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c index 5c9e4d8b57de..197fddc6271f 100644 --- a/net/smc/smc_rx.c +++ b/net/smc/smc_rx.c @@ -115,16 +115,15 @@ static void smc_rx_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { struct smc_spd_priv *priv = (struct smc_spd_priv *)buf->private; + struct smc_connection *conn = &priv->smc->conn; struct smc_sock *smc = priv->smc; - struct smc_connection *conn; struct sock *sk = &smc->sk; - if (sk->sk_state == SMC_CLOSED || - sk->sk_state == SMC_PEERFINCLOSEWAIT || - sk->sk_state == SMC_APPFINCLOSEWAIT) - goto out; - conn = &smc->conn; lock_sock(sk); + if (conn->freed) { + release_sock(sk); + goto out; + } smc_rx_update_cons(smc, priv->len); release_sock(sk); if (atomic_sub_and_test(priv->len, &conn->splice_pending)) diff --git a/net/smc/smc_wr.c b/net/smc/smc_wr.c index 59c92b46945c..97ba46893b17 100644 --- a/net/smc/smc_wr.c +++ b/net/smc/smc_wr.c @@ -602,9 +602,9 @@ static void smc_wr_init_sge(struct smc_link *lnk) /* With SMC-Rv2 there can be messages larger than SMC_WR_TX_SIZE. * Each ib_recv_wr gets 2 sges, the second one is a spillover buffer - * and the same buffer for all sges. When a larger message arrived then - * the content of the first small sge is copied to the beginning of - * the larger spillover buffer, allowing easy data mapping. + * and the same buffer for all sges. The spillover sge starts at + * SMC_WR_TX_SIZE, so the leading bytes of that buffer are never + * written. */ for (i = 0; i < lnk->wr_rx_cnt; i++) { int x = i * lnk->wr_rx_sge_cnt; diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 37bb06a8e8f5..f11d0528fc43 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -531,7 +531,8 @@ handle_error: if (!size) { last_record: tls_push_record_flags = flags; - if (flags & MSG_MORE) { + if ((flags & MSG_MORE) && + record->num_frags < MAX_SKB_FRAGS - 1) { more = true; break; } diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c index 96c9fe8d357c..4f9aa9c4c3aa 100644 --- a/net/vmw_vsock/virtio_transport.c +++ b/net/vmw_vsock/virtio_transport.c @@ -872,10 +872,10 @@ static void virtio_vsock_remove(struct virtio_device *vdev) /* Other works can be queued before 'config->del_vqs()', so we flush * all works before to free the vsock object to avoid use after free. */ - flush_work(&vsock->rx_work); flush_work(&vsock->tx_work); flush_work(&vsock->event_work); flush_work(&vsock->send_pkt_work); + flush_work(&vsock->rx_work); mutex_unlock(&the_virtio_vsock_mutex); diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 7855ee09c4b6..33475b180ea6 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool); void *copy_from = xsk_copy_xdp_start(xdp), *copy_to; u32 from_len, meta_len, rem, num_desc; - struct xdp_buff_xsk *xskb; + struct xdp_buff_xsk *xskb, *tmp; struct xdp_buff *xsk_xdp; + LIST_HEAD(xsk_buffs); skb_frag_t *frag; + u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from; @@ -343,23 +345,45 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0]; } + for (i = 0; i < num_desc; i++) { + xsk_xdp = xsk_buff_alloc(xs->pool); + if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); + if (unlikely(!list_empty(&xskb->list_node))) + goto err_alloc; + + list_add_tail(&xskb->list_node, &xsk_buffs); + } + do { u32 to_len = frame_size + meta_len; u32 copied; - xsk_xdp = xsk_buff_alloc(xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); + list_del_init(&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied = xsk_copy_xdp(copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); __xsk_rcv_zc_safe(xs, xskb, copied - meta_len, rem ? XDP_PKT_CONTD : 0); meta_len = 0; } while (rem); return 0; + +err_alloc: + list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) { + list_del_init(&xskb->list_node); + xsk_buff_free(&xskb->xdp); + } + xs->rx_dropped++; + return -ENOMEM; } static bool xsk_tx_writeable(struct xdp_sock *xs) diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index 78c14f106395..9d2d94f1fb75 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -763,6 +763,7 @@ EXPORT_SYMBOL(xp_raw_get_dma); * xp_raw_get_ctx - get &xdp_desc context * @pool: XSk buff pool desc address belongs to * @addr: desc address (from userspace) + * @options: desc options (from userspace) * * Helper for getting desc's DMA address and metadata pointer, if present. * Saves one call on hotpath and double calculation of the actual address. @@ -771,14 +772,16 @@ EXPORT_SYMBOL(xp_raw_get_dma); * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata * pointer, if it is present (initialized to %NULL otherwise). */ -struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, + u32 options) { struct xdp_desc_ctx ret; addr = __xp_raw_get_addr(pool, addr); ret.dma = __xp_raw_get_dma(pool, addr); - ret.meta = __xsk_buff_get_metadata(pool, __xp_raw_get_data(pool, addr)); + ret.meta = __xsk_buff_get_metadata(pool, __xp_raw_get_data(pool, addr), + options); return ret; } diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index 374e1b964438..674aedc5af5a 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -37,6 +37,11 @@ static void handle_esp(struct sk_buff *skb, struct sock *sk) rcu_read_lock(); skb->dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif); + if (!skb->dev) { + XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR); + kfree_skb(skb); + goto out; + } local_bh_disable(); #if IS_ENABLED(CONFIG_IPV6) if (sk->sk_family == AF_INET6) @@ -45,6 +50,7 @@ static void handle_esp(struct sk_buff *skb, struct sock *sk) #endif xfrm4_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP); local_bh_enable(); +out: rcu_read_unlock(); } @@ -515,7 +521,8 @@ static void espintcp_close(struct sock *sk, long timeout) strp_stop(&ctx->strp); sk->sk_prot = &tcp_prot; - barrier(); + + synchronize_rcu(); disable_work_sync(&ctx->work); strp_done(&ctx->strp); diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index 0e21ab72f0fe..8ed30e6c263b 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c @@ -155,25 +155,50 @@ static void nat_keepalive_send(struct nat_keepalive *ka) } } +enum { + NAT_KEEPALIVE_BATCH_SIZE = 16, + NAT_KEEPALIVE_BATCH_FULL = 1, +}; + struct nat_keepalive_work_ctx { + struct xfrm_state *batch[NAT_KEEPALIVE_BATCH_SIZE]; + unsigned int nr; time64_t next_run; time64_t now; }; -static int nat_keepalive_work_single(struct xfrm_state *x, int count, void *ptr) +static int nat_keepalive_work_collect(struct xfrm_state *x, int count, void *ptr) { struct nat_keepalive_work_ctx *ctx = ptr; + + if (!READ_ONCE(x->nat_keepalive_interval)) + return 0; + + if (ctx->nr == ARRAY_SIZE(ctx->batch)) + return NAT_KEEPALIVE_BATCH_FULL; + + xfrm_state_hold(x); + ctx->batch[ctx->nr++] = x; + return 0; +} + +static void nat_keepalive_work_single(struct xfrm_state *x, + struct nat_keepalive_work_ctx *ctx) +{ bool send_keepalive = false; struct nat_keepalive ka; - time64_t next_run; + time64_t next_run = 0; u32 interval; int delta; + spin_lock_bh(&x->lock); + + if (x->km.state == XFRM_STATE_DEAD) + goto out; + interval = x->nat_keepalive_interval; if (!interval) - return 0; - - spin_lock(&x->lock); + goto out; delta = (int)(ctx->now - x->lastused); if (delta < interval) { @@ -187,14 +212,14 @@ static int nat_keepalive_work_single(struct xfrm_state *x, int count, void *ptr) send_keepalive = true; } - spin_unlock(&x->lock); +out: + spin_unlock_bh(&x->lock); if (send_keepalive) nat_keepalive_send(&ka); - if (!ctx->next_run || next_run < ctx->next_run) + if (next_run && (!ctx->next_run || next_run < ctx->next_run)) ctx->next_run = next_run; - return 0; } static void nat_keepalive_work(struct work_struct *work) @@ -202,13 +227,23 @@ static void nat_keepalive_work(struct work_struct *work) struct nat_keepalive_work_ctx ctx; struct xfrm_state_walk walk; struct net *net; + int err, i; ctx.next_run = 0; ctx.now = ktime_get_real_seconds(); net = container_of(work, struct net, xfrm.nat_keepalive_work.work); xfrm_state_walk_init(&walk, IPPROTO_ESP, NULL); - xfrm_state_walk(net, &walk, nat_keepalive_work_single, &ctx); + do { + ctx.nr = 0; + err = xfrm_state_walk(net, &walk, nat_keepalive_work_collect, &ctx); + local_bh_disable(); + for (i = 0; i < ctx.nr; i++) { + nat_keepalive_work_single(ctx.batch[i], &ctx); + xfrm_state_put(ctx.batch[i]); + } + local_bh_enable(); + } while (err == NAT_KEEPALIVE_BATCH_FULL); xfrm_state_walk_done(&walk, net); if (ctx.next_run) schedule_delayed_work(&net->xfrm.nat_keepalive_work, diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index cc35c2fcbbe0..e305ba32e356 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c @@ -636,10 +636,8 @@ static int xfrm_dev_direct_output(struct sock *sk, struct xfrm_state *x, nf_reset_ct(skb); err = skb_dst(skb)->ops->local_out(net, sk, skb); - if (unlikely(err != 1)) { - kfree_skb(skb); + if (unlikely(err != 1)) return err; - } /* In transport mode, network destination is * directly reachable, while in tunnel mode, diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index d6db63304ba6..6266a92cf302 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -940,7 +940,7 @@ static struct xfrm_state *xfrm_state_construct(struct net *net, if ((err = attach_auth_trunc(&x->aalg, &x->props.aalgo, attrs[XFRMA_ALG_AUTH_TRUNC], extack))) goto error; - if (!x->props.aalgo) { + if (!x->aalg) { if ((err = attach_auth(&x->aalg, &x->props.aalgo, attrs[XFRMA_ALG_AUTH], extack))) goto error; |