diff options
Diffstat (limited to 'net/sched')
34 files changed, 336 insertions, 48 deletions
diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 600b7804befd..b4415d358c91 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -452,7 +452,10 @@ static size_t tcf_action_shared_attrs_size(const struct tc_action *act) /* TCA_STATS_QUEUE */ + nla_total_size_64bit(sizeof(struct gnet_stats_queue)) + nla_total_size(0) /* TCA_ACT_OPTIONS nested */ - + nla_total_size(sizeof(struct tcf_t)); /* TCA_GACT_TM */ + /* TCA_GACT_TM; actions dump their tcf_t with nla_put_64bit(), + * which may emit an extra NLA_PAD attribute. + */ + + nla_total_size_64bit(sizeof(struct tcf_t)); } static size_t tcf_action_full_attrs_size(size_t sz) diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index 09d46e195e33..06d8f78b7368 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -389,6 +389,31 @@ static void tcf_bpf_cleanup(struct tc_action *act) tcf_bpf_cfg_cleanup(&tmp); } +static size_t tcf_bpf_get_fill_size(const struct tc_action *act) +{ + struct tcf_bpf *prog = to_bpf(act); + size_t size = nla_total_size(sizeof(struct tc_act_bpf)); + + /* bpf_ops and bpf_num_ops are published as separate stores under + * tcf_lock, so take it here as tcf_bpf_dump() does. + */ + spin_lock_bh(&prog->tcf_lock); + if (tcf_bpf_is_ebpf(prog)) { + /* TCA_ACT_BPF_NAME */ + size += nla_total_size(ACT_BPF_NAME_LEN + 1); + size += nla_total_size(sizeof(u32)); /* TCA_ACT_BPF_ID */ + size += nla_total_size(BPF_TAG_SIZE); /* TCA_ACT_BPF_TAG */ + } else { + size += nla_total_size(sizeof(u16)); /* TCA_ACT_BPF_OPS_LEN */ + /* TCA_ACT_BPF_OPS */ + size += nla_total_size(prog->bpf_num_ops * + sizeof(struct sock_filter)); + } + spin_unlock_bh(&prog->tcf_lock); + + return size; +} + static struct tc_action_ops act_bpf_ops __read_mostly = { .kind = "bpf", .id = TCA_ID_BPF, @@ -397,6 +422,7 @@ static struct tc_action_ops act_bpf_ops __read_mostly = { .dump = tcf_bpf_dump, .cleanup = tcf_bpf_cleanup, .init = tcf_bpf_init, + .get_fill_size = tcf_bpf_get_fill_size, .size = sizeof(struct tcf_bpf), }; MODULE_ALIAS_NET_ACT("bpf"); diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index dc0b5724785d..9080cb386c16 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1657,6 +1657,51 @@ static int tcf_ct_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_ct_get_fill_size(const struct tc_action *act) +{ + const struct tcf_ct_params *p; + size_t size; + + size = nla_total_size(sizeof(struct tc_ct)) /* TCA_CT_PARMS */ + + nla_total_size(sizeof(u16)); /* TCA_CT_ACTION */ + + rcu_read_lock(); + p = rcu_dereference(to_ct(act)->params); + + if (p->ct_action & TCA_CT_ACT_CLEAR) + goto out; + + /* TCA_CT_MARK, TCA_CT_MARK_MASK */ + if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) + size += nla_total_size(sizeof(p->mark)) + + nla_total_size(sizeof(p->mark_mask)); + + /* TCA_CT_LABELS, TCA_CT_LABELS_MASK */ + if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) + size += nla_total_size(sizeof(p->labels)) + + nla_total_size(sizeof(p->labels_mask)); + + if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) + size += nla_total_size(sizeof(p->zone)); /* TCA_CT_ZONE */ + + if (p->ct_action & TCA_CT_ACT_NAT) + /* TCA_CT_NAT_IPV6_{MIN,MAX}, the larger of the two address + * variants, plus TCA_CT_NAT_PORT_{MIN,MAX}. + */ + size += 2 * nla_total_size(sizeof(struct in6_addr)) + + 2 * nla_total_size(sizeof(__be16)); + + /* TCA_CT_HELPER_{NAME,FAMILY,PROTO} */ + if (p->helper) + size += nla_total_size(NF_CT_HELPER_NAME_LEN) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); +out: + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_ct_ops = { .kind = "ct", .id = TCA_ID_CT, @@ -1666,6 +1711,7 @@ static struct tc_action_ops act_ct_ops = { .init = tcf_ct_init, .cleanup = tcf_ct_cleanup, .stats_update = tcf_stats_update, + .get_fill_size = tcf_ct_get_fill_size, .offload_act_setup = tcf_ct_offload_act_setup, .size = sizeof(struct tcf_ct), }; diff --git a/net/sched/act_ctinfo.c b/net/sched/act_ctinfo.c index 1886ffd2ca95..fced4b1094af 100644 --- a/net/sched/act_ctinfo.c +++ b/net/sched/act_ctinfo.c @@ -356,6 +356,16 @@ static void tcf_ctinfo_cleanup(struct tc_action *a) kfree_rcu(cp, rcu); } +static size_t tcf_ctinfo_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_ctinfo)) /* TCA_CTINFO_ACT */ + + nla_total_size(sizeof(u16)) /* TCA_CTINFO_ZONE */ + /* TCA_CTINFO_PARMS_{DSCP_MASK,DSCP_STATEMASK,CPMARK_MASK} */ + + 3 * nla_total_size(sizeof(u32)) + /* TCA_CTINFO_STATS_{DSCP_SET,DSCP_ERROR,CPMARK_SET} */ + + 3 * nla_total_size_64bit(sizeof(u64)); +} + static struct tc_action_ops act_ctinfo_ops = { .kind = "ctinfo", .id = TCA_ID_CTINFO, @@ -364,6 +374,7 @@ static struct tc_action_ops act_ctinfo_ops = { .dump = tcf_ctinfo_dump, .init = tcf_ctinfo_init, .cleanup= tcf_ctinfo_cleanup, + .get_fill_size = tcf_ctinfo_get_fill_size, .size = sizeof(struct tcf_ctinfo), }; MODULE_ALIAS_NET_ACT("ctinfo"); diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 065228026c58..9cea71fc1db3 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -28,6 +28,7 @@ #include <uapi/linux/tc_act/tc_ife.h> #include <net/tc_act/tc_ife.h> #include <linux/etherdevice.h> +#include <linux/if_arp.h> #include <net/ife.h> #include <net/tc_wrapper.h> @@ -723,7 +724,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, tcf_lastuse_update(&ife->tcf_tm); if (skb_at_tc_ingress(skb)) - skb_push(skb, skb->dev->hard_header_len); + skb_push(skb, ETH_HLEN); tlv_data = ife_decode(skb, &metalen); if (unlikely(!tlv_data)) { @@ -795,7 +796,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, where ORIGDATA = original ethernet header ... */ u16 metalen = ife_get_sz(skb, p); - int hdrm = metalen + skb->dev->hard_header_len + IFE_METAHDRLEN; + int hdrm = metalen + ETH_HLEN + IFE_METAHDRLEN; unsigned int skboff = 0; int new_len = skb->len + hdrm; bool exceed_mtu = false; @@ -826,7 +827,7 @@ drop: } if (skb_at_tc_ingress(skb)) - skb_push(skb, skb->dev->hard_header_len); + skb_push(skb, ETH_HLEN); ife_meta = ife_encode(skb, metalen); if (!ife_meta) @@ -856,11 +857,27 @@ drop: oethh->h_proto = htons(p->eth_type); if (skb_at_tc_ingress(skb)) - skb_pull(skb, skb->dev->hard_header_len); + skb_pull(skb, ETH_HLEN); return action; } +/* IFE encapsulates the original Ethernet header and, on decode, expects to + * find one, so it can only ever work on skbs that carry one. Loopback carries + * Ethernet header as well, so it qualifies here. + * At ingress, also verify that the L2 header about to be pushed back really + * is an Ethernet header because the skb could've been redirected with mirred + * from a non-Ethernet device. + */ +static bool tcf_ife_is_eth_skb(const struct sk_buff *skb) +{ + if (skb->dev->type != ARPHRD_ETHER && + skb->dev->type != ARPHRD_LOOPBACK) + return false; + + return !skb_at_tc_ingress(skb) || skb->mac_len == ETH_HLEN; +} + TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) @@ -869,6 +886,13 @@ TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, struct tcf_ife_params *p; int ret; + if (unlikely(!tcf_ife_is_eth_skb(skb))) { + bstats_update(this_cpu_ptr(ife->common.cpu_bstats), skb); + tcf_lastuse_update(&ife->tcf_tm); + qstats_cpu_drop_inc(ife->common.cpu_qstats); + return TC_ACT_SHOT; + } + p = rcu_dereference_bh(ife->params); if (p->flags & IFE_ENCODE) { ret = tcf_ife_encode(skb, a, res, p); @@ -878,6 +902,28 @@ TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, return tcf_ife_decode(skb, a, res); } +static size_t tcf_ife_get_fill_size(const struct tc_action *act) +{ + struct tcf_ife_info *ife = to_ife(act); + const struct tcf_ife_params *p; + struct tcf_meta_info *e; + size_t size = nla_total_size(sizeof(struct tc_ife)) /* TCA_IFE_PARMS */ + + nla_total_size(ETH_ALEN) /* TCA_IFE_DMAC */ + + nla_total_size(ETH_ALEN) /* TCA_IFE_SMAC */ + + nla_total_size(2) /* TCA_IFE_TYPE */ + + nla_total_size(0); /* TCA_IFE_METALST */ + + rcu_read_lock(); + p = rcu_dereference(ife->params); + if (p) { + list_for_each_entry_rcu(e, &p->metalist, metalist) + size += nla_total_size(sizeof(u32)); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_ife_ops = { .kind = "ife", .id = TCA_ID_IFE, @@ -886,6 +932,7 @@ static struct tc_action_ops act_ife_ops = { .dump = tcf_ife_dump, .cleanup = tcf_ife_cleanup, .init = tcf_ife_init, + .get_fill_size = tcf_ife_get_fill_size, .size = sizeof(struct tcf_ife_info), }; MODULE_ALIAS_NET_ACT("ife"); diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index d4d47a9921f4..99d7e36510bd 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -626,6 +626,29 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_pedit_get_fill_size(const struct tc_action *act) +{ + const struct tcf_pedit_parms *parms; + size_t size; + + rcu_read_lock(); + parms = rcu_dereference(to_pedit(act)->parms); + size = nla_total_size(struct_size_t(struct tc_pedit, keys, + parms->tcfp_nkeys)); + if (parms->tcfp_keys_ex) { + /* TCA_PEDIT_KEYS_EX, holding one TCA_PEDIT_KEY_EX nest with a + * HTYPE and a CMD attribute per key. + */ + size += nla_total_size(0) + + parms->tcfp_nkeys * (nla_total_size(0) + + nla_total_size(sizeof(u16)) + + nla_total_size(sizeof(u16))); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_pedit_ops = { .kind = "pedit", .id = TCA_ID_PEDIT, @@ -635,6 +658,7 @@ static struct tc_action_ops act_pedit_ops = { .dump = tcf_pedit_dump, .cleanup = tcf_pedit_cleanup, .init = tcf_pedit_init, + .get_fill_size = tcf_pedit_get_fill_size, .offload_act_setup = tcf_pedit_offload_act_setup, .size = sizeof(struct tcf_pedit), }; diff --git a/net/sched/act_police.c b/net/sched/act_police.c index ce08f6840ef7..3f8147f37549 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -490,6 +490,17 @@ static int tcf_police_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_police_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_police)) /* TCA_POLICE_TBF */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_RATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PEAKRATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PKTRATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PKTBURST64 */ + + nla_total_size(sizeof(u32)) /* TCA_POLICE_RESULT */ + + nla_total_size(sizeof(u32)); /* TCA_POLICE_AVRATE */ +} + MODULE_AUTHOR("Alexey Kuznetsov"); MODULE_DESCRIPTION("Policing actions"); MODULE_LICENSE("GPL"); @@ -503,6 +514,7 @@ static struct tc_action_ops act_police_ops = { .dump = tcf_police_dump, .init = tcf_police_init, .cleanup = tcf_police_cleanup, + .get_fill_size = tcf_police_get_fill_size, .offload_act_setup = tcf_police_offload_act_setup, .size = sizeof(struct tcf_police), }; diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c index 2ceb4d141b71..44319a159b55 100644 --- a/net/sched/act_sample.c +++ b/net/sched/act_sample.c @@ -315,6 +315,14 @@ static int tcf_sample_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_sample_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_sample)) /* TCA_SAMPLE_PARMS */ + + nla_total_size(sizeof(u32)) /* TCA_SAMPLE_RATE */ + + nla_total_size(sizeof(u32)) /* TCA_SAMPLE_TRUNC_SIZE */ + + nla_total_size(sizeof(u32)); /* TCA_SAMPLE_PSAMPLE_GROUP */ +} + static struct tc_action_ops act_sample_ops = { .kind = "sample", .id = TCA_ID_SAMPLE, @@ -324,6 +332,7 @@ static struct tc_action_ops act_sample_ops = { .dump = tcf_sample_dump, .init = tcf_sample_init, .cleanup = tcf_sample_cleanup, + .get_fill_size = tcf_sample_get_fill_size, .get_psample_group = tcf_sample_get_group, .offload_act_setup = tcf_sample_offload_act_setup, .size = sizeof(struct tcf_sample), diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c index a464b0a3c1b8..7579cf1e0ff3 100644 --- a/net/sched/act_skbmod.c +++ b/net/sched/act_skbmod.c @@ -38,7 +38,6 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, if (unlikely(p->action == TC_ACT_SHOT)) goto drop; - max_edit_len = skb_mac_header_len(skb); flags = p->flags; /* tcf_skbmod_init() guarantees "flags" to be one of the following: @@ -51,14 +50,19 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, if (flags == SKBMOD_F_ECN) { switch (skb_protocol(skb, true)) { case cpu_to_be16(ETH_P_IP): + max_edit_len = sizeof(struct iphdr); + break; case cpu_to_be16(ETH_P_IPV6): - max_edit_len += skb_network_header_len(skb); + max_edit_len = sizeof(struct ipv6hdr); break; default: goto out; } - } else if (!skb->dev || skb->dev->type != ARPHRD_ETHER) { - goto out; + max_edit_len += skb_network_offset(skb); + } else { + if (!skb->dev || skb->dev->type != ARPHRD_ETHER) + goto out; + max_edit_len = ETH_HLEN; } err = skb_ensure_writable(skb, max_edit_len); diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index b14807761d82..ff401ace4f3d 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -835,6 +835,85 @@ static int tcf_tunnel_key_offload_act_setup(struct tc_action *act, return 0; } +static size_t +tunnel_key_geneve_opts_fill_size(const struct ip_tunnel_info *info) +{ + const u8 *src = ip_tunnel_info_opts(info); + int len = info->options_len; + size_t size = 0; + + while (len > 0) { + const struct geneve_opt *opt = (const struct geneve_opt *)src; + + /* TCA_TUNNEL_KEY_ENC_OPT_GENEVE_{CLASS,TYPE,DATA} */ + size += nla_total_size(2) + + nla_total_size(1) + + nla_total_size(opt->length * 4); + + len -= sizeof(struct geneve_opt) + opt->length * 4; + src += sizeof(struct geneve_opt) + opt->length * 4; + } + + return size; +} + +static size_t tunnel_key_opts_fill_size(const struct ip_tunnel_info *info) +{ + size_t size; + + if (!info->options_len) + return 0; + + /* TCA_TUNNEL_KEY_ENC_OPTS and the per-protocol nest inside it */ + size = nla_total_size(0) + nla_total_size(0); + + if (test_bit(IP_TUNNEL_GENEVE_OPT_BIT, info->key.tun_flags)) { + size += tunnel_key_geneve_opts_fill_size(info); + } else if (test_bit(IP_TUNNEL_VXLAN_OPT_BIT, info->key.tun_flags)) { + /* TCA_TUNNEL_KEY_ENC_OPT_VXLAN_GBP */ + size += nla_total_size(sizeof(u32)); + } else if (test_bit(IP_TUNNEL_ERSPAN_OPT_BIT, info->key.tun_flags)) { + /* TCA_TUNNEL_KEY_ENC_OPT_ERSPAN_{VER,INDEX,DIR,HWID} */ + size += nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(__be32)) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); + } + + return size; +} + +static size_t tunnel_key_get_fill_size(const struct tc_action *act) +{ + struct tcf_tunnel_key *t = to_tunnel_key(act); + const struct tcf_tunnel_key_params *params; + /* TCA_TUNNEL_KEY_PARMS */ + size_t size = nla_total_size(sizeof(struct tc_tunnel_key)); + + rcu_read_lock(); + params = rcu_dereference(t->params); + if (params->tcft_action == TCA_TUNNEL_KEY_ACT_SET) { + const struct ip_tunnel_info *info = + ¶ms->tcft_enc_metadata->u.tun_info; + + /* In dump order: TCA_TUNNEL_KEY_ENC_KEY_ID, the IPv6 address + * pair (larger than the IPv4 one), ..._ENC_DST_PORT, + * ..._NO_CSUM, ..._NO_FRAG, the options and ..._ENC_{TOS,TTL}. + */ + size += nla_total_size(sizeof(__be32)) + + 2 * nla_total_size(sizeof(struct in6_addr)) + + nla_total_size(sizeof(__be16)) + + nla_total_size(sizeof(u8)) + + nla_total_size(0) + + tunnel_key_opts_fill_size(info) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_tunnel_key_ops = { .kind = "tunnel_key", .id = TCA_ID_TUNNEL_KEY, @@ -843,6 +922,7 @@ static struct tc_action_ops act_tunnel_key_ops = { .dump = tunnel_key_dump, .init = tunnel_key_init, .cleanup = tunnel_key_release, + .get_fill_size = tunnel_key_get_fill_size, .offload_act_setup = tcf_tunnel_key_offload_act_setup, .size = sizeof(struct tcf_tunnel_key), }; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 3271963c945d..9966766661d5 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -3372,7 +3372,8 @@ int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action, * This reference might be taken later from tcf_exts_get_net(). */ exts->net = net; - exts->actions = kzalloc_objs(struct tc_action *, TCA_ACT_MAX_PRIO); + exts->actions = kzalloc_objs(struct tc_action *, TCA_ACT_MAX_PRIO, + GFP_KERNEL_ACCOUNT); if (!exts->actions) return -ENOMEM; #endif diff --git a/net/sched/cls_basic.c b/net/sched/cls_basic.c index 492cd9ce8d46..e2a94ba9fba7 100644 --- a/net/sched/cls_basic.c +++ b/net/sched/cls_basic.c @@ -193,7 +193,7 @@ static int basic_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; @@ -212,9 +212,11 @@ static int basic_change(struct net *net, struct sk_buff *in_skb, if (err) goto errout; fnew->handle = handle; - fnew->pf = alloc_percpu(struct tc_basic_pcnt); + fnew->pf = alloc_percpu_gfp(struct tc_basic_pcnt, GFP_KERNEL_ACCOUNT); if (!fnew->pf) { err = -ENOMEM; + if (!fold) + idr_remove(&head->handle_idr, fnew->handle); goto errout; } diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index 6d19155becc8..188cf0f949dd 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -352,7 +352,7 @@ static int cls_bpf_prog_from_ops(struct nlattr **tb, struct cls_bpf_prog *prog) if (bpf_size != nla_len(tb[TCA_BPF_OPS])) return -EINVAL; - bpf_ops = kmemdup(nla_data(tb[TCA_BPF_OPS]), bpf_size, GFP_KERNEL); + bpf_ops = kmemdup(nla_data(tb[TCA_BPF_OPS]), bpf_size, GFP_KERNEL_ACCOUNT); if (bpf_ops == NULL) return -ENOMEM; @@ -403,7 +403,7 @@ static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog, } if (tb[TCA_BPF_NAME]) { - name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL); + name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL_ACCOUNT); if (!name) { bpf_prog_put(fp); return -ENOMEM; @@ -443,7 +443,7 @@ static int cls_bpf_change(struct net *net, struct sk_buff *in_skb, if (ret < 0) return ret; - prog = kzalloc_obj(*prog); + prog = kzalloc_obj(*prog, GFP_KERNEL_ACCOUNT); if (!prog) return -ENOBUFS; diff --git a/net/sched/cls_cgroup.c b/net/sched/cls_cgroup.c index 680a5c308094..210fd9fd26d8 100644 --- a/net/sched/cls_cgroup.c +++ b/net/sched/cls_cgroup.c @@ -95,7 +95,7 @@ static int cls_cgroup_change(struct net *net, struct sk_buff *in_skb, if (head && handle != head->handle) return -ENOENT; - new = kzalloc_obj(*head); + new = kzalloc_obj(*head, GFP_KERNEL_ACCOUNT); if (!new) return -ENOBUFS; diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index 356c68ebc389..a9ac3acf6eda 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -438,7 +438,7 @@ static int flow_change(struct net *net, struct sk_buff *in_skb, return -EOPNOTSUPP; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index 88f8a32fab2b..0e275b58151c 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -2233,7 +2233,7 @@ static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head, struct fl_flow_mask *newmask; int err; - newmask = kzalloc_obj(*newmask); + newmask = kzalloc_obj(*newmask, GFP_KERNEL_ACCOUNT); if (!newmask) return ERR_PTR(-ENOMEM); @@ -2394,7 +2394,7 @@ static int fl_change(struct net *net, struct sk_buff *in_skb, goto errout_tb; } - fnew = kzalloc_obj(*fnew); + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); if (!fnew) { err = -ENOBUFS; goto errout_tb; diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c index 646a730dca93..a462b262719c 100644 --- a/net/sched/cls_fw.c +++ b/net/sched/cls_fw.c @@ -276,7 +276,7 @@ static int fw_change(struct net *net, struct sk_buff *in_skb, if (f->id != handle && handle) return -EINVAL; - fnew = kzalloc_obj(struct fw_filter); + fnew = kzalloc_obj(struct fw_filter, GFP_KERNEL_ACCOUNT); if (!fnew) return -ENOBUFS; @@ -330,7 +330,7 @@ static int fw_change(struct net *net, struct sk_buff *in_skb, rcu_assign_pointer(tp->root, head); } - f = kzalloc_obj(struct fw_filter); + f = kzalloc_obj(struct fw_filter, GFP_KERNEL_ACCOUNT); if (f == NULL) return -ENOBUFS; diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c index 6f126872c14a..c14899b935bf 100644 --- a/net/sched/cls_matchall.c +++ b/net/sched/cls_matchall.c @@ -189,7 +189,7 @@ static int mall_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; } - new = kzalloc_obj(*new); + new = kzalloc_obj(*new, GFP_KERNEL_ACCOUNT); if (!new) return -ENOBUFS; @@ -201,7 +201,7 @@ static int mall_change(struct net *net, struct sk_buff *in_skb, handle = 1; new->handle = handle; new->flags = userflags; - new->pf = alloc_percpu(struct tc_matchall_pcnt); + new->pf = alloc_percpu_gfp(struct tc_matchall_pcnt, GFP_KERNEL_ACCOUNT); if (!new->pf) { err = -ENOMEM; goto err_alloc_percpu; diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c index eded7aacd3f7..0d1324c90583 100644 --- a/net/sched/cls_route.c +++ b/net/sched/cls_route.c @@ -455,7 +455,7 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, h1 = to_hash(nhandle); b = rtnl_dereference(head->table[h1]); if (!b) { - b = kzalloc_obj(struct route4_bucket); + b = kzalloc_obj(struct route4_bucket, GFP_KERNEL_ACCOUNT); if (b == NULL) return -ENOBUFS; @@ -524,7 +524,7 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, return -EINVAL; err = -ENOBUFS; - f = kzalloc_obj(struct route4_filter); + f = kzalloc_obj(struct route4_filter, GFP_KERNEL_ACCOUNT); if (!f) goto errout; diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index c297d7dbcf91..ac6d0fa5a40e 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -825,7 +825,7 @@ static struct tc_u_knode *u32_init_knode(struct net *net, struct tcf_proto *tp, struct tc_u32_sel *s = &n->sel; struct tc_u_knode *new; - new = kzalloc_flex(*new, sel.keys, s->nkeys); + new = kzalloc_flex(*new, sel.keys, s->nkeys, GFP_KERNEL_ACCOUNT); if (!new) return NULL; @@ -1114,15 +1114,16 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, goto erridr; } - n = kzalloc_flex(*n, sel.keys, s->nkeys); + n = kzalloc_flex(*n, sel.keys, s->nkeys, GFP_KERNEL_ACCOUNT); if (n == NULL) { err = -ENOBUFS; goto erridr; } #ifdef CONFIG_CLS_U32_PERF - n->pf = __alloc_percpu(struct_size(n->pf, kcnts, s->nkeys), - __alignof__(struct tc_u32_pcnt)); + n->pf = __alloc_percpu_gfp(struct_size(n->pf, kcnts, s->nkeys), + __alignof__(struct tc_u32_pcnt), + GFP_KERNEL_ACCOUNT); if (!n->pf) { err = -ENOBUFS; goto errfree; @@ -1144,7 +1145,7 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, goto errout; #ifdef CONFIG_CLS_U32_MARK - n->pcpu_success = alloc_percpu(u32); + n->pcpu_success = alloc_percpu_gfp(u32, GFP_KERNEL_ACCOUNT); if (!n->pcpu_success) { err = -ENOMEM; goto errout; diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 65b35528d125..90503e59e6e3 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -610,8 +610,11 @@ void __qdisc_calculate_pkt_len(struct sk_buff *skb, pkt_len <<= stab->szopts.size_log; out: - if (unlikely(pkt_len < 1)) - pkt_len = 1; + /* A size table can inflate qdisc_pkt_len() beyond any real packet + * (via overhead, the data table, or size_log); cap it so deficit + * schedulers such as DRR/ETS terminate their refill loops. + */ + pkt_len = clamp_t(int, pkt_len, 1, QDISC_PKT_LEN_MAX); qdisc_skb_cb(skb)->pkt_len = pkt_len; } diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index f25f60978631..dc93267029e7 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1907,6 +1907,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, ktime_add_ms(q->last_reconfig_time, 250))) { q->config->rate_bps = (q->avg_peak_bandwidth * 15) >> 4; + q->last_reconfig_time = now; cake_reconfigure(sch); } } diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index cacf5244958e..6aa5829d6961 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -205,7 +205,7 @@ static int codel_init(struct Qdisc *sch, struct nlattr *opt, codel_params_init(&q->params); codel_vars_init(&q->vars); codel_stats_init(&q->stats); - q->params.mtu = psched_mtu(qdisc_dev(sch)); + q->params.mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, 1 << 20); if (opt) { int err = codel_change(sch, opt, extack); diff --git a/net/sched/sch_fifo.c b/net/sched/sch_fifo.c index e6bfd39ff339..1b6388d50967 100644 --- a/net/sched/sch_fifo.c +++ b/net/sched/sch_fifo.c @@ -19,7 +19,7 @@ static int bfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) { - if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= + if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <= READ_ONCE(sch->limit))) return qdisc_enqueue_tail(skb, sch); diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index 7cae082a9847..6144b5686f13 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -828,8 +828,12 @@ begin: * f->time_next_packet was set when prior packet was sent, * and current time (@now) can be too late by tens of us. */ - if (f->time_next_packet) - len -= min(len/2, now - f->time_next_packet); + if (f->time_next_packet) { + s64 drift = now - f->time_next_packet; + + if (drift > 0) + len -= min_t(u64, len / 2, drift); + } f->time_next_packet = now + len; } out: @@ -1222,12 +1226,14 @@ static int fq_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct fq_sched_data *q = qdisc_priv(sch); + u32 mtu; int i, err; sch->limit = 10000; q->flow_plimit = 100; - q->quantum = 2 * psched_mtu(qdisc_dev(sch)); - q->initial_quantum = 10 * psched_mtu(qdisc_dev(sch)); + mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); + q->quantum = min_t(u32, 2 * mtu, 1 << 20); + q->initial_quantum = min_t(u32, 10 * mtu, 1 << 20); q->flow_refill_delay = msecs_to_jiffies(40); q->flow_max_rate = ~0UL; q->time_next_delayed_flow = ~0ULL; diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 6cce86ba383c..969b2510b0b8 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -509,6 +509,7 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct fq_codel_sched_data *q = qdisc_priv(sch); + u32 mtu; int i; int err; @@ -516,13 +517,14 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, q->flows_cnt = 1024; q->memory_limit = 32 << 20; /* 32 MBytes */ q->drop_batch_size = 64; - q->quantum = psched_mtu(qdisc_dev(sch)); + mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, FQ_CODEL_QUANTUM_MAX); + q->quantum = mtu; INIT_LIST_HEAD(&q->new_flows); INIT_LIST_HEAD(&q->old_flows); codel_params_init(&q->cparams); codel_stats_init(&q->cstats); q->cparams.ecn = true; - q->cparams.mtu = psched_mtu(qdisc_dev(sch)); + q->cparams.mtu = mtu; if (opt) { err = fq_codel_change(sch, opt, extack); diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 069e1facd413..b27d95418707 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -427,7 +427,8 @@ static int fq_pie_init(struct Qdisc *sch, struct nlattr *opt, pie_params_init(&q->p_params); sch->limit = 10 * 1024; q->p_params.limit = sch->limit; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); q->sch = sch; q->ecn_prob = 10; q->flows_cnt = 1024; diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index ef2b4bf51564..4539dc2c6d38 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -1278,7 +1278,7 @@ static void transition_one_qdisc(struct net_device *dev, rcu_assign_pointer(dev_queue->qdisc, new_qdisc); if (need_watchdog_p) { - WRITE_ONCE(dev_queue->trans_start, 0); + WRITE_ONCE(dev_queue->trans_start, jiffies); *need_watchdog_p = 1; } } diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c index fcc1a4c03636..f04f425c6c44 100644 --- a/net/sched/sch_gred.c +++ b/net/sched/sch_gred.c @@ -179,7 +179,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch, * if no default DP has been configured. This * allows for DP flows to be left untouched. */ - if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= + if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <= sch->limit)) return qdisc_enqueue_tail(skb, sch); else @@ -244,7 +244,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch, break; } - if (gred_backlog(t, q, sch) + qdisc_pkt_len(skb) <= q->limit) { + if ((u64)gred_backlog(t, q, sch) + qdisc_pkt_len(skb) <= q->limit) { q->backlog += qdisc_pkt_len(skb); return qdisc_enqueue_tail(skb, sch); } diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index d85cb0263b67..96acab6a8da0 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -624,6 +624,10 @@ static int hhf_init(struct Qdisc *sch, struct nlattr *opt, q->hhf_evict_timeout = HZ; /* 1 sec */ q->hhf_non_hh_weight = 2; + if ((int)q->quantum <= 0 || + (u64)q->quantum * q->hhf_non_hh_weight > INT_MAX) + q->quantum = 256; + if (opt) { int err = hhf_change(sch, opt, extack); diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index fdac0dc8f35a..1ba67b121de4 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -224,6 +224,7 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, struct htb_class *cl; struct tcf_result res; struct tcf_proto *tcf; + unsigned int hops = 0; int result; /* allow to select class by setting skb->priority to valid classid; @@ -266,6 +267,10 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, if (!cl->level) return cl; /* we hit leaf; return it */ + if (++hops > TC_HTB_MAXDEPTH) { + pr_warn_ratelimited("htb: classify loop detected, dropping packet\n"); + return NULL; + } /* we have got inner class; apply inner filter chain */ tcf = rcu_dereference_bh(cl->filter_list); } @@ -633,13 +638,11 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } else { return qdisc_drop(skb, sch, to_free); } -#ifdef CONFIG_NET_CLS_ACT } else if (!cl) { if (ret & __NET_XMIT_BYPASS) qdisc_qstats_drop(sch); __qdisc_drop(skb, to_free); return ret; -#endif } else if ((ret = qdisc_enqueue(skb, cl->leaf.q, to_free)) != NET_XMIT_SUCCESS) { if (net_xmit_drop_count(ret)) { diff --git a/net/sched/sch_plug.c b/net/sched/sch_plug.c index cefb65201e17..b60ddfee6a68 100644 --- a/net/sched/sch_plug.c +++ b/net/sched/sch_plug.c @@ -89,7 +89,7 @@ static int plug_enqueue(struct sk_buff *skb, struct Qdisc *sch, { struct plug_sched_data *q = qdisc_priv(sch); - if (likely(sch->qstats.backlog + skb->len <= q->limit)) { + if (likely((u64)sch->qstats.backlog + skb->len <= q->limit)) { if (!q->unplug_indefinite) q->pkts_current_epoch++; return qdisc_enqueue_tail(skb, sch); diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index 77675f9a4c46..187d3ed578f2 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -799,7 +799,8 @@ static int sfq_init(struct Qdisc *sch, struct nlattr *opt, q->tail = NULL; q->divisor = SFQ_DEFAULT_HASH_DIVISOR; q->maxflows = SFQ_DEFAULT_FLOWS; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); q->perturb_period = 0; get_random_bytes(&q->perturbation, sizeof(q->perturbation)); diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 5c42a29a981c..9e52afc2d980 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -377,6 +377,7 @@ restart: nores = 1; break; } + skb->dev = dev; __skb_pull(skb, skb_network_offset(skb)); } while ((q = rcu_dereference(NEXT_SLAVE(q))) != start); |