net/sched: avoid indirect classify functions on retpoline kernels

Expose the necessary tc classifier functions and wire up cls_api to use
direct calls in retpoline kernels.

Signed-off-by: Pedro Tammela <pctammela@mojatatu.com>
Reviewed-by: Jamal Hadi Salim <jhs@mojatatu.com>
Reviewed-by: Victor Nogueira <victor@mojatatu.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
index 23d1cfa..668130f 100644 (file)
--- a/net/sched/cls_api.c
+++ b/net/sched/cls_api.c
@@ -40,6 +40,7 @@
 #include <net/tc_act/tc_mpls.h>
 #include <net/tc_act/tc_gate.h>
 #include <net/flow_offload.h>
+#include <net/tc_wrapper.h>
 
 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
 
@@ -1564,7 +1565,7 @@ reclassify:
                    tp->protocol != htons(ETH_P_ALL))
                        continue;
 
-               err = tp->classify(skb, tp, res);
+               err = tc_classify(skb, tp, res);
 #ifdef CONFIG_NET_CLS_ACT
                if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
                        first_tp = orig_tp;

diff --git a/net/sched/cls_basic.c b/net/sched/cls_basic.c
index d229ce9..1b92c33 100644 (file)
--- a/net/sched/cls_basic.c
+++ b/net/sched/cls_basic.c
@@ -18,6 +18,7 @@
 #include <net/netlink.h>
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
+#include <net/tc_wrapper.h>
 
 struct basic_head {
        struct list_head        flist;
@@ -36,8 +37,9 @@ struct basic_filter {
        struct rcu_work         rwork;
 };
 
-static int basic_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                         struct tcf_result *res)
+TC_INDIRECT_SCOPE int basic_classify(struct sk_buff *skb,
+                                    const struct tcf_proto *tp,
+                                    struct tcf_result *res)
 {
        int r;
        struct basic_head *head = rcu_dereference_bh(tp->root);

diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c
index bc317b3..466c26d 100644 (file)
--- a/net/sched/cls_bpf.c
+++ b/net/sched/cls_bpf.c
@@ -19,6 +19,7 @@
 #include <net/rtnetlink.h>
 #include <net/pkt_cls.h>
 #include <net/sock.h>
+#include <net/tc_wrapper.h>
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
@@ -77,8 +78,9 @@ static int cls_bpf_exec_opcode(int code)
        }
 }
 
-static int cls_bpf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                           struct tcf_result *res)
+TC_INDIRECT_SCOPE int cls_bpf_classify(struct sk_buff *skb,
+                                      const struct tcf_proto *tp,
+                                      struct tcf_result *res)
 {
        struct cls_bpf_head *head = rcu_dereference_bh(tp->root);
        bool at_ingress = skb_at_tc_ingress(skb);

diff --git a/net/sched/cls_cgroup.c b/net/sched/cls_cgroup.c
index ed00001..bd9322d 100644 (file)
--- a/net/sched/cls_cgroup.c
+++ b/net/sched/cls_cgroup.c
@@ -13,6 +13,7 @@
 #include <net/pkt_cls.h>
 #include <net/sock.h>
 #include <net/cls_cgroup.h>
+#include <net/tc_wrapper.h>
 
 struct cls_cgroup_head {
        u32                     handle;
@@ -22,8 +23,9 @@ struct cls_cgroup_head {
        struct rcu_work         rwork;
 };
 
-static int cls_cgroup_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                              struct tcf_result *res)
+TC_INDIRECT_SCOPE int cls_cgroup_classify(struct sk_buff *skb,
+                                         const struct tcf_proto *tp,
+                                         struct tcf_result *res)
 {
        struct cls_cgroup_head *head = rcu_dereference_bh(tp->root);
        u32 classid = task_get_classid(skb);

diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c
index 014cd3d..535668e 100644 (file)
--- a/net/sched/cls_flow.c
+++ b/net/sched/cls_flow.c
@@ -24,6 +24,7 @@
 #include <net/ip.h>
 #include <net/route.h>
 #include <net/flow_dissector.h>
+#include <net/tc_wrapper.h>
 
 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
 #include <net/netfilter/nf_conntrack.h>
@@ -292,8 +293,9 @@ static u32 flow_key_get(struct sk_buff *skb, int key, struct flow_keys *flow)
                          (1 << FLOW_KEY_NFCT_PROTO_SRC) |      \
                          (1 << FLOW_KEY_NFCT_PROTO_DST))
 
-static int flow_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                        struct tcf_result *res)
+TC_INDIRECT_SCOPE int flow_classify(struct sk_buff *skb,
+                                   const struct tcf_proto *tp,
+                                   struct tcf_result *res)
 {
        struct flow_head *head = rcu_dereference_bh(tp->root);
        struct flow_filter *f;

diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c
index 25bc57e..0b15698 100644 (file)
--- a/net/sched/cls_flower.c
+++ b/net/sched/cls_flower.c
@@ -27,6 +27,7 @@
 #include <net/vxlan.h>
 #include <net/erspan.h>
 #include <net/gtp.h>
+#include <net/tc_wrapper.h>
 
 #include <net/dst.h>
 #include <net/dst_metadata.h>
@@ -305,8 +306,9 @@ static u16 fl_ct_info_to_flower_map[] = {
                                        TCA_FLOWER_KEY_CT_FLAGS_NEW,
 };
 
-static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                      struct tcf_result *res)
+TC_INDIRECT_SCOPE int fl_classify(struct sk_buff *skb,
+                                 const struct tcf_proto *tp,
+                                 struct tcf_result *res)
 {
        struct cls_fl_head *head = rcu_dereference_bh(tp->root);
        bool post_ct = tc_skb_cb(skb)->post_ct;

diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c
index a32351d..ae9439a 100644 (file)
--- a/net/sched/cls_fw.c
+++ b/net/sched/cls_fw.c
@@ -21,6 +21,7 @@
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
 #include <net/sch_generic.h>
+#include <net/tc_wrapper.h>
 
 #define HTSIZE 256
 
@@ -47,8 +48,9 @@ static u32 fw_hash(u32 handle)
        return handle % HTSIZE;
 }
 
-static int fw_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                      struct tcf_result *res)
+TC_INDIRECT_SCOPE int fw_classify(struct sk_buff *skb,
+                                 const struct tcf_proto *tp,
+                                 struct tcf_result *res)
 {
        struct fw_head *head = rcu_dereference_bh(tp->root);
        struct fw_filter *f;

diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c
index 39a5d9c..705f63d 100644 (file)
--- a/net/sched/cls_matchall.c
+++ b/net/sched/cls_matchall.c
@@ -12,6 +12,7 @@
 
 #include <net/sch_generic.h>
 #include <net/pkt_cls.h>
+#include <net/tc_wrapper.h>
 
 struct cls_mall_head {
        struct tcf_exts exts;
@@ -24,8 +25,9 @@ struct cls_mall_head {
        bool deleting;
 };
 
-static int mall_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                        struct tcf_result *res)
+TC_INDIRECT_SCOPE int mall_classify(struct sk_buff *skb,
+                                   const struct tcf_proto *tp,
+                                   struct tcf_result *res)
 {
        struct cls_mall_head *head = rcu_dereference_bh(tp->root);
 

diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c
index 9e43b92..d0c5372 100644 (file)
--- a/net/sched/cls_route.c
+++ b/net/sched/cls_route.c
@@ -17,6 +17,7 @@
 #include <net/netlink.h>
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
+#include <net/tc_wrapper.h>
 
 /*
  * 1. For now we assume that route tags < 256.
@@ -121,8 +122,9 @@ static inline int route4_hash_wild(void)
        return 0;                                               \
 }
 
-static int route4_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                          struct tcf_result *res)
+TC_INDIRECT_SCOPE int route4_classify(struct sk_buff *skb,
+                                     const struct tcf_proto *tp,
+                                     struct tcf_result *res)
 {
        struct route4_head *head = rcu_dereference_bh(tp->root);
        struct dst_entry *dst;

diff --git a/net/sched/cls_rsvp.c b/net/sched/cls_rsvp.c
index de1c1d4..03d8619 100644 (file)
--- a/net/sched/cls_rsvp.c
+++ b/net/sched/cls_rsvp.c
@@ -15,10 +15,12 @@
 #include <net/netlink.h>
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
+#include <net/tc_wrapper.h>
 
 #define RSVP_DST_LEN   1
 #define RSVP_ID                "rsvp"
 #define RSVP_OPS       cls_rsvp_ops
+#define RSVP_CLS       rsvp_classify
 
 #include "cls_rsvp.h"
 MODULE_LICENSE("GPL");

diff --git a/net/sched/cls_rsvp.h b/net/sched/cls_rsvp.h
index b00a7db..869efba 100644 (file)
--- a/net/sched/cls_rsvp.h
+++ b/net/sched/cls_rsvp.h
@@ -124,8 +124,8 @@ static inline unsigned int hash_src(__be32 *src)
                return r;                               \
 }
 
-static int rsvp_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                        struct tcf_result *res)
+TC_INDIRECT_SCOPE int RSVP_CLS(struct sk_buff *skb, const struct tcf_proto *tp,
+                              struct tcf_result *res)
 {
        struct rsvp_head *head = rcu_dereference_bh(tp->root);
        struct rsvp_session *s;
@@ -738,7 +738,7 @@ static void rsvp_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
 
 static struct tcf_proto_ops RSVP_OPS __read_mostly = {
        .kind           =       RSVP_ID,
-       .classify       =       rsvp_classify,
+       .classify       =       RSVP_CLS,
        .init           =       rsvp_init,
        .destroy        =       rsvp_destroy,
        .get            =       rsvp_get,

diff --git a/net/sched/cls_rsvp6.c b/net/sched/cls_rsvp6.c
index 6407884..e627cc3 100644 (file)
--- a/net/sched/cls_rsvp6.c
+++ b/net/sched/cls_rsvp6.c
@@ -15,10 +15,12 @@
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
 #include <net/netlink.h>
+#include <net/tc_wrapper.h>
 
 #define RSVP_DST_LEN   4
 #define RSVP_ID                "rsvp6"
 #define RSVP_OPS       cls_rsvp6_ops
+#define RSVP_CLS rsvp6_classify
 
 #include "cls_rsvp.h"
 MODULE_LICENSE("GPL");

diff --git a/net/sched/cls_tcindex.c b/net/sched/cls_tcindex.c
index 1c9eeb9..eb0e945 100644 (file)
--- a/net/sched/cls_tcindex.c
+++ b/net/sched/cls_tcindex.c
@@ -16,6 +16,7 @@
 #include <net/netlink.h>
 #include <net/pkt_cls.h>
 #include <net/sch_generic.h>
+#include <net/tc_wrapper.h>
 
 /*
  * Passing parameters to the root seems to be done more awkwardly than really
@@ -98,9 +99,9 @@ static struct tcindex_filter_result *tcindex_lookup(struct tcindex_data *p,
        return NULL;
 }
 
-
-static int tcindex_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                           struct tcf_result *res)
+TC_INDIRECT_SCOPE int tcindex_classify(struct sk_buff *skb,
+                                      const struct tcf_proto *tp,
+                                      struct tcf_result *res)
 {
        struct tcindex_data *p = rcu_dereference_bh(tp->root);
        struct tcindex_filter_result *f;

diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c
index 34d25f7..4e2e269 100644 (file)
--- a/net/sched/cls_u32.c
+++ b/net/sched/cls_u32.c
@@ -39,6 +39,7 @@
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
 #include <linux/idr.h>
+#include <net/tc_wrapper.h>
 
 struct tc_u_knode {
        struct tc_u_knode __rcu *next;
@@ -100,8 +101,9 @@ static inline unsigned int u32_hash_fold(__be32 key,
        return h;
 }
 
-static int u32_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                       struct tcf_result *res)
+TC_INDIRECT_SCOPE int u32_classify(struct sk_buff *skb,
+                                  const struct tcf_proto *tp,
+                                  struct tcf_result *res)
 {
        struct {
                struct tc_u_knode *knode;