Merge branch 'for-5.14' of git://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu

[linux-2.6-microblaze.git] / include / linux / filter.h

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 9a09547..472f970 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -646,6 +646,7 @@ struct bpf_redirect_info {
        u32 flags;
        u32 tgt_index;
        void *tgt_value;
+       struct bpf_map *map;
        u32 map_id;
        enum bpf_map_type map_type;
        u32 kern_flags;
@@ -762,11 +763,9 @@ DECLARE_BPF_DISPATCHER(xdp)
 static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
                                            struct xdp_buff *xdp)
 {
-       /* Caller needs to hold rcu_read_lock() (!), otherwise program
-        * can be released while still running, or map elements could be
-        * freed early while still having concurrent users. XDP fastpath
-        * already takes rcu_read_lock() when fetching the program, so
-        * it's not necessary here anymore.
+       /* Driver XDP hooks are invoked within a single NAPI poll cycle and thus
+        * under local_bh_disable(), which provides the needed RCU protection
+        * for accessing map entries.
         */
        return __BPF_PROG_RUN(prog, xdp, BPF_DISPATCHER_FUNC(xdp));
 }
@@ -995,11 +994,13 @@ void bpf_warn_invalid_xdp_action(u32 act);
 #ifdef CONFIG_INET
 struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
                                  struct bpf_prog *prog, struct sk_buff *skb,
+                                 struct sock *migrating_sk,
                                  u32 hash);
 #else
 static inline struct sock *
 bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
                     struct bpf_prog *prog, struct sk_buff *skb,
+                    struct sock *migrating_sk,
                     u32 hash)
 {
        return NULL;
@@ -1464,17 +1465,19 @@ static inline bool bpf_sk_lookup_run_v6(struct net *net, int protocol,
 }
 #endif /* IS_ENABLED(CONFIG_IPV6) */
 
-static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u32 ifindex, u64 flags,
+static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u32 ifindex,
+                                                 u64 flags, const u64 flag_mask,
                                                  void *lookup_elem(struct bpf_map *map, u32 key))
 {
        struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
+       const u64 action_mask = XDP_ABORTED | XDP_DROP | XDP_PASS | XDP_TX;
 
        /* Lower bits of the flags are used as return code on lookup failure */
-       if (unlikely(flags > XDP_TX))
+       if (unlikely(flags & ~(action_mask | flag_mask)))
                return XDP_ABORTED;
 
        ri->tgt_value = lookup_elem(map, ifindex);
-       if (unlikely(!ri->tgt_value)) {
+       if (unlikely(!ri->tgt_value) && !(flags & BPF_F_BROADCAST)) {
                /* If the lookup fails we want to clear out the state in the
                 * redirect_info struct completely, so that if an eBPF program
                 * performs multiple lookups, the last one always takes
@@ -1482,13 +1485,21 @@ static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u32 ifind
                 */
                ri->map_id = INT_MAX; /* Valid map id idr range: [1,INT_MAX[ */
                ri->map_type = BPF_MAP_TYPE_UNSPEC;
-               return flags;
+               return flags & action_mask;
        }
 
        ri->tgt_index = ifindex;
        ri->map_id = map->id;
        ri->map_type = map->map_type;
 
+       if (flags & BPF_F_BROADCAST) {
+               WRITE_ONCE(ri->map, map);
+               ri->flags = flags;
+       } else {
+               WRITE_ONCE(ri->map, NULL);
+               ri->flags = 0;
+       }
+
        return XDP_REDIRECT;
 }