sock_map: Introduce BPF_SK_SKB_VERDICT

Reusing BPF_SK_SKB_STREAM_VERDICT is possible but its name is
confusing and more importantly we still want to distinguish them
from user-space. So we can just reuse the stream verdict code but
introduce a new type of eBPF program, skb_verdict. Users are not
allowed to attach stream_verdict and skb_verdict programs to the
same map.

Signed-off-by: Cong Wang <cong.wang@bytedance.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: John Fastabend <john.fastabend@gmail.com>
Link: https://lore.kernel.org/bpf/20210331023237.41094-10-xiyou.wangcong@gmail.com

diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h
index e7aba15..c83dbc2 100644 (file)
--- a/include/linux/skmsg.h
+++ b/include/linux/skmsg.h
@@ -58,6 +58,7 @@ struct sk_psock_progs {
        struct bpf_prog                 *msg_parser;
        struct bpf_prog                 *stream_parser;
        struct bpf_prog                 *stream_verdict;
+       struct bpf_prog                 *skb_verdict;
 };
 
 enum sk_psock_state_bits {
@@ -487,6 +488,7 @@ static inline void psock_progs_drop(struct sk_psock_progs *progs)
        psock_set_prog(&progs->msg_parser, NULL);
        psock_set_prog(&progs->stream_parser, NULL);
        psock_set_prog(&progs->stream_verdict, NULL);
+       psock_set_prog(&progs->skb_verdict, NULL);
 }
 
 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb);

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 5987167..49371eb 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -957,6 +957,7 @@ enum bpf_attach_type {
        BPF_XDP_CPUMAP,
        BPF_SK_LOOKUP,
        BPF_XDP,
+       BPF_SK_SKB_VERDICT,
        __MAX_BPF_ATTACH_TYPE
 };
 

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 9603de8..6428634 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2948,6 +2948,7 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type)
                return BPF_PROG_TYPE_SK_MSG;
        case BPF_SK_SKB_STREAM_PARSER:
        case BPF_SK_SKB_STREAM_VERDICT:
+       case BPF_SK_SKB_VERDICT:
                return BPF_PROG_TYPE_SK_SKB;
        case BPF_LIRC_MODE2:
                return BPF_PROG_TYPE_LIRC_MODE2;

diff --git a/net/core/skmsg.c b/net/core/skmsg.c
index 656ecea..a045812 100644 (file)
--- a/net/core/skmsg.c
+++ b/net/core/skmsg.c
@@ -697,7 +697,7 @@ void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
        rcu_assign_sk_user_data(sk, NULL);
        if (psock->progs.stream_parser)
                sk_psock_stop_strp(sk, psock);
-       else if (psock->progs.stream_verdict)
+       else if (psock->progs.stream_verdict || psock->progs.skb_verdict)
                sk_psock_stop_verdict(sk, psock);
        write_unlock_bh(&sk->sk_callback_lock);
 
@@ -1024,6 +1024,8 @@ static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb,
        }
        skb_set_owner_r(skb, sk);
        prog = READ_ONCE(psock->progs.stream_verdict);
+       if (!prog)
+               prog = READ_ONCE(psock->progs.skb_verdict);
        if (likely(prog)) {
                skb_dst_drop(skb);
                skb_bpf_redirect_clear(skb);

diff --git a/net/core/sock_map.c b/net/core/sock_map.c
index 42d7972..c2a0411 100644 (file)
--- a/net/core/sock_map.c
+++ b/net/core/sock_map.c
@@ -156,6 +156,8 @@ static void sock_map_del_link(struct sock *sk,
                                strp_stop = true;
                        if (psock->saved_data_ready && stab->progs.stream_verdict)
                                verdict_stop = true;
+                       if (psock->saved_data_ready && stab->progs.skb_verdict)
+                               verdict_stop = true;
                        list_del(&link->list);
                        sk_psock_free_link(link);
                }
@@ -232,6 +234,7 @@ static int sock_map_link(struct bpf_map *map, struct sock *sk)
        struct sk_psock_progs *progs = sock_map_progs(map);
        struct bpf_prog *stream_verdict = NULL;
        struct bpf_prog *stream_parser = NULL;
+       struct bpf_prog *skb_verdict = NULL;
        struct bpf_prog *msg_parser = NULL;
        struct sk_psock *psock;
        int ret;
@@ -268,6 +271,15 @@ static int sock_map_link(struct bpf_map *map, struct sock *sk)
                }
        }
 
+       skb_verdict = READ_ONCE(progs->skb_verdict);
+       if (skb_verdict) {
+               skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
+               if (IS_ERR(skb_verdict)) {
+                       ret = PTR_ERR(skb_verdict);
+                       goto out_put_msg_parser;
+               }
+       }
+
 no_progs:
        psock = sock_map_psock_get_checked(sk);
        if (IS_ERR(psock)) {
@@ -278,6 +290,9 @@ no_progs:
        if (psock) {
                if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||
                    (stream_parser  && READ_ONCE(psock->progs.stream_parser)) ||
+                   (skb_verdict && READ_ONCE(psock->progs.skb_verdict)) ||
+                   (skb_verdict && READ_ONCE(psock->progs.stream_verdict)) ||
+                   (stream_verdict && READ_ONCE(psock->progs.skb_verdict)) ||
                    (stream_verdict && READ_ONCE(psock->progs.stream_verdict))) {
                        sk_psock_put(sk, psock);
                        ret = -EBUSY;
@@ -309,6 +324,9 @@ no_progs:
        } else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {
                psock_set_prog(&psock->progs.stream_verdict, stream_verdict);
                sk_psock_start_verdict(sk,psock);
+       } else if (!stream_verdict && skb_verdict && !psock->saved_data_ready) {
+               psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
+               sk_psock_start_verdict(sk, psock);
        }
        write_unlock_bh(&sk->sk_callback_lock);
        return 0;
@@ -317,6 +335,9 @@ out_unlock_drop:
 out_drop:
        sk_psock_put(sk, psock);
 out_progs:
+       if (skb_verdict)
+               bpf_prog_put(skb_verdict);
+out_put_msg_parser:
        if (msg_parser)
                bpf_prog_put(msg_parser);
 out_put_stream_parser:
@@ -1442,8 +1463,15 @@ static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,
                break;
 #endif
        case BPF_SK_SKB_STREAM_VERDICT:
+               if (progs->skb_verdict)
+                       return -EBUSY;
                pprog = &progs->stream_verdict;
                break;
+       case BPF_SK_SKB_VERDICT:
+               if (progs->stream_verdict)
+                       return -EBUSY;
+               pprog = &progs->skb_verdict;
+               break;
        default:
                return -EOPNOTSUPP;
        }

diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c
index 6530366..1828bba 100644 (file)
--- a/tools/bpf/bpftool/common.c
+++ b/tools/bpf/bpftool/common.c
@@ -57,6 +57,7 @@ const char * const attach_type_name[__MAX_BPF_ATTACH_TYPE] = {
 
        [BPF_SK_SKB_STREAM_PARSER]      = "sk_skb_stream_parser",
        [BPF_SK_SKB_STREAM_VERDICT]     = "sk_skb_stream_verdict",
+       [BPF_SK_SKB_VERDICT]            = "sk_skb_verdict",
        [BPF_SK_MSG_VERDICT]            = "sk_msg_verdict",
        [BPF_LIRC_MODE2]                = "lirc_mode2",
        [BPF_FLOW_DISSECTOR]            = "flow_dissector",

diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index f2b915b..3f067d2 100644 (file)
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -76,6 +76,7 @@ enum dump_mode {
 static const char * const attach_type_strings[] = {
        [BPF_SK_SKB_STREAM_PARSER] = "stream_parser",
        [BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",
+       [BPF_SK_SKB_VERDICT] = "skb_verdict",
        [BPF_SK_MSG_VERDICT] = "msg_verdict",
        [BPF_FLOW_DISSECTOR] = "flow_dissector",
        [__MAX_BPF_ATTACH_TYPE] = NULL,

diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index ab9f223..6990260 100644 (file)
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -957,6 +957,7 @@ enum bpf_attach_type {
        BPF_XDP_CPUMAP,
        BPF_SK_LOOKUP,
        BPF_XDP,
+       BPF_SK_SKB_VERDICT,
        __MAX_BPF_ATTACH_TYPE
 };