bpf: Allow helpers to accept pointers with a fixed size

Before this commit, the BPF verifier required ARG_PTR_TO_MEM arguments
to be followed by ARG_CONST_SIZE holding the size of the memory region.
The helpers had to check that size in runtime.

There are cases where the size expected by a helper is a compile-time
constant. Checking it in runtime is an unnecessary overhead and waste of
BPF registers.

This commit allows helpers to accept pointers to memory without the
corresponding ARG_CONST_SIZE, given that they define the memory region
size in struct bpf_func_proto and use ARG_PTR_TO_FIXED_SIZE_MEM type.

arg_size is unionized with arg_btf_id to reduce the kernel image size,
and it's valid because they are used by different argument types.

Signed-off-by: Maxim Mikityanskiy <maximmi@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Link: https://lore.kernel.org/r/20220615134847.3753567-3-maximmi@nvidia.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index f3e88af..a945319 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -401,6 +401,9 @@ enum bpf_type_flag {
        /* DYNPTR points to a ringbuf record. */
        DYNPTR_TYPE_RINGBUF     = BIT(9 + BPF_BASE_TYPE_BITS),
 
+       /* Size is known at compile time. */
+       MEM_FIXED_SIZE          = BIT(10 + BPF_BASE_TYPE_BITS),
+
        __BPF_TYPE_FLAG_MAX,
        __BPF_TYPE_LAST_FLAG    = __BPF_TYPE_FLAG_MAX - 1,
 };
@@ -464,6 +467,8 @@ enum bpf_arg_type {
         * all bytes or clear them in error case.
         */
        ARG_PTR_TO_UNINIT_MEM           = MEM_UNINIT | ARG_PTR_TO_MEM,
+       /* Pointer to valid memory of size known at compile time. */
+       ARG_PTR_TO_FIXED_SIZE_MEM       = MEM_FIXED_SIZE | ARG_PTR_TO_MEM,
 
        /* This must be the last entry. Its purpose is to ensure the enum is
         * wide enough to hold the higher bits reserved for bpf_type_flag.
@@ -529,6 +534,14 @@ struct bpf_func_proto {
                        u32 *arg5_btf_id;
                };
                u32 *arg_btf_id[5];
+               struct {
+                       size_t arg1_size;
+                       size_t arg2_size;
+                       size_t arg3_size;
+                       size_t arg4_size;
+                       size_t arg5_size;
+               };
+               size_t arg_size[5];
        };
        int *ret_btf_id; /* return value btf_id */
        bool (*allowed)(const struct bpf_prog *prog);

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index eadc23a..2859901 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5848,6 +5848,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        enum bpf_arg_type arg_type = fn->arg_type[arg];
        enum bpf_reg_type type = reg->type;
+       u32 *arg_btf_id = NULL;
        int err = 0;
 
        if (arg_type == ARG_DONTCARE)
@@ -5884,7 +5885,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
                 */
                goto skip_type_check;
 
-       err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg], meta);
+       /* arg_btf_id and arg_size are in a union. */
+       if (base_type(arg_type) == ARG_PTR_TO_BTF_ID)
+               arg_btf_id = fn->arg_btf_id[arg];
+
+       err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
        if (err)
                return err;
 
@@ -6011,6 +6016,11 @@ skip_type_check:
                 * next is_mem_size argument below.
                 */
                meta->raw_mode = arg_type & MEM_UNINIT;
+               if (arg_type & MEM_FIXED_SIZE) {
+                       err = check_helper_mem_access(env, regno,
+                                                     fn->arg_size[arg], false,
+                                                     meta);
+               }
        } else if (arg_type_is_mem_size(arg_type)) {
                bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
 
@@ -6400,11 +6410,19 @@ static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
        return count <= 1;
 }
 
-static bool check_args_pair_invalid(enum bpf_arg_type arg_curr,
-                                   enum bpf_arg_type arg_next)
+static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)
 {
-       return (base_type(arg_curr) == ARG_PTR_TO_MEM) !=
-               arg_type_is_mem_size(arg_next);
+       bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE;
+       bool has_size = fn->arg_size[arg] != 0;
+       bool is_next_size = false;
+
+       if (arg + 1 < ARRAY_SIZE(fn->arg_type))
+               is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]);
+
+       if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM)
+               return is_next_size;
+
+       return has_size == is_next_size || is_next_size == is_fixed;
 }
 
 static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
@@ -6415,11 +6433,11 @@ static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
         * helper function specification.
         */
        if (arg_type_is_mem_size(fn->arg1_type) ||
-           base_type(fn->arg5_type) == ARG_PTR_TO_MEM ||
-           check_args_pair_invalid(fn->arg1_type, fn->arg2_type) ||
-           check_args_pair_invalid(fn->arg2_type, fn->arg3_type) ||
-           check_args_pair_invalid(fn->arg3_type, fn->arg4_type) ||
-           check_args_pair_invalid(fn->arg4_type, fn->arg5_type))
+           check_args_pair_invalid(fn, 0) ||
+           check_args_pair_invalid(fn, 1) ||
+           check_args_pair_invalid(fn, 2) ||
+           check_args_pair_invalid(fn, 3) ||
+           check_args_pair_invalid(fn, 4))
                return false;
 
        return true;
@@ -6460,7 +6478,10 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn)
                if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i])
                        return false;
 
-               if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i])
+               if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] &&
+                   /* arg_btf_id and arg_size are in a union. */
+                   (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM ||
+                    !(fn->arg_type[i] & MEM_FIXED_SIZE)))
                        return false;
        }