bpf: Enable private stack for eligible subprogs

If private stack is used by any subprog, set that subprog
prog->aux->jits_use_priv_stack to be true so later jit can allocate
private stack for that subprog properly.

Also set env->prog->aux->jits_use_priv_stack to be true if
any subprog uses private stack. This is a use case for a
single main prog (no subprogs) to use private stack, and
also a use case for later struct-ops progs where
env->prog->aux->jits_use_priv_stack will enable recursion
check if any subprog uses private stack.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
Link: https://lore.kernel.org/r/20241112163912.2224007-1-yonghong.song@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 7da41ae..129b29e 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1523,6 +1523,7 @@ struct bpf_prog_aux {
        bool exception_cb;
        bool exception_boundary;
        bool is_extended; /* true if extended by freplace program */
+       bool jits_use_priv_stack;
        u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
        struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
        struct bpf_arena *arena;

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index fb23793..176d19a 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -6334,6 +6334,14 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
                                return ret;
                }
        }
+
+       for (int i = 0; i < env->subprog_cnt; i++) {
+               if (si[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
+                       env->prog->aux->jits_use_priv_stack = true;
+                       break;
+               }
+       }
+
        return 0;
 }
 
@@ -20274,6 +20282,9 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 
                func[i]->aux->name[0] = 'F';
                func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
+               if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE)
+                       func[i]->aux->jits_use_priv_stack = true;
+
                func[i]->jit_requested = 1;
                func[i]->blinding_requested = prog->blinding_requested;
                func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;