Merge https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

[linux-2.6-microblaze.git] / arch / x86 / net / bpf_jit_comp.c

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index c98b8c0..b88f43c 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1771,6 +1771,10 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
                           struct bpf_tramp_link *l, int stack_size,
                           int run_ctx_off, bool save_ret)
 {
+       void (*exit)(struct bpf_prog *prog, u64 start,
+                    struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_exit;
+       u64 (*enter)(struct bpf_prog *prog,
+                    struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_enter;
        u8 *prog = *pprog;
        u8 *jmp_insn;
        int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
@@ -1789,15 +1793,21 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
         */
        emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off);
 
+       if (p->aux->sleepable) {
+               enter = __bpf_prog_enter_sleepable;
+               exit = __bpf_prog_exit_sleepable;
+       } else if (p->expected_attach_type == BPF_LSM_CGROUP) {
+               enter = __bpf_prog_enter_lsm_cgroup;
+               exit = __bpf_prog_exit_lsm_cgroup;
+       }
+
        /* arg1: mov rdi, progs[i] */
        emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
        /* arg2: lea rsi, [rbp - ctx_cookie_off] */
        EMIT4(0x48, 0x8D, 0x75, -run_ctx_off);
 
-       if (emit_call(&prog,
-                     p->aux->sleepable ? __bpf_prog_enter_sleepable :
-                     __bpf_prog_enter, prog))
-                       return -EINVAL;
+       if (emit_call(&prog, enter, prog))
+               return -EINVAL;
        /* remember prog start time returned by __bpf_prog_enter */
        emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
 
@@ -1841,10 +1851,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
        emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
        /* arg3: lea rdx, [rbp - run_ctx_off] */
        EMIT4(0x48, 0x8D, 0x55, -run_ctx_off);
-       if (emit_call(&prog,
-                     p->aux->sleepable ? __bpf_prog_exit_sleepable :
-                     __bpf_prog_exit, prog))
-                       return -EINVAL;
+       if (emit_call(&prog, exit, prog))
+               return -EINVAL;
 
        *pprog = prog;
        return 0;
@@ -2492,3 +2500,9 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len)
                return ERR_PTR(-EINVAL);
        return dst;
 }
+
+/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
+bool bpf_jit_supports_subprog_tailcalls(void)
+{
+       return true;
+}