bpf: Add batched ops support for percpu array

[linux-2.6-microblaze.git] / kernel / bpf / core.c

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 75244ec..5e31ee9 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -143,25 +143,25 @@ int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)
        if (!prog->aux->nr_linfo || !prog->jit_requested)
                return 0;
 
-       prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo,
-                                        sizeof(*prog->aux->jited_linfo),
-                                        GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
+       prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,
+                                         sizeof(*prog->aux->jited_linfo),
+                                         GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
        if (!prog->aux->jited_linfo)
                return -ENOMEM;
 
        return 0;
 }
 
-void bpf_prog_free_jited_linfo(struct bpf_prog *prog)
+void bpf_prog_jit_attempt_done(struct bpf_prog *prog)
 {
-       kfree(prog->aux->jited_linfo);
-       prog->aux->jited_linfo = NULL;
-}
+       if (prog->aux->jited_linfo &&
+           (!prog->jited || !prog->aux->jited_linfo[0])) {
+               kvfree(prog->aux->jited_linfo);
+               prog->aux->jited_linfo = NULL;
+       }
 
-void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog)
-{
-       if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0])
-               bpf_prog_free_jited_linfo(prog);
+       kfree(prog->aux->kfunc_tab);
+       prog->aux->kfunc_tab = NULL;
 }
 
 /* The jit engine is responsible to provide an array
@@ -217,12 +217,6 @@ void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
                        insn_to_jit_off[linfo[i].insn_off - insn_start - 1];
 }
 
-void bpf_prog_free_linfo(struct bpf_prog *prog)
-{
-       bpf_prog_free_jited_linfo(prog);
-       kvfree(prog->aux->linfo);
-}
-
 struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
                                  gfp_t gfp_extra_flags)
 {
@@ -1369,11 +1363,10 @@ u64 __weak bpf_probe_read_kernel(void *dst, u32 size, const void *unsafe_ptr)
  *     __bpf_prog_run - run eBPF program on a given context
  *     @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers
  *     @insn: is the array of eBPF instructions
- *     @stack: is the eBPF storage stack
  *
  * Decode and execute eBPF instructions.
  */
-static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack)
+static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn)
 {
 #define BPF_INSN_2_LBL(x, y)    [BPF_##x | BPF_##y] = &&x##_##y
 #define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z
@@ -1707,7 +1700,7 @@ static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn
 \
        FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \
        ARG1 = (u64) (unsigned long) ctx; \
-       return ___bpf_prog_run(regs, insn, stack); \
+       return ___bpf_prog_run(regs, insn); \
 }
 
 #define PROG_NAME_ARGS(stack_size) __bpf_prog_run_args##stack_size
@@ -1724,7 +1717,7 @@ static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \
        BPF_R3 = r3; \
        BPF_R4 = r4; \
        BPF_R5 = r5; \
-       return ___bpf_prog_run(regs, insn, stack); \
+       return ___bpf_prog_run(regs, insn); \
 }
 
 #define EVAL1(FN, X) FN(X)
@@ -1849,9 +1842,15 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
        /* In case of BPF to BPF calls, verifier did all the prep
         * work with regards to JITing, etc.
         */
+       bool jit_needed = false;
+
        if (fp->bpf_func)
                goto finalize;
 
+       if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) ||
+           bpf_prog_has_kfunc_call(fp))
+               jit_needed = true;
+
        bpf_prog_select_func(fp);
 
        /* eBPF JITs can rewrite the program in case constant
@@ -1866,14 +1865,10 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
                        return fp;
 
                fp = bpf_int_jit_compile(fp);
-               if (!fp->jited) {
-                       bpf_prog_free_jited_linfo(fp);
-#ifdef CONFIG_BPF_JIT_ALWAYS_ON
+               bpf_prog_jit_attempt_done(fp);
+               if (!fp->jited && jit_needed) {
                        *err = -ENOTSUPP;
                        return fp;
-#endif
-               } else {
-                       bpf_prog_free_unused_jited_linfo(fp);
                }
        } else {
                *err = bpf_prog_offload_compile(fp);
@@ -2354,6 +2349,11 @@ bool __weak bpf_jit_needs_zext(void)
        return false;
 }
 
+bool __weak bpf_jit_supports_kfunc_call(void)
+{
+       return false;
+}
+
 /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
  * skb_copy_bits(), so provide a weak definition of it for NET-less config.
  */