bpf: Fix s16 truncation for large bpf-to-bpf call offsets

Currently, the BPF instruction set allows bpf-to-bpf calls (or internal
calls, pseudo calls) to use a 32-bit imm field to represent the relative
jump offset.

However, when JIT is disabled or falls back to the interpreter, the
verifier invokes bpf_patch_call_args() to rewrite the call instruction.
In this function, the 32-bit imm is downcast to s16 and stored in the off
field.

    void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
    {
        stack_depth = max_t(u32, stack_depth, 1);
        insn->off = (s16) insn->imm;
        insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] -
            __bpf_call_base_args;
        insn->code = BPF_JMP | BPF_CALL_ARGS;
    }

If the original imm exceeds the s16 range (i.e., a jump offset greater
than 32767 instructions), this downcast silently truncates the offset,
resulting in an incorrect call target.

Fix this by:
1. In bpf_patch_call_args(), keeping the imm field unchanged and using the
   off field to store the index of the interpreter function.
2. In ___bpf_prog_run() for the JMP_CALL_ARGS case, retrieving the
   interpreter function pointer from the interpreters_args array using the
   off field as the index, and passing the original imm to calculate the
   last argument of the interpreter function.

After these changes, the truncation issue is resolved, and __bpf_call_base_args
is also no longer needed and can be removed, which makes the code cleaner.

Performance: In ___bpf_prog_run() for the JMP_CALL_ARGS case, changing the
retrieval of the interpreter function pointer from pointer addition to
direct array indexing improves performance. The possible reason is that the
latter has better instruction-level parallelism. See the v5 discussion [1]
for more details.

[1] https://lore.kernel.org/bpf/f120c3c4-6999-414a-b514-518bb64b4758@zju.edu.cn/

To avoid requiring bpftool changes, keep the new imm/off encoding internal
and restore the legacy xlated dump layout in bpf_insn_prepare_dump().
For bpf-to-bpf call offsets that do not fit in s16, export off as 0 instead
of a truncated and misleading value.

Fixes: 1ea47e01ad6e ("bpf: add support for bpf_call to interpreter")
Fixes: 7105e828c087 ("bpf: allow for correlation of maps and helpers in dump")
Suggested-by: Xu Kuohai <xukuohai@huaweicloud.com>
Suggested-by: Puranjay Mohan <puranjay@kernel.org>
Co-developed-by: Tianci Cao <ziye@zju.edu.cn>
Signed-off-by: Tianci Cao <ziye@zju.edu.cn>
Co-developed-by: Shenghao Yuan <shenghaoyuan0928@163.com>
Signed-off-by: Shenghao Yuan <shenghaoyuan0928@163.com>
Signed-off-by: Yazhou Tang <tangyazhou518@outlook.com>
Link: https://lore.kernel.org/r/20260506094714.419842-3-tangyazhou@zju.edu.cn
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 52b30e9..cd191c5 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2918,6 +2918,12 @@ int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 ua
 
 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
 int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+s32 bpf_call_args_imm(s16 idx);
+#else
+static inline s32 bpf_call_args_imm(s16 idx)
+{
+       return 0;
+}
 #endif
 
 struct btf *bpf_get_btf_vmlinux(void);

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 1ec6d5b..88a241a 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1151,9 +1151,6 @@ bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
 
 u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
-#define __bpf_call_base_args \
-       ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
-        (void *)__bpf_call_base)
 
 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
 void bpf_jit_compile(struct bpf_prog *prog);

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 63044eb..6aa2a8b 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1771,6 +1771,9 @@ static u32 abs_s32(s32 x)
        return x >= 0 ? (u32)x : -(u32)x;
 }
 
+static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5,
+                                 const struct bpf_insn *insn);
+
 /**
  *     ___bpf_prog_run - run eBPF program on a given context
  *     @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers
@@ -2077,10 +2080,9 @@ select_insn:
                CONT;
 
        JMP_CALL_ARGS:
-               BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2,
-                                                           BPF_R3, BPF_R4,
-                                                           BPF_R5,
-                                                           insn + insn->off + 1);
+               BPF_R0 = interpreters_args[insn->off](BPF_R1, BPF_R2, BPF_R3,
+                                                     BPF_R4, BPF_R5,
+                                                     insn + insn->imm + 1);
                CONT;
 
        JMP_TAIL_CALL: {
@@ -2400,12 +2402,17 @@ int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
        /* Prevent out-of-bounds read to interpreters_args */
        if (stack_depth > MAX_BPF_STACK)
                return -EINVAL;
-       insn->off = (s16) insn->imm;
-       insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] -
-               __bpf_call_base_args;
+       insn->off = (round_up(stack_depth, 32) / 32) - 1;
        insn->code = BPF_JMP | BPF_CALL_ARGS;
        return 0;
 }
+
+s32 bpf_call_args_imm(s16 idx)
+{
+       if (WARN_ON_ONCE(idx < 0 || idx >= ARRAY_SIZE(interpreters_args)))
+               return 0;
+       return BPF_CALL_IMM(interpreters_args[idx]);
+}
 #endif
 #endif
 

diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index df8f480..3692adf 100644 (file)
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1250,9 +1250,9 @@ static int jit_subprogs(struct bpf_verifier_env *env)
                }
                if (!bpf_pseudo_call(insn))
                        continue;
-               insn->off = env->insn_aux_data[i].call_imm;
-               subprog = bpf_find_subprog(env, i + insn->off + 1);
-               insn->imm = subprog;
+               insn->imm = env->insn_aux_data[i].call_imm;
+               subprog = bpf_find_subprog(env, i + insn->imm + 1);
+               insn->off = subprog;
        }
 
        prog->jited = 1;

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index a3c0214..630d530 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -4919,6 +4919,29 @@ out:
        return map;
 }
 
+static void prepare_dump_pseudo_call(struct bpf_insn *insn)
+{
+       s32 call_off = insn->imm;
+
+       /*
+        * BPF_CALL_ARGS only exists for interpreter fallback.
+        * 1. For interpreter (BPF_CALL_ARGS): insn->off is the index of
+        *    interpreters_args array, so here using bpf_call_args_imm()
+        *    to get the real address offset.
+        * 2. For JIT (BPF_CALL): insn->off is the subprog id.
+        */
+       if (insn->code == (BPF_JMP | BPF_CALL_ARGS))
+               insn->imm = bpf_call_args_imm(insn->off);
+       else
+               insn->imm = insn->off;
+
+       /* Avoid dumping a truncated and misleading pc-relative offset. */
+       if (call_off > S16_MAX || call_off < S16_MIN)
+               insn->off = 0;
+       else
+               insn->off = call_off;
+}
+
 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
                                              const struct cred *f_cred)
 {
@@ -4944,6 +4967,9 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
                }
                if (code == (BPF_JMP | BPF_CALL) ||
                    code == (BPF_JMP | BPF_CALL_ARGS)) {
+                       /* Restore the legacy xlated dump layout. */
+                       if (insns[i].src_reg == BPF_PSEUDO_CALL)
+                               prepare_dump_pseudo_call(&insns[i]);
                        if (code == (BPF_JMP | BPF_CALL_ARGS))
                                insns[i].code = BPF_JMP | BPF_CALL;
                        if (!bpf_dump_raw_ok(f_cred))