bpf, arm64: use more scalable stadd over ldxr / stxr loop in xadd

[linux-2.6-microblaze.git] / arch / arm64 / net / bpf_jit_comp.c

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index a142062..df845ce 100644 (file)
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -365,7 +365,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
        const bool is64 = BPF_CLASS(code) == BPF_ALU64 ||
                          BPF_CLASS(code) == BPF_JMP;
        const bool isdw = BPF_SIZE(code) == BPF_DW;
-       u8 jmp_cond;
+       u8 jmp_cond, reg;
        s32 jmp_offset;
 
 #define check_imm(bits, imm) do {                              \
@@ -756,18 +756,28 @@ emit_cond_jmp:
                        break;
                }
                break;
+
        /* STX XADD: lock *(u32 *)(dst + off) += src */
        case BPF_STX | BPF_XADD | BPF_W:
        /* STX XADD: lock *(u64 *)(dst + off) += src */
        case BPF_STX | BPF_XADD | BPF_DW:
-               emit_a64_mov_i(1, tmp, off, ctx);
-               emit(A64_ADD(1, tmp, tmp, dst), ctx);
-               emit(A64_LDXR(isdw, tmp2, tmp), ctx);
-               emit(A64_ADD(isdw, tmp2, tmp2, src), ctx);
-               emit(A64_STXR(isdw, tmp2, tmp, tmp3), ctx);
-               jmp_offset = -3;
-               check_imm19(jmp_offset);
-               emit(A64_CBNZ(0, tmp3, jmp_offset), ctx);
+               if (!off) {
+                       reg = dst;
+               } else {
+                       emit_a64_mov_i(1, tmp, off, ctx);
+                       emit(A64_ADD(1, tmp, tmp, dst), ctx);
+                       reg = tmp;
+               }
+               if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS)) {
+                       emit(A64_STADD(isdw, reg, src), ctx);
+               } else {
+                       emit(A64_LDXR(isdw, tmp2, reg), ctx);
+                       emit(A64_ADD(isdw, tmp2, tmp2, src), ctx);
+                       emit(A64_STXR(isdw, tmp2, reg, tmp3), ctx);
+                       jmp_offset = -3;
+                       check_imm19(jmp_offset);
+                       emit(A64_CBNZ(0, tmp3, jmp_offset), ctx);
+               }
                break;
 
        default: