bpf, tests: Add tests for atomic operations

Tests for each atomic arithmetic operation and BPF_XCHG, derived from
old BPF_XADD tests. The tests include BPF_W/DW and BPF_FETCH variants.

Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Yonghong Song <yhs@fb.com>
Link: https://lore.kernel.org/bpf/20210809091829.810076-13-johan.almbladh@anyfinetworks.com

diff --git a/lib/test_bpf.c b/lib/test_bpf.c
index fcfaf45..855f640 100644 (file)
--- a/lib/test_bpf.c
+++ b/lib/test_bpf.c
@@ -5508,49 +5508,6 @@ static struct bpf_test tests[] = {
                .stack_depth = 40,
        },
        /* BPF_STX | BPF_ATOMIC | BPF_W/DW */
-       {
-               "STX_XADD_W: Test: 0x12 + 0x10 = 0x22",
-               .u.insns_int = {
-                       BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
-                       BPF_ST_MEM(BPF_W, R10, -40, 0x10),
-                       BPF_ATOMIC_OP(BPF_W, BPF_ADD, R10, R0, -40),
-                       BPF_LDX_MEM(BPF_W, R0, R10, -40),
-                       BPF_EXIT_INSN(),
-               },
-               INTERNAL,
-               { },
-               { { 0, 0x22 } },
-               .stack_depth = 40,
-       },
-       {
-               "STX_XADD_W: Test side-effects, r10: 0x12 + 0x10 = 0x22",
-               .u.insns_int = {
-                       BPF_ALU64_REG(BPF_MOV, R1, R10),
-                       BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
-                       BPF_ST_MEM(BPF_W, R10, -40, 0x10),
-                       BPF_ATOMIC_OP(BPF_W, BPF_ADD, R10, R0, -40),
-                       BPF_ALU64_REG(BPF_MOV, R0, R10),
-                       BPF_ALU64_REG(BPF_SUB, R0, R1),
-                       BPF_EXIT_INSN(),
-               },
-               INTERNAL,
-               { },
-               { { 0, 0 } },
-               .stack_depth = 40,
-       },
-       {
-               "STX_XADD_W: Test side-effects, r0: 0x12 + 0x10 = 0x22",
-               .u.insns_int = {
-                       BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
-                       BPF_ST_MEM(BPF_W, R10, -40, 0x10),
-                       BPF_ATOMIC_OP(BPF_W, BPF_ADD, R10, R0, -40),
-                       BPF_EXIT_INSN(),
-               },
-               INTERNAL,
-               { },
-               { { 0, 0x12 } },
-               .stack_depth = 40,
-       },
        {
                "STX_XADD_W: X + 1 + 1 + 1 + ...",
                { },
@@ -5559,49 +5516,6 @@ static struct bpf_test tests[] = {
                { { 0, 4134 } },
                .fill_helper = bpf_fill_stxw,
        },
-       {
-               "STX_XADD_DW: Test: 0x12 + 0x10 = 0x22",
-               .u.insns_int = {
-                       BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
-                       BPF_ST_MEM(BPF_DW, R10, -40, 0x10),
-                       BPF_ATOMIC_OP(BPF_DW, BPF_ADD, R10, R0, -40),
-                       BPF_LDX_MEM(BPF_DW, R0, R10, -40),
-                       BPF_EXIT_INSN(),
-               },
-               INTERNAL,
-               { },
-               { { 0, 0x22 } },
-               .stack_depth = 40,
-       },
-       {
-               "STX_XADD_DW: Test side-effects, r10: 0x12 + 0x10 = 0x22",
-               .u.insns_int = {
-                       BPF_ALU64_REG(BPF_MOV, R1, R10),
-                       BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
-                       BPF_ST_MEM(BPF_DW, R10, -40, 0x10),
-                       BPF_ATOMIC_OP(BPF_DW, BPF_ADD, R10, R0, -40),
-                       BPF_ALU64_REG(BPF_MOV, R0, R10),
-                       BPF_ALU64_REG(BPF_SUB, R0, R1),
-                       BPF_EXIT_INSN(),
-               },
-               INTERNAL,
-               { },
-               { { 0, 0 } },
-               .stack_depth = 40,
-       },
-       {
-               "STX_XADD_DW: Test side-effects, r0: 0x12 + 0x10 = 0x22",
-               .u.insns_int = {
-                       BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
-                       BPF_ST_MEM(BPF_DW, R10, -40, 0x10),
-                       BPF_ATOMIC_OP(BPF_DW, BPF_ADD, R10, R0, -40),
-                       BPF_EXIT_INSN(),
-               },
-               INTERNAL,
-               { },
-               { { 0, 0x12 } },
-               .stack_depth = 40,
-       },
        {
                "STX_XADD_DW: X + 1 + 1 + 1 + ...",
                { },
@@ -5610,6 +5524,172 @@ static struct bpf_test tests[] = {
                { { 0, 4134 } },
                .fill_helper = bpf_fill_stxdw,
        },
+       /*
+        * Exhaustive tests of atomic operation variants.
+        * Individual tests are expanded from template macros for all
+        * combinations of ALU operation, word size and fetching.
+        */
+#define BPF_ATOMIC_OP_TEST1(width, op, logic, old, update, result)     \
+{                                                                      \
+       "BPF_ATOMIC | " #width ", " #op ": Test: "                      \
+               #old " " #logic " " #update " = " #result,              \
+       .u.insns_int = {                                                \
+               BPF_ALU32_IMM(BPF_MOV, R5, update),                     \
+               BPF_ST_MEM(width, R10, -40, old),                       \
+               BPF_ATOMIC_OP(width, op, R10, R5, -40),                 \
+               BPF_LDX_MEM(width, R0, R10, -40),                       \
+               BPF_EXIT_INSN(),                                        \
+       },                                                              \
+       INTERNAL,                                                       \
+       { },                                                            \
+       { { 0, result } },                                              \
+       .stack_depth = 40,                                              \
+}
+#define BPF_ATOMIC_OP_TEST2(width, op, logic, old, update, result)     \
+{                                                                      \
+       "BPF_ATOMIC | " #width ", " #op ": Test side effects, r10: "    \
+               #old " " #logic " " #update " = " #result,              \
+       .u.insns_int = {                                                \
+               BPF_ALU64_REG(BPF_MOV, R1, R10),                        \
+               BPF_ALU32_IMM(BPF_MOV, R0, update),                     \
+               BPF_ST_MEM(BPF_W, R10, -40, old),                       \
+               BPF_ATOMIC_OP(width, op, R10, R0, -40),                 \
+               BPF_ALU64_REG(BPF_MOV, R0, R10),                        \
+               BPF_ALU64_REG(BPF_SUB, R0, R1),                         \
+               BPF_EXIT_INSN(),                                        \
+       },                                                              \
+       INTERNAL,                                                       \
+       { },                                                            \
+       { { 0, 0 } },                                                   \
+       .stack_depth = 40,                                              \
+}
+#define BPF_ATOMIC_OP_TEST3(width, op, logic, old, update, result)     \
+{                                                                      \
+       "BPF_ATOMIC | " #width ", " #op ": Test side effects, r0: "     \
+               #old " " #logic " " #update " = " #result,              \
+       .u.insns_int = {                                                \
+               BPF_ALU64_REG(BPF_MOV, R0, R10),                        \
+               BPF_ALU32_IMM(BPF_MOV, R1, update),                     \
+               BPF_ST_MEM(width, R10, -40, old),                       \
+               BPF_ATOMIC_OP(width, op, R10, R1, -40),                 \
+               BPF_ALU64_REG(BPF_SUB, R0, R10),                        \
+               BPF_EXIT_INSN(),                                        \
+       },                                                              \
+       INTERNAL,                                                       \
+       { },                                                            \
+       { { 0, 0 } },                                                   \
+       .stack_depth = 40,                                              \
+}
+#define BPF_ATOMIC_OP_TEST4(width, op, logic, old, update, result)     \
+{                                                                      \
+       "BPF_ATOMIC | " #width ", " #op ": Test fetch: "                \
+               #old " " #logic " " #update " = " #result,              \
+       .u.insns_int = {                                                \
+               BPF_ALU32_IMM(BPF_MOV, R3, update),                     \
+               BPF_ST_MEM(width, R10, -40, old),                       \
+               BPF_ATOMIC_OP(width, op, R10, R3, -40),                 \
+               BPF_ALU64_REG(BPF_MOV, R0, R3),                         \
+               BPF_EXIT_INSN(),                                        \
+       },                                                              \
+       INTERNAL,                                                       \
+       { },                                                            \
+       { { 0, (op) & BPF_FETCH ? old : update } },                     \
+       .stack_depth = 40,                                              \
+}
+       /* BPF_ATOMIC | BPF_W: BPF_ADD */
+       BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd),
+       /* BPF_ATOMIC | BPF_W: BPF_ADD | BPF_FETCH */
+       BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd),
+       /* BPF_ATOMIC | BPF_DW: BPF_ADD */
+       BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd),
+       /* BPF_ATOMIC | BPF_DW: BPF_ADD | BPF_FETCH */
+       BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd),
+       BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd),
+       /* BPF_ATOMIC | BPF_W: BPF_AND */
+       BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02),
+       /* BPF_ATOMIC | BPF_W: BPF_AND | BPF_FETCH */
+       BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02),
+       /* BPF_ATOMIC | BPF_DW: BPF_AND */
+       BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02),
+       /* BPF_ATOMIC | BPF_DW: BPF_AND | BPF_FETCH */
+       BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02),
+       BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02),
+       /* BPF_ATOMIC | BPF_W: BPF_OR */
+       BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb),
+       /* BPF_ATOMIC | BPF_W: BPF_OR | BPF_FETCH */
+       BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb),
+       /* BPF_ATOMIC | BPF_DW: BPF_OR */
+       BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb),
+       /* BPF_ATOMIC | BPF_DW: BPF_OR | BPF_FETCH */
+       BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb),
+       BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb),
+       /* BPF_ATOMIC | BPF_W: BPF_XOR */
+       BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9),
+       /* BPF_ATOMIC | BPF_W: BPF_XOR | BPF_FETCH */
+       BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9),
+       /* BPF_ATOMIC | BPF_DW: BPF_XOR */
+       BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9),
+       /* BPF_ATOMIC | BPF_DW: BPF_XOR | BPF_FETCH */
+       BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9),
+       BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9),
+       /* BPF_ATOMIC | BPF_W: BPF_XCHG */
+       BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab),
+       BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab),
+       BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab),
+       BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab),
+       /* BPF_ATOMIC | BPF_DW: BPF_XCHG */
+       BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab),
+       BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab),
+       BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab),
+       BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab),
+#undef BPF_ATOMIC_OP_TEST1
+#undef BPF_ATOMIC_OP_TEST2
+#undef BPF_ATOMIC_OP_TEST3
+#undef BPF_ATOMIC_OP_TEST4
        /* BPF_JMP32 | BPF_JEQ | BPF_K */
        {
                "JMP32_JEQ_K: Small immediate",