1 // SPDX-License-Identifier: GPL-2.0-only
3 * eBPF JIT compiler for PPC32
5 * Copyright 2020 Christophe Leroy <christophe.leroy@csgroup.eu>
8 * Based on PPC64 eBPF JIT compiler by Naveen N. Rao
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <asm/asm-compat.h>
13 #include <linux/netdevice.h>
14 #include <linux/filter.h>
15 #include <linux/if_vlan.h>
16 #include <asm/kprobes.h>
17 #include <linux/bpf.h>
24 * [ prev sp ] <-------------
25 * [ nv gpr save area ] 16 * 4 |
26 * fp (r31) --> [ ebpf stack space ] upto 512 |
27 * [ frame header ] 16 |
28 * sp (r1) ---> [ stack pointer ] --------------
31 /* for gpr non volatile registers r17 to r31 (14) + tail call */
32 #define BPF_PPC_STACK_SAVE (15 * 4 + 4)
33 /* stack frame, ensure this is quadword aligned */
34 #define BPF_PPC_STACKFRAME(ctx) (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size)
36 /* BPF register usage */
37 #define TMP_REG (MAX_BPF_JIT_REG + 0)
39 /* BPF to ppc register mappings */
40 const int b2p[MAX_BPF_JIT_REG + 1] = {
41 /* function return value */
43 /* function arguments */
49 /* non volatile registers */
54 /* frame pointer aka BPF_REG_10 */
56 /* eBPF jit internal registers */
58 [TMP_REG] = 31, /* 32 bits */
61 static int bpf_to_ppc(struct codegen_context *ctx, int reg)
66 /* PPC NVR range -- update this if we ever use NVRs below r17 */
67 #define BPF_PPC_NVR_MIN 17
70 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
72 if ((reg >= BPF_PPC_NVR_MIN && reg < 32) || reg == BPF_PPC_TC)
73 return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg);
75 WARN(true, "BPF JIT is asking about unknown registers, will crash the stack");
76 /* Use the hole we have left for alignment */
77 return BPF_PPC_STACKFRAME(ctx) - 4;
80 void bpf_jit_realloc_regs(struct codegen_context *ctx)
82 if (ctx->seen & SEEN_FUNC)
85 while (ctx->seen & SEEN_NVREG_MASK &&
86 (ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) {
87 int old = 32 - fls(ctx->seen & (SEEN_NVREG_MASK & 0xaaaaaaab));
88 int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa));
91 for (i = BPF_REG_0; i <= TMP_REG; i++) {
92 if (ctx->b2p[i] != old)
95 bpf_set_seen_register(ctx, new);
96 bpf_clear_seen_register(ctx, old);
98 bpf_set_seen_register(ctx, new - 1);
99 bpf_clear_seen_register(ctx, old - 1);
106 void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
110 /* First arg comes in as a 32 bits pointer. */
111 EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_1), _R3));
112 EMIT(PPC_RAW_LI(bpf_to_ppc(ctx, BPF_REG_1) - 1, 0));
113 EMIT(PPC_RAW_STWU(_R1, _R1, -BPF_PPC_STACKFRAME(ctx)));
116 * Initialize tail_call_cnt in stack frame if we do tail calls.
117 * Otherwise, put in NOPs so that it can be skipped when we are
118 * invoked through a tail call.
120 if (ctx->seen & SEEN_TAILCALL)
121 EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_1) - 1, _R1,
122 bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
126 #define BPF_TAILCALL_PROLOGUE_SIZE 16
129 * We need a stack frame, but we don't necessarily need to
130 * save/restore LR unless we call other functions
132 if (ctx->seen & SEEN_FUNC)
133 EMIT(PPC_RAW_MFLR(_R0));
136 * Back up non-volatile regs -- registers r18-r31
138 for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
139 if (bpf_is_seen_register(ctx, i))
140 EMIT(PPC_RAW_STW(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
142 /* If needed retrieve arguments 9 and 10, ie 5th 64 bits arg.*/
143 if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_5))) {
144 EMIT(PPC_RAW_LWZ(bpf_to_ppc(ctx, BPF_REG_5) - 1, _R1, BPF_PPC_STACKFRAME(ctx)) + 8);
145 EMIT(PPC_RAW_LWZ(bpf_to_ppc(ctx, BPF_REG_5), _R1, BPF_PPC_STACKFRAME(ctx)) + 12);
148 /* Setup frame pointer to point to the bpf stack area */
149 if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_FP))) {
150 EMIT(PPC_RAW_LI(bpf_to_ppc(ctx, BPF_REG_FP) - 1, 0));
151 EMIT(PPC_RAW_ADDI(bpf_to_ppc(ctx, BPF_REG_FP), _R1,
152 STACK_FRAME_MIN_SIZE + ctx->stack_size));
155 if (ctx->seen & SEEN_FUNC)
156 EMIT(PPC_RAW_STW(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
159 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
164 for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
165 if (bpf_is_seen_register(ctx, i))
166 EMIT(PPC_RAW_LWZ(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
169 void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
171 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(ctx, BPF_REG_0)));
173 bpf_jit_emit_common_epilogue(image, ctx);
175 /* Tear down our stack frame */
177 if (ctx->seen & SEEN_FUNC)
178 EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
180 EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME(ctx)));
182 if (ctx->seen & SEEN_FUNC)
183 EMIT(PPC_RAW_MTLR(_R0));
188 void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func)
190 s32 rel = (s32)func - (s32)(image + ctx->idx);
192 if (image && rel < 0x2000000 && rel >= -0x2000000) {
195 /* Load function address into r0 */
196 EMIT(PPC_RAW_LIS(_R0, IMM_H(func)));
197 EMIT(PPC_RAW_ORI(_R0, _R0, IMM_L(func)));
198 EMIT(PPC_RAW_MTCTR(_R0));
199 EMIT(PPC_RAW_BCTRL());
203 static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
206 * By now, the eBPF program has already setup parameters in r3-r6
207 * r3-r4/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
208 * r5-r6/BPF_REG_2 - pointer to bpf_array
209 * r7-r8/BPF_REG_3 - index in bpf_array
211 int b2p_bpf_array = bpf_to_ppc(ctx, BPF_REG_2);
212 int b2p_index = bpf_to_ppc(ctx, BPF_REG_3);
215 * if (index >= array->map.max_entries)
218 EMIT(PPC_RAW_LWZ(_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
219 EMIT(PPC_RAW_CMPLW(b2p_index, _R0));
220 EMIT(PPC_RAW_LWZ(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
221 PPC_BCC(COND_GE, out);
224 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT)
227 EMIT(PPC_RAW_CMPLWI(_R0, MAX_TAIL_CALL_CNT));
228 /* tail_call_cnt++; */
229 EMIT(PPC_RAW_ADDIC(_R0, _R0, 1));
230 PPC_BCC(COND_GE, out);
232 /* prog = array->ptrs[index]; */
233 EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29));
234 EMIT(PPC_RAW_ADD(_R3, _R3, b2p_bpf_array));
235 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_array, ptrs)));
236 EMIT(PPC_RAW_STW(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
242 EMIT(PPC_RAW_CMPLWI(_R3, 0));
243 PPC_BCC(COND_EQ, out);
245 /* goto *(prog->bpf_func + prologue_size); */
246 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_prog, bpf_func)));
248 if (ctx->seen & SEEN_FUNC)
249 EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
251 EMIT(PPC_RAW_ADDIC(_R3, _R3, BPF_TAILCALL_PROLOGUE_SIZE));
253 if (ctx->seen & SEEN_FUNC)
254 EMIT(PPC_RAW_MTLR(_R0));
256 EMIT(PPC_RAW_MTCTR(_R3));
258 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(ctx, BPF_REG_1)));
260 /* tear restore NVRs, ... */
261 bpf_jit_emit_common_epilogue(image, ctx);
263 EMIT(PPC_RAW_BCTR());
269 /* Assemble the body code between the prologue & epilogue */
270 int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
271 u32 *addrs, int pass)
273 const struct bpf_insn *insn = fp->insnsi;
277 /* Start of epilogue code - will only be valid 2nd pass onwards */
278 u32 exit_addr = addrs[flen];
280 for (i = 0; i < flen; i++) {
281 u32 code = insn[i].code;
282 u32 dst_reg = bpf_to_ppc(ctx, insn[i].dst_reg);
283 u32 dst_reg_h = dst_reg - 1;
284 u32 src_reg = bpf_to_ppc(ctx, insn[i].src_reg);
285 u32 src_reg_h = src_reg - 1;
286 u32 tmp_reg = bpf_to_ppc(ctx, TMP_REG);
287 u32 size = BPF_SIZE(code);
288 s16 off = insn[i].off;
289 s32 imm = insn[i].imm;
290 bool func_addr_fixed;
295 * addrs[] maps a BPF bytecode address into a real offset from
296 * the start of the body code.
298 addrs[i] = ctx->idx * 4;
301 * As an optimization, we note down which registers
302 * are used so that we can only save/restore those in our
303 * prologue and epilogue. We do this here regardless of whether
304 * the actual BPF instruction uses src/dst registers or not
305 * (for instance, BPF_CALL does not use them). The expectation
306 * is that those instructions will have src_reg/dst_reg set to
307 * 0. Even otherwise, we just lose some prologue/epilogue
308 * optimization but everything else should work without
311 if (dst_reg >= 3 && dst_reg < 32) {
312 bpf_set_seen_register(ctx, dst_reg);
313 bpf_set_seen_register(ctx, dst_reg_h);
316 if (src_reg >= 3 && src_reg < 32) {
317 bpf_set_seen_register(ctx, src_reg);
318 bpf_set_seen_register(ctx, src_reg_h);
323 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
325 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
326 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
328 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
329 EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, src_reg));
330 EMIT(PPC_RAW_ADDE(dst_reg_h, dst_reg_h, src_reg_h));
332 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
333 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
335 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
336 EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, dst_reg));
337 EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, dst_reg_h));
339 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
342 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
343 if (IMM_HA(imm) & 0xffff)
344 EMIT(PPC_RAW_ADDIS(dst_reg, dst_reg, IMM_HA(imm)));
346 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
348 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
351 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
355 if (imm >= -32768 && imm < 32768) {
356 EMIT(PPC_RAW_ADDIC(dst_reg, dst_reg, imm));
359 EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, _R0));
361 if (imm >= 0 || (BPF_OP(code) == BPF_SUB && imm == 0x80000000))
362 EMIT(PPC_RAW_ADDZE(dst_reg_h, dst_reg_h));
364 EMIT(PPC_RAW_ADDME(dst_reg_h, dst_reg_h));
366 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
367 bpf_set_seen_register(ctx, tmp_reg);
368 EMIT(PPC_RAW_MULW(_R0, dst_reg, src_reg_h));
369 EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, src_reg));
370 EMIT(PPC_RAW_MULHWU(tmp_reg, dst_reg, src_reg));
371 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
372 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
373 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg));
375 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
376 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
378 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
379 if (imm >= -32768 && imm < 32768) {
380 EMIT(PPC_RAW_MULI(dst_reg, dst_reg, imm));
383 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, _R0));
386 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
388 PPC_LI32(dst_reg, 0);
389 PPC_LI32(dst_reg_h, 0);
395 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
396 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
399 bpf_set_seen_register(ctx, tmp_reg);
400 PPC_LI32(tmp_reg, imm);
401 EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, tmp_reg));
403 EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, dst_reg));
404 EMIT(PPC_RAW_MULHWU(_R0, dst_reg, tmp_reg));
405 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp_reg));
406 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
408 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
409 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
411 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
412 EMIT(PPC_RAW_DIVWU(_R0, dst_reg, src_reg));
413 EMIT(PPC_RAW_MULW(_R0, src_reg, _R0));
414 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0));
416 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
418 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
420 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
427 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, _R0));
429 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
433 if (!is_power_of_2((u32)imm)) {
434 bpf_set_seen_register(ctx, tmp_reg);
435 PPC_LI32(tmp_reg, imm);
436 EMIT(PPC_RAW_DIVWU(_R0, dst_reg, tmp_reg));
437 EMIT(PPC_RAW_MULW(_R0, tmp_reg, _R0));
438 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0));
442 EMIT(PPC_RAW_LI(dst_reg, 0));
444 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2((u32)imm), 31));
447 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
452 if (!is_power_of_2(imm))
455 EMIT(PPC_RAW_LI(dst_reg, 0));
457 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2(imm), 31));
458 EMIT(PPC_RAW_LI(dst_reg_h, 0));
460 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
463 if (!is_power_of_2(abs(imm)))
467 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
468 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
474 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
475 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
476 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
478 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
479 EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
481 case BPF_ALU64 | BPF_NEG: /* dst = -dst */
482 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
483 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
487 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
489 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
490 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
491 EMIT(PPC_RAW_AND(dst_reg_h, dst_reg_h, src_reg_h));
493 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
494 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
496 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
498 EMIT(PPC_RAW_LI(dst_reg_h, 0));
500 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
502 EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
503 } else if (!IMM_L(imm)) {
504 EMIT(PPC_RAW_ANDIS(dst_reg, dst_reg, IMM_H(imm)));
505 } else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) {
506 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0,
507 32 - fls(imm), 32 - ffs(imm)));
510 EMIT(PPC_RAW_AND(dst_reg, dst_reg, _R0));
513 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
514 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
515 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, src_reg_h));
517 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
518 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
520 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
523 EMIT(PPC_RAW_LI(dst_reg_h, -1));
525 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
527 EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
529 EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
531 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
532 if (dst_reg == src_reg) {
533 EMIT(PPC_RAW_LI(dst_reg, 0));
534 EMIT(PPC_RAW_LI(dst_reg_h, 0));
536 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
537 EMIT(PPC_RAW_XOR(dst_reg_h, dst_reg_h, src_reg_h));
540 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
541 if (dst_reg == src_reg)
542 EMIT(PPC_RAW_LI(dst_reg, 0));
544 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
546 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
548 EMIT(PPC_RAW_NOR(dst_reg_h, dst_reg_h, dst_reg_h));
550 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
552 EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
554 EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
556 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
557 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
559 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
560 bpf_set_seen_register(ctx, tmp_reg);
561 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
562 EMIT(PPC_RAW_SLW(dst_reg_h, dst_reg_h, src_reg));
563 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
564 EMIT(PPC_RAW_SRW(_R0, dst_reg, _R0));
565 EMIT(PPC_RAW_SLW(tmp_reg, dst_reg, tmp_reg));
566 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, _R0));
567 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
568 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg));
570 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<= (u32) imm */
573 EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
575 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<= imm */
581 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, imm, 0, 31 - imm));
582 EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31));
583 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, imm, 0, 31 - imm));
587 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg, imm, 0, 31 - imm));
589 EMIT(PPC_RAW_LI(dst_reg_h, 0));
590 EMIT(PPC_RAW_LI(dst_reg, 0));
592 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
593 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
595 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
596 bpf_set_seen_register(ctx, tmp_reg);
597 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
598 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
599 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
600 EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0));
601 EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg));
602 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
603 EMIT(PPC_RAW_SRW(dst_reg_h, dst_reg_h, src_reg));
604 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
606 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
609 EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
611 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
617 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
618 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
619 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, 32 - imm, imm, 31));
623 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg_h, 64 - imm, imm - 32, 31));
625 EMIT(PPC_RAW_LI(dst_reg, 0));
626 EMIT(PPC_RAW_LI(dst_reg_h, 0));
628 case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
629 EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
631 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
632 bpf_set_seen_register(ctx, tmp_reg);
633 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
634 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
635 EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0));
636 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
637 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
638 EMIT(PPC_RAW_RLWINM(_R0, tmp_reg, 0, 26, 26));
639 EMIT(PPC_RAW_SRAW(tmp_reg, dst_reg_h, tmp_reg));
640 EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg_h, src_reg));
641 EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, _R0));
642 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
644 case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
647 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
649 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
655 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
656 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
657 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
661 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, imm - 32));
663 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, 31));
664 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, 31));
670 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
671 if (dst_reg == src_reg)
673 EMIT(PPC_RAW_MR(dst_reg, src_reg));
674 EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h));
676 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
677 /* special mov32 for zext */
679 EMIT(PPC_RAW_LI(dst_reg_h, 0));
680 else if (dst_reg != src_reg)
681 EMIT(PPC_RAW_MR(dst_reg, src_reg));
683 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
684 PPC_LI32(dst_reg, imm);
685 PPC_EX32(dst_reg_h, imm);
687 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
688 PPC_LI32(dst_reg, imm);
694 case BPF_ALU | BPF_END | BPF_FROM_LE:
697 /* Copy 16 bits to upper part */
698 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg, 16, 0, 15));
699 /* Rotate 8 bits right & mask */
700 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31));
704 * Rotate word left by 8 bits:
705 * 2 bytes are already in their final position
706 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
708 EMIT(PPC_RAW_RLWINM(_R0, dst_reg, 8, 0, 31));
709 /* Rotate 24 bits and insert byte 1 */
710 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 0, 7));
711 /* Rotate 24 bits and insert byte 3 */
712 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 16, 23));
713 EMIT(PPC_RAW_MR(dst_reg, _R0));
716 bpf_set_seen_register(ctx, tmp_reg);
717 EMIT(PPC_RAW_RLWINM(tmp_reg, dst_reg, 8, 0, 31));
718 EMIT(PPC_RAW_RLWINM(_R0, dst_reg_h, 8, 0, 31));
719 /* Rotate 24 bits and insert byte 1 */
720 EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 0, 7));
721 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 0, 7));
722 /* Rotate 24 bits and insert byte 3 */
723 EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 16, 23));
724 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 16, 23));
725 EMIT(PPC_RAW_MR(dst_reg, _R0));
726 EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg));
730 case BPF_ALU | BPF_END | BPF_FROM_BE:
733 /* zero-extend 16 bits into 32 bits */
734 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 16, 31));
744 * BPF_ST NOSPEC (speculation barrier)
746 case BPF_ST | BPF_NOSPEC:
752 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
753 EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
755 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
757 EMIT(PPC_RAW_STB(_R0, dst_reg, off));
759 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
760 EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
762 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
764 EMIT(PPC_RAW_STH(_R0, dst_reg, off));
766 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
767 EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
769 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
771 EMIT(PPC_RAW_STW(_R0, dst_reg, off));
773 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
774 EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off));
775 EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4));
777 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
779 EMIT(PPC_RAW_STW(_R0, dst_reg, off + 4));
781 EMIT(PPC_RAW_STW(_R0, dst_reg, off));
785 * BPF_STX ATOMIC (atomic ops)
787 case BPF_STX | BPF_ATOMIC | BPF_W:
788 if (imm != BPF_ADD) {
789 pr_err_ratelimited("eBPF filter atomic op code %02x (@%d) unsupported\n",
794 /* *(u32 *)(dst + off) += src */
796 bpf_set_seen_register(ctx, tmp_reg);
797 /* Get offset into TMP_REG */
798 EMIT(PPC_RAW_LI(tmp_reg, off));
799 /* load value from memory into r0 */
800 EMIT(PPC_RAW_LWARX(_R0, tmp_reg, dst_reg, 0));
801 /* add value from src_reg into this */
802 EMIT(PPC_RAW_ADD(_R0, _R0, src_reg));
803 /* store result back */
804 EMIT(PPC_RAW_STWCX(_R0, tmp_reg, dst_reg));
805 /* we're done if this succeeded */
806 PPC_BCC_SHORT(COND_NE, (ctx->idx - 3) * 4);
809 case BPF_STX | BPF_ATOMIC | BPF_DW: /* *(u64 *)(dst + off) += src */
815 case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
816 case BPF_LDX | BPF_PROBE_MEM | BPF_B:
817 case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
818 case BPF_LDX | BPF_PROBE_MEM | BPF_H:
819 case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
820 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
821 case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
822 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
824 * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid
825 * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM
826 * load only if addr is kernel address (see is_kernel_addr()), otherwise
827 * set dst_reg=0 and move on.
829 if (BPF_MODE(code) == BPF_PROBE_MEM) {
830 PPC_LI32(_R0, TASK_SIZE - off);
831 EMIT(PPC_RAW_CMPLW(src_reg, _R0));
832 PPC_BCC(COND_GT, (ctx->idx + 5) * 4);
833 EMIT(PPC_RAW_LI(dst_reg, 0));
835 * For BPF_DW case, "li reg_h,0" would be needed when
836 * !fp->aux->verifier_zext. Emit NOP otherwise.
838 * Note that "li reg_h,0" is emitted for BPF_B/H/W case,
839 * if necessary. So, jump there insted of emitting an
840 * additional "li reg_h,0" instruction.
842 if (size == BPF_DW && !fp->aux->verifier_zext)
843 EMIT(PPC_RAW_LI(dst_reg_h, 0));
847 * Need to jump two instructions instead of one for BPF_DW case
848 * as there are two load instructions for dst_reg_h & dst_reg
852 PPC_JMP((ctx->idx + 3) * 4);
854 PPC_JMP((ctx->idx + 2) * 4);
859 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
862 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
865 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
868 EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off));
869 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4));
873 if (size != BPF_DW && !fp->aux->verifier_zext)
874 EMIT(PPC_RAW_LI(dst_reg_h, 0));
876 if (BPF_MODE(code) == BPF_PROBE_MEM) {
877 int insn_idx = ctx->idx - 1;
881 * In case of BPF_DW, two lwz instructions are emitted, one
882 * for higher 32-bit and another for lower 32-bit. So, set
883 * ex->insn to the first of the two and jump over both
884 * instructions in fixup.
886 * Similarly, with !verifier_zext, two instructions are
887 * emitted for BPF_B/H/W case. So, set ex->insn to the
888 * instruction that could fault and skip over both
891 if (size == BPF_DW || !fp->aux->verifier_zext) {
896 ret = bpf_add_extable_entry(fp, image, pass, ctx, insn_idx,
905 * 16 byte instruction that uses two 'struct bpf_insn'
907 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
908 PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm);
909 PPC_LI32(dst_reg, (u32)insn[i].imm);
910 /* Adjust for two bpf instructions */
911 addrs[++i] = ctx->idx * 4;
917 case BPF_JMP | BPF_EXIT:
919 * If this isn't the very last instruction, branch to
920 * the epilogue. If we _are_ the last instruction,
921 * we'll just fall through to the epilogue.
925 /* else fall through to the epilogue */
929 * Call kernel helper or bpf function
931 case BPF_JMP | BPF_CALL:
932 ctx->seen |= SEEN_FUNC;
934 ret = bpf_jit_get_func_addr(fp, &insn[i], false,
935 &func_addr, &func_addr_fixed);
939 if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_5))) {
940 EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_5) - 1, _R1, 8));
941 EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_5), _R1, 12));
944 bpf_jit_emit_func_call_rel(image, ctx, func_addr);
946 EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_0) - 1, _R3));
947 EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_0), _R4));
953 case BPF_JMP | BPF_JA:
954 PPC_JMP(addrs[i + 1 + off]);
957 case BPF_JMP | BPF_JGT | BPF_K:
958 case BPF_JMP | BPF_JGT | BPF_X:
959 case BPF_JMP | BPF_JSGT | BPF_K:
960 case BPF_JMP | BPF_JSGT | BPF_X:
961 case BPF_JMP32 | BPF_JGT | BPF_K:
962 case BPF_JMP32 | BPF_JGT | BPF_X:
963 case BPF_JMP32 | BPF_JSGT | BPF_K:
964 case BPF_JMP32 | BPF_JSGT | BPF_X:
967 case BPF_JMP | BPF_JLT | BPF_K:
968 case BPF_JMP | BPF_JLT | BPF_X:
969 case BPF_JMP | BPF_JSLT | BPF_K:
970 case BPF_JMP | BPF_JSLT | BPF_X:
971 case BPF_JMP32 | BPF_JLT | BPF_K:
972 case BPF_JMP32 | BPF_JLT | BPF_X:
973 case BPF_JMP32 | BPF_JSLT | BPF_K:
974 case BPF_JMP32 | BPF_JSLT | BPF_X:
977 case BPF_JMP | BPF_JGE | BPF_K:
978 case BPF_JMP | BPF_JGE | BPF_X:
979 case BPF_JMP | BPF_JSGE | BPF_K:
980 case BPF_JMP | BPF_JSGE | BPF_X:
981 case BPF_JMP32 | BPF_JGE | BPF_K:
982 case BPF_JMP32 | BPF_JGE | BPF_X:
983 case BPF_JMP32 | BPF_JSGE | BPF_K:
984 case BPF_JMP32 | BPF_JSGE | BPF_X:
987 case BPF_JMP | BPF_JLE | BPF_K:
988 case BPF_JMP | BPF_JLE | BPF_X:
989 case BPF_JMP | BPF_JSLE | BPF_K:
990 case BPF_JMP | BPF_JSLE | BPF_X:
991 case BPF_JMP32 | BPF_JLE | BPF_K:
992 case BPF_JMP32 | BPF_JLE | BPF_X:
993 case BPF_JMP32 | BPF_JSLE | BPF_K:
994 case BPF_JMP32 | BPF_JSLE | BPF_X:
997 case BPF_JMP | BPF_JEQ | BPF_K:
998 case BPF_JMP | BPF_JEQ | BPF_X:
999 case BPF_JMP32 | BPF_JEQ | BPF_K:
1000 case BPF_JMP32 | BPF_JEQ | BPF_X:
1001 true_cond = COND_EQ;
1003 case BPF_JMP | BPF_JNE | BPF_K:
1004 case BPF_JMP | BPF_JNE | BPF_X:
1005 case BPF_JMP32 | BPF_JNE | BPF_K:
1006 case BPF_JMP32 | BPF_JNE | BPF_X:
1007 true_cond = COND_NE;
1009 case BPF_JMP | BPF_JSET | BPF_K:
1010 case BPF_JMP | BPF_JSET | BPF_X:
1011 case BPF_JMP32 | BPF_JSET | BPF_K:
1012 case BPF_JMP32 | BPF_JSET | BPF_X:
1013 true_cond = COND_NE;
1018 case BPF_JMP | BPF_JGT | BPF_X:
1019 case BPF_JMP | BPF_JLT | BPF_X:
1020 case BPF_JMP | BPF_JGE | BPF_X:
1021 case BPF_JMP | BPF_JLE | BPF_X:
1022 case BPF_JMP | BPF_JEQ | BPF_X:
1023 case BPF_JMP | BPF_JNE | BPF_X:
1024 /* unsigned comparison */
1025 EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h));
1026 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1027 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1029 case BPF_JMP32 | BPF_JGT | BPF_X:
1030 case BPF_JMP32 | BPF_JLT | BPF_X:
1031 case BPF_JMP32 | BPF_JGE | BPF_X:
1032 case BPF_JMP32 | BPF_JLE | BPF_X:
1033 case BPF_JMP32 | BPF_JEQ | BPF_X:
1034 case BPF_JMP32 | BPF_JNE | BPF_X:
1035 /* unsigned comparison */
1036 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1038 case BPF_JMP | BPF_JSGT | BPF_X:
1039 case BPF_JMP | BPF_JSLT | BPF_X:
1040 case BPF_JMP | BPF_JSGE | BPF_X:
1041 case BPF_JMP | BPF_JSLE | BPF_X:
1042 /* signed comparison */
1043 EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h));
1044 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1045 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1047 case BPF_JMP32 | BPF_JSGT | BPF_X:
1048 case BPF_JMP32 | BPF_JSLT | BPF_X:
1049 case BPF_JMP32 | BPF_JSGE | BPF_X:
1050 case BPF_JMP32 | BPF_JSLE | BPF_X:
1051 /* signed comparison */
1052 EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
1054 case BPF_JMP | BPF_JSET | BPF_X:
1055 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg_h, src_reg_h));
1056 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1057 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1059 case BPF_JMP32 | BPF_JSET | BPF_X: {
1060 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1062 case BPF_JMP | BPF_JNE | BPF_K:
1063 case BPF_JMP | BPF_JEQ | BPF_K:
1064 case BPF_JMP | BPF_JGT | BPF_K:
1065 case BPF_JMP | BPF_JLT | BPF_K:
1066 case BPF_JMP | BPF_JGE | BPF_K:
1067 case BPF_JMP | BPF_JLE | BPF_K:
1069 * Need sign-extended load, so only positive
1070 * values can be used as imm in cmplwi
1072 if (imm >= 0 && imm < 32768) {
1073 EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0));
1074 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1075 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1077 /* sign-extending load ... but unsigned comparison */
1079 EMIT(PPC_RAW_CMPLW(dst_reg_h, _R0));
1081 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1082 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1085 case BPF_JMP32 | BPF_JNE | BPF_K:
1086 case BPF_JMP32 | BPF_JEQ | BPF_K:
1087 case BPF_JMP32 | BPF_JGT | BPF_K:
1088 case BPF_JMP32 | BPF_JLT | BPF_K:
1089 case BPF_JMP32 | BPF_JGE | BPF_K:
1090 case BPF_JMP32 | BPF_JLE | BPF_K:
1091 if (imm >= 0 && imm < 65536) {
1092 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1095 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1099 case BPF_JMP | BPF_JSGT | BPF_K:
1100 case BPF_JMP | BPF_JSLT | BPF_K:
1101 case BPF_JMP | BPF_JSGE | BPF_K:
1102 case BPF_JMP | BPF_JSLE | BPF_K:
1103 if (imm >= 0 && imm < 65536) {
1104 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1105 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1106 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1108 /* sign-extending load */
1109 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1111 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1112 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1115 case BPF_JMP32 | BPF_JSGT | BPF_K:
1116 case BPF_JMP32 | BPF_JSLT | BPF_K:
1117 case BPF_JMP32 | BPF_JSGE | BPF_K:
1118 case BPF_JMP32 | BPF_JSLE | BPF_K:
1120 * signed comparison, so any 16-bit value
1121 * can be used in cmpwi
1123 if (imm >= -32768 && imm < 32768) {
1124 EMIT(PPC_RAW_CMPWI(dst_reg, imm));
1126 /* sign-extending load */
1128 EMIT(PPC_RAW_CMPW(dst_reg, _R0));
1131 case BPF_JMP | BPF_JSET | BPF_K:
1132 /* andi does not sign-extend the immediate */
1133 if (imm >= 0 && imm < 32768) {
1134 /* PPC_ANDI is _only/always_ dot-form */
1135 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1139 EMIT(PPC_RAW_CMPWI(dst_reg_h, 0));
1140 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1142 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1145 case BPF_JMP32 | BPF_JSET | BPF_K:
1146 /* andi does not sign-extend the immediate */
1147 if (imm >= 0 && imm < 32768) {
1148 /* PPC_ANDI is _only/always_ dot-form */
1149 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1152 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1156 PPC_BCC(true_cond, addrs[i + 1 + off]);
1162 case BPF_JMP | BPF_TAIL_CALL:
1163 ctx->seen |= SEEN_TAILCALL;
1164 ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
1171 * The filter contains something cruel & unusual.
1172 * We don't handle it, but also there shouldn't be
1173 * anything missing from our list.
1175 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i);
1178 if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext &&
1179 !insn_is_zext(&insn[i + 1]) && !(BPF_OP(code) == BPF_END && imm == 64))
1180 EMIT(PPC_RAW_LI(dst_reg_h, 0));
1183 /* Set end-of-body-code address for exit. */
1184 addrs[i] = ctx->idx * 4;