1 /* SPDX-License-Identifier: GPL-2.0 */
3 * Common functionality for RV32 and RV64 BPF JIT compilers
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
12 #include <linux/bpf.h>
13 #include <linux/filter.h>
14 #include <asm/cacheflush.h>
17 RV_REG_ZERO = 0, /* The constant value 0 */
18 RV_REG_RA = 1, /* Return address */
19 RV_REG_SP = 2, /* Stack pointer */
20 RV_REG_GP = 3, /* Global pointer */
21 RV_REG_TP = 4, /* Thread pointer */
22 RV_REG_T0 = 5, /* Temporaries */
25 RV_REG_FP = 8, /* Saved register/frame pointer */
26 RV_REG_S1 = 9, /* Saved register */
27 RV_REG_A0 = 10, /* Function argument/return values */
28 RV_REG_A1 = 11, /* Function arguments */
35 RV_REG_S2 = 18, /* Saved registers */
45 RV_REG_T3 = 28, /* Temporaries */
51 struct rv_jit_context {
52 struct bpf_prog *prog;
53 u32 *insns; /* RV insns */
56 int *offset; /* BPF to RV */
62 struct bpf_binary_header *header;
64 struct rv_jit_context ctx;
67 static inline void bpf_fill_ill_insns(void *area, unsigned int size)
69 memset(area, 0, size);
72 static inline void bpf_flush_icache(void *start, void *end)
74 flush_icache_range((unsigned long)start, (unsigned long)end);
77 static inline void emit(const u32 insn, struct rv_jit_context *ctx)
80 ctx->insns[ctx->ninsns] = insn;
85 static inline int epilogue_offset(struct rv_jit_context *ctx)
87 int to = ctx->epilogue_offset, from = ctx->ninsns;
89 return (to - from) << 2;
92 /* Return -1 or inverted cond. */
93 static inline int invert_bpf_cond(u8 cond)
120 static inline bool is_12b_int(long val)
122 return -(1L << 11) <= val && val < (1L << 11);
125 static inline int is_12b_check(int off, int insn)
127 if (!is_12b_int(off)) {
128 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
135 static inline bool is_13b_int(long val)
137 return -(1L << 12) <= val && val < (1L << 12);
140 static inline bool is_21b_int(long val)
142 return -(1L << 20) <= val && val < (1L << 20);
145 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx)
149 off++; /* BPF branch is from PC+1, RV is from PC */
150 from = (insn > 0) ? ctx->offset[insn - 1] : 0;
151 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0;
152 return (to - from) << 2;
155 /* Instruction formats. */
157 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd,
160 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
164 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
166 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
170 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
172 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
174 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
175 (imm4_0 << 7) | opcode;
178 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
180 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
181 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
183 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
184 (imm4_1 << 7) | opcode;
187 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
189 return (imm31_12 << 12) | (rd << 7) | opcode;
192 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode)
196 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
197 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
199 return (imm << 12) | (rd << 7) | opcode;
202 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
203 u8 funct3, u8 rd, u8 opcode)
205 u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
207 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
210 /* Instructions shared by both RV32 and RV64. */
212 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
214 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
217 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
219 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
222 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
224 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
227 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
229 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
232 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
234 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
237 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
239 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
242 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
244 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
247 static inline u32 rv_lui(u8 rd, u32 imm31_12)
249 return rv_u_insn(imm31_12, rd, 0x37);
252 static inline u32 rv_auipc(u8 rd, u32 imm31_12)
254 return rv_u_insn(imm31_12, rd, 0x17);
257 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2)
259 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
262 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
264 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
267 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2)
269 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33);
272 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2)
274 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
277 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2)
279 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
282 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
284 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
287 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
289 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
292 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
294 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
297 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
299 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
302 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
304 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
307 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
309 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
312 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
314 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
317 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
319 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
322 static inline u32 rv_jal(u8 rd, u32 imm20_1)
324 return rv_j_insn(imm20_1, rd, 0x6f);
327 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
329 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
332 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
334 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63);
337 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
339 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63);
342 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
344 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63);
347 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1)
349 return rv_bltu(rs2, rs1, imm12_1);
352 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
354 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63);
357 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1)
359 return rv_bgeu(rs2, rs1, imm12_1);
362 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
364 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63);
367 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1)
369 return rv_blt(rs2, rs1, imm12_1);
372 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
374 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63);
377 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
379 return rv_bge(rs2, rs1, imm12_1);
382 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
384 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
387 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
389 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
392 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
394 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
397 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
399 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
402 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
404 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
407 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
409 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
412 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
414 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
418 * RV64-only instructions.
420 * These instructions are not available on RV32. Wrap them below a #if to
421 * ensure that the RV32 JIT doesn't emit any of these instructions.
424 #if __riscv_xlen == 64
426 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
428 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
431 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
433 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
436 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
438 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
441 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
443 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
446 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
448 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
451 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
453 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
456 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
458 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
461 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
463 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
466 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
468 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
471 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
473 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
476 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
478 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
481 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
483 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
486 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
488 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
491 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
493 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
496 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
498 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
501 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
503 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
506 #endif /* __riscv_xlen == 64 */
508 void bpf_jit_build_prologue(struct rv_jit_context *ctx);
509 void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
511 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
514 #endif /* _BPF_JIT_H */