arch/x86/net/bpf_jit_comp.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * bpf_jit_comp.c: BPF JIT compiler
   4  *
   5  * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com)
   6  * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   7  */
   8 #include <linux/netdevice.h>
   9 #include <linux/filter.h>
  10 #include <linux/if_vlan.h>
  11 #include <linux/bpf.h>
  12
  13 #include <asm/set_memory.h>
  14 #include <asm/nospec-branch.h>
  15
  16 static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
  17 {
  18         if (len == 1)
  19                 *ptr = bytes;
  20         else if (len == 2)
  21                 *(u16 *)ptr = bytes;
  22         else {
  23                 *(u32 *)ptr = bytes;
  24                 barrier();
  25         }
  26         return ptr + len;
  27 }
  28
  29 #define EMIT(bytes, len) \
  30         do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
  31
  32 #define EMIT1(b1)               EMIT(b1, 1)
  33 #define EMIT2(b1, b2)           EMIT((b1) + ((b2) << 8), 2)
  34 #define EMIT3(b1, b2, b3)       EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
  35 #define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
  36
  37 #define EMIT1_off32(b1, off) \
  38         do { EMIT1(b1); EMIT(off, 4); } while (0)
  39 #define EMIT2_off32(b1, b2, off) \
  40         do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
  41 #define EMIT3_off32(b1, b2, b3, off) \
  42         do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
  43 #define EMIT4_off32(b1, b2, b3, b4, off) \
  44         do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
  45
  46 static bool is_imm8(int value)
  47 {
  48         return value <= 127 && value >= -128;
  49 }
  50
  51 static bool is_simm32(s64 value)
  52 {
  53         return value == (s64)(s32)value;
  54 }
  55
  56 static bool is_uimm32(u64 value)
  57 {
  58         return value == (u64)(u32)value;
  59 }
  60
  61 /* mov dst, src */
  62 #define EMIT_mov(DST, SRC)                                                               \
  63         do {                                                                             \
  64                 if (DST != SRC)                                                          \
  65                         EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
  66         } while (0)
  67
  68 static int bpf_size_to_x86_bytes(int bpf_size)
  69 {
  70         if (bpf_size == BPF_W)
  71                 return 4;
  72         else if (bpf_size == BPF_H)
  73                 return 2;
  74         else if (bpf_size == BPF_B)
  75                 return 1;
  76         else if (bpf_size == BPF_DW)
  77                 return 4; /* imm32 */
  78         else
  79                 return 0;
  80 }
  81
  82 /*
  83  * List of x86 cond jumps opcodes (. + s8)
  84  * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
  85  */
  86 #define X86_JB  0x72
  87 #define X86_JAE 0x73
  88 #define X86_JE  0x74
  89 #define X86_JNE 0x75
  90 #define X86_JBE 0x76
  91 #define X86_JA  0x77
  92 #define X86_JL  0x7C
  93 #define X86_JGE 0x7D
  94 #define X86_JLE 0x7E
  95 #define X86_JG  0x7F
  96
  97 /* Pick a register outside of BPF range for JIT internal work */
  98 #define AUX_REG (MAX_BPF_JIT_REG + 1)
  99
 100 /*
 101  * The following table maps BPF registers to x86-64 registers.
 102  *
 103  * x86-64 register R12 is unused, since if used as base address
 104  * register in load/store instructions, it always needs an
 105  * extra byte of encoding and is callee saved.
 106  *
 107  * Also x86-64 register R9 is unused. x86-64 register R10 is
 108  * used for blinding (if enabled).
 109  */
 110 static const int reg2hex[] = {
 111         [BPF_REG_0] = 0,  /* RAX */
 112         [BPF_REG_1] = 7,  /* RDI */
 113         [BPF_REG_2] = 6,  /* RSI */
 114         [BPF_REG_3] = 2,  /* RDX */
 115         [BPF_REG_4] = 1,  /* RCX */
 116         [BPF_REG_5] = 0,  /* R8  */
 117         [BPF_REG_6] = 3,  /* RBX callee saved */
 118         [BPF_REG_7] = 5,  /* R13 callee saved */
 119         [BPF_REG_8] = 6,  /* R14 callee saved */
 120         [BPF_REG_9] = 7,  /* R15 callee saved */
 121         [BPF_REG_FP] = 5, /* RBP readonly */
 122         [BPF_REG_AX] = 2, /* R10 temp register */
 123         [AUX_REG] = 3,    /* R11 temp register */
 124 };
 125
 126 /*
 127  * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15
 128  * which need extra byte of encoding.
 129  * rax,rcx,...,rbp have simpler encoding
 130  */
 131 static bool is_ereg(u32 reg)
 132 {
 133         return (1 << reg) & (BIT(BPF_REG_5) |
 134                              BIT(AUX_REG) |
 135                              BIT(BPF_REG_7) |
 136                              BIT(BPF_REG_8) |
 137                              BIT(BPF_REG_9) |
 138                              BIT(BPF_REG_AX));
 139 }
 140
 141 static bool is_axreg(u32 reg)
 142 {
 143         return reg == BPF_REG_0;
 144 }
 145
 146 /* Add modifiers if 'reg' maps to x86-64 registers R8..R15 */
 147 static u8 add_1mod(u8 byte, u32 reg)
 148 {
 149         if (is_ereg(reg))
 150                 byte |= 1;
 151         return byte;
 152 }
 153
 154 static u8 add_2mod(u8 byte, u32 r1, u32 r2)
 155 {
 156         if (is_ereg(r1))
 157                 byte |= 1;
 158         if (is_ereg(r2))
 159                 byte |= 4;
 160         return byte;
 161 }
 162
 163 /* Encode 'dst_reg' register into x86-64 opcode 'byte' */
 164 static u8 add_1reg(u8 byte, u32 dst_reg)
 165 {
 166         return byte + reg2hex[dst_reg];
 167 }
 168
 169 /* Encode 'dst_reg' and 'src_reg' registers into x86-64 opcode 'byte' */
 170 static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)
 171 {
 172         return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3);
 173 }
 174
 175 static void jit_fill_hole(void *area, unsigned int size)
 176 {
 177         /* Fill whole space with INT3 instructions */
 178         memset(area, 0xcc, size);
 179 }
 180
 181 struct jit_context {
 182         int cleanup_addr; /* Epilogue code offset */
 183 };
 184
 185 /* Maximum number of bytes emitted while JITing one eBPF insn */
 186 #define BPF_MAX_INSN_SIZE       128
 187 #define BPF_INSN_SAFETY         64
 188
 189 #define AUX_STACK_SPACE         40 /* Space for RBX, R13, R14, R15, tailcnt */
 190
 191 #define PROLOGUE_SIZE           37
 192
 193 /*
 194  * Emit x86-64 prologue code for BPF program and check its size.
 195  * bpf_tail_call helper will skip it while jumping into another program
 196  */
 197 static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
 198 {
 199         u8 *prog = *pprog;
 200         int cnt = 0;
 201
 202         /* push rbp */
 203         EMIT1(0x55);
 204
 205         /* mov rbp,rsp */
 206         EMIT3(0x48, 0x89, 0xE5);
 207
 208         /* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */
 209         EMIT3_off32(0x48, 0x81, 0xEC,
 210                     round_up(stack_depth, 8) + AUX_STACK_SPACE);
 211
 212         /* sub rbp, AUX_STACK_SPACE */
 213         EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE);
 214
 215         /* mov qword ptr [rbp+0],rbx */
 216         EMIT4(0x48, 0x89, 0x5D, 0);
 217         /* mov qword ptr [rbp+8],r13 */
 218         EMIT4(0x4C, 0x89, 0x6D, 8);
 219         /* mov qword ptr [rbp+16],r14 */
 220         EMIT4(0x4C, 0x89, 0x75, 16);
 221         /* mov qword ptr [rbp+24],r15 */
 222         EMIT4(0x4C, 0x89, 0x7D, 24);
 223
 224         if (!ebpf_from_cbpf) {
 225                 /*
 226                  * Clear the tail call counter (tail_call_cnt): for eBPF tail
 227                  * calls we need to reset the counter to 0. It's done in two
 228                  * instructions, resetting RAX register to 0, and moving it
 229                  * to the counter location.
 230                  */
 231
 232                 /* xor eax, eax */
 233                 EMIT2(0x31, 0xc0);
 234                 /* mov qword ptr [rbp+32], rax */
 235                 EMIT4(0x48, 0x89, 0x45, 32);
 236
 237                 BUILD_BUG_ON(cnt != PROLOGUE_SIZE);
 238         }
 239
 240         *pprog = prog;
 241 }
 242
 243 /*
 244  * Generate the following code:
 245  *
 246  * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ...
 247  *   if (index >= array->map.max_entries)
 248  *     goto out;
 249  *   if (++tail_call_cnt > MAX_TAIL_CALL_CNT)
 250  *     goto out;
 251  *   prog = array->ptrs[index];
 252  *   if (prog == NULL)
 253  *     goto out;
 254  *   goto *(prog->bpf_func + prologue_size);
 255  * out:
 256  */
 257 static void emit_bpf_tail_call(u8 **pprog)
 258 {
 259         u8 *prog = *pprog;
 260         int label1, label2, label3;
 261         int cnt = 0;
 262
 263         /*
 264          * rdi - pointer to ctx
 265          * rsi - pointer to bpf_array
 266          * rdx - index in bpf_array
 267          */
 268
 269         /*
 270          * if (index >= array->map.max_entries)
 271          *      goto out;
 272          */
 273         EMIT2(0x89, 0xD2);                        /* mov edx, edx */
 274         EMIT3(0x39, 0x56,                         /* cmp dword ptr [rsi + 16], edx */
 275               offsetof(struct bpf_array, map.max_entries));
 276 #define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */
 277         EMIT2(X86_JBE, OFFSET1);                  /* jbe out */
 278         label1 = cnt;
 279
 280         /*
 281          * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
 282          *      goto out;
 283          */
 284         EMIT2_off32(0x8B, 0x85, 36);              /* mov eax, dword ptr [rbp + 36] */
 285         EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT);     /* cmp eax, MAX_TAIL_CALL_CNT */
 286 #define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE)
 287         EMIT2(X86_JA, OFFSET2);                   /* ja out */
 288         label2 = cnt;
 289         EMIT3(0x83, 0xC0, 0x01);                  /* add eax, 1 */
 290         EMIT2_off32(0x89, 0x85, 36);              /* mov dword ptr [rbp + 36], eax */
 291
 292         /* prog = array->ptrs[index]; */
 293         EMIT4_off32(0x48, 0x8B, 0x84, 0xD6,       /* mov rax, [rsi + rdx * 8 + offsetof(...)] */
 294                     offsetof(struct bpf_array, ptrs));
 295
 296         /*
 297          * if (prog == NULL)
 298          *      goto out;
 299          */
 300         EMIT3(0x48, 0x85, 0xC0);                  /* test rax,rax */
 301 #define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE)
 302         EMIT2(X86_JE, OFFSET3);                   /* je out */
 303         label3 = cnt;
 304
 305         /* goto *(prog->bpf_func + prologue_size); */
 306         EMIT4(0x48, 0x8B, 0x40,                   /* mov rax, qword ptr [rax + 32] */
 307               offsetof(struct bpf_prog, bpf_func));
 308         EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE);   /* add rax, prologue_size */
 309
 310         /*
 311          * Wow we're ready to jump into next BPF program
 312          * rdi == ctx (1st arg)
 313          * rax == prog->bpf_func + prologue_size
 314          */
 315         RETPOLINE_RAX_BPF_JIT();
 316
 317         /* out: */
 318         BUILD_BUG_ON(cnt - label1 != OFFSET1);
 319         BUILD_BUG_ON(cnt - label2 != OFFSET2);
 320         BUILD_BUG_ON(cnt - label3 != OFFSET3);
 321         *pprog = prog;
 322 }
 323
 324 static void emit_mov_imm32(u8 **pprog, bool sign_propagate,
 325                            u32 dst_reg, const u32 imm32)
 326 {
 327         u8 *prog = *pprog;
 328         u8 b1, b2, b3;
 329         int cnt = 0;
 330
 331         /*
 332          * Optimization: if imm32 is positive, use 'mov %eax, imm32'
 333          * (which zero-extends imm32) to save 2 bytes.
 334          */
 335         if (sign_propagate && (s32)imm32 < 0) {
 336                 /* 'mov %rax, imm32' sign extends imm32 */
 337                 b1 = add_1mod(0x48, dst_reg);
 338                 b2 = 0xC7;
 339                 b3 = 0xC0;
 340                 EMIT3_off32(b1, b2, add_1reg(b3, dst_reg), imm32);
 341                 goto done;
 342         }
 343
 344         /*
 345          * Optimization: if imm32 is zero, use 'xor %eax, %eax'
 346          * to save 3 bytes.
 347          */
 348         if (imm32 == 0) {
 349                 if (is_ereg(dst_reg))
 350                         EMIT1(add_2mod(0x40, dst_reg, dst_reg));
 351                 b2 = 0x31; /* xor */
 352                 b3 = 0xC0;
 353                 EMIT2(b2, add_2reg(b3, dst_reg, dst_reg));
 354                 goto done;
 355         }
 356
 357         /* mov %eax, imm32 */
 358         if (is_ereg(dst_reg))
 359                 EMIT1(add_1mod(0x40, dst_reg));
 360         EMIT1_off32(add_1reg(0xB8, dst_reg), imm32);
 361 done:
 362         *pprog = prog;
 363 }
 364
 365 static void emit_mov_imm64(u8 **pprog, u32 dst_reg,
 366                            const u32 imm32_hi, const u32 imm32_lo)
 367 {
 368         u8 *prog = *pprog;
 369         int cnt = 0;
 370
 371         if (is_uimm32(((u64)imm32_hi << 32) | (u32)imm32_lo)) {
 372                 /*
 373                  * For emitting plain u32, where sign bit must not be
 374                  * propagated LLVM tends to load imm64 over mov32
 375                  * directly, so save couple of bytes by just doing
 376                  * 'mov %eax, imm32' instead.
 377                  */
 378                 emit_mov_imm32(&prog, false, dst_reg, imm32_lo);
 379         } else {
 380                 /* movabsq %rax, imm64 */
 381                 EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg));
 382                 EMIT(imm32_lo, 4);
 383                 EMIT(imm32_hi, 4);
 384         }
 385
 386         *pprog = prog;
 387 }
 388
 389 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)
 390 {
 391         u8 *prog = *pprog;
 392         int cnt = 0;
 393
 394         if (is64) {
 395                 /* mov dst, src */
 396                 EMIT_mov(dst_reg, src_reg);
 397         } else {
 398                 /* mov32 dst, src */
 399                 if (is_ereg(dst_reg) || is_ereg(src_reg))
 400                         EMIT1(add_2mod(0x40, dst_reg, src_reg));
 401                 EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg));
 402         }
 403
 404         *pprog = prog;
 405 }
 406
 407 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 408                   int oldproglen, struct jit_context *ctx)
 409 {
 410         struct bpf_insn *insn = bpf_prog->insnsi;
 411         int insn_cnt = bpf_prog->len;
 412         bool seen_exit = false;
 413         u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
 414         int i, cnt = 0;
 415         int proglen = 0;
 416         u8 *prog = temp;
 417
 418         emit_prologue(&prog, bpf_prog->aux->stack_depth,
 419                       bpf_prog_was_classic(bpf_prog));
 420
 421         for (i = 0; i < insn_cnt; i++, insn++) {
 422                 const s32 imm32 = insn->imm;
 423                 u32 dst_reg = insn->dst_reg;
 424                 u32 src_reg = insn->src_reg;
 425                 u8 b2 = 0, b3 = 0;
 426                 s64 jmp_offset;
 427                 u8 jmp_cond;
 428                 int ilen;
 429                 u8 *func;
 430
 431                 switch (insn->code) {
 432                         /* ALU */
 433                 case BPF_ALU | BPF_ADD | BPF_X:
 434                 case BPF_ALU | BPF_SUB | BPF_X:
 435                 case BPF_ALU | BPF_AND | BPF_X:
 436                 case BPF_ALU | BPF_OR | BPF_X:
 437                 case BPF_ALU | BPF_XOR | BPF_X:
 438                 case BPF_ALU64 | BPF_ADD | BPF_X:
 439                 case BPF_ALU64 | BPF_SUB | BPF_X:
 440                 case BPF_ALU64 | BPF_AND | BPF_X:
 441                 case BPF_ALU64 | BPF_OR | BPF_X:
 442                 case BPF_ALU64 | BPF_XOR | BPF_X:
 443                         switch (BPF_OP(insn->code)) {
 444                         case BPF_ADD: b2 = 0x01; break;
 445                         case BPF_SUB: b2 = 0x29; break;
 446                         case BPF_AND: b2 = 0x21; break;
 447                         case BPF_OR: b2 = 0x09; break;
 448                         case BPF_XOR: b2 = 0x31; break;
 449                         }
 450                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 451                                 EMIT1(add_2mod(0x48, dst_reg, src_reg));
 452                         else if (is_ereg(dst_reg) || is_ereg(src_reg))
 453                                 EMIT1(add_2mod(0x40, dst_reg, src_reg));
 454                         EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));
 455                         break;
 456
 457                 case BPF_ALU64 | BPF_MOV | BPF_X:
 458                 case BPF_ALU | BPF_MOV | BPF_X:
 459                         emit_mov_reg(&prog,
 460                                      BPF_CLASS(insn->code) == BPF_ALU64,
 461                                      dst_reg, src_reg);
 462                         break;
 463
 464                         /* neg dst */
 465                 case BPF_ALU | BPF_NEG:
 466                 case BPF_ALU64 | BPF_NEG:
 467                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 468                                 EMIT1(add_1mod(0x48, dst_reg));
 469                         else if (is_ereg(dst_reg))
 470                                 EMIT1(add_1mod(0x40, dst_reg));
 471                         EMIT2(0xF7, add_1reg(0xD8, dst_reg));
 472                         break;
 473
 474                 case BPF_ALU | BPF_ADD | BPF_K:
 475                 case BPF_ALU | BPF_SUB | BPF_K:
 476                 case BPF_ALU | BPF_AND | BPF_K:
 477                 case BPF_ALU | BPF_OR | BPF_K:
 478                 case BPF_ALU | BPF_XOR | BPF_K:
 479                 case BPF_ALU64 | BPF_ADD | BPF_K:
 480                 case BPF_ALU64 | BPF_SUB | BPF_K:
 481                 case BPF_ALU64 | BPF_AND | BPF_K:
 482                 case BPF_ALU64 | BPF_OR | BPF_K:
 483                 case BPF_ALU64 | BPF_XOR | BPF_K:
 484                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 485                                 EMIT1(add_1mod(0x48, dst_reg));
 486                         else if (is_ereg(dst_reg))
 487                                 EMIT1(add_1mod(0x40, dst_reg));
 488
 489                         /*
 490                          * b3 holds 'normal' opcode, b2 short form only valid
 491                          * in case dst is eax/rax.
 492                          */
 493                         switch (BPF_OP(insn->code)) {
 494                         case BPF_ADD:
 495                                 b3 = 0xC0;
 496                                 b2 = 0x05;
 497                                 break;
 498                         case BPF_SUB:
 499                                 b3 = 0xE8;
 500                                 b2 = 0x2D;
 501                                 break;
 502                         case BPF_AND:
 503                                 b3 = 0xE0;
 504                                 b2 = 0x25;
 505                                 break;
 506                         case BPF_OR:
 507                                 b3 = 0xC8;
 508                                 b2 = 0x0D;
 509                                 break;
 510                         case BPF_XOR:
 511                                 b3 = 0xF0;
 512                                 b2 = 0x35;
 513                                 break;
 514                         }
 515
 516                         if (is_imm8(imm32))
 517                                 EMIT3(0x83, add_1reg(b3, dst_reg), imm32);
 518                         else if (is_axreg(dst_reg))
 519                                 EMIT1_off32(b2, imm32);
 520                         else
 521                                 EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32);
 522                         break;
 523
 524                 case BPF_ALU64 | BPF_MOV | BPF_K:
 525                 case BPF_ALU | BPF_MOV | BPF_K:
 526                         emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64,
 527                                        dst_reg, imm32);
 528                         break;
 529
 530                 case BPF_LD | BPF_IMM | BPF_DW:
 531                         emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm);
 532                         insn++;
 533                         i++;
 534                         break;
 535
 536                         /* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */
 537                 case BPF_ALU | BPF_MOD | BPF_X:
 538                 case BPF_ALU | BPF_DIV | BPF_X:
 539                 case BPF_ALU | BPF_MOD | BPF_K:
 540                 case BPF_ALU | BPF_DIV | BPF_K:
 541                 case BPF_ALU64 | BPF_MOD | BPF_X:
 542                 case BPF_ALU64 | BPF_DIV | BPF_X:
 543                 case BPF_ALU64 | BPF_MOD | BPF_K:
 544                 case BPF_ALU64 | BPF_DIV | BPF_K:
 545                         EMIT1(0x50); /* push rax */
 546                         EMIT1(0x52); /* push rdx */
 547
 548                         if (BPF_SRC(insn->code) == BPF_X)
 549                                 /* mov r11, src_reg */
 550                                 EMIT_mov(AUX_REG, src_reg);
 551                         else
 552                                 /* mov r11, imm32 */
 553                                 EMIT3_off32(0x49, 0xC7, 0xC3, imm32);
 554
 555                         /* mov rax, dst_reg */
 556                         EMIT_mov(BPF_REG_0, dst_reg);
 557
 558                         /*
 559                          * xor edx, edx
 560                          * equivalent to 'xor rdx, rdx', but one byte less
 561                          */
 562                         EMIT2(0x31, 0xd2);
 563
 564                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 565                                 /* div r11 */
 566                                 EMIT3(0x49, 0xF7, 0xF3);
 567                         else
 568                                 /* div r11d */
 569                                 EMIT3(0x41, 0xF7, 0xF3);
 570
 571                         if (BPF_OP(insn->code) == BPF_MOD)
 572                                 /* mov r11, rdx */
 573                                 EMIT3(0x49, 0x89, 0xD3);
 574                         else
 575                                 /* mov r11, rax */
 576                                 EMIT3(0x49, 0x89, 0xC3);
 577
 578                         EMIT1(0x5A); /* pop rdx */
 579                         EMIT1(0x58); /* pop rax */
 580
 581                         /* mov dst_reg, r11 */
 582                         EMIT_mov(dst_reg, AUX_REG);
 583                         break;
 584
 585                 case BPF_ALU | BPF_MUL | BPF_K:
 586                 case BPF_ALU | BPF_MUL | BPF_X:
 587                 case BPF_ALU64 | BPF_MUL | BPF_K:
 588                 case BPF_ALU64 | BPF_MUL | BPF_X:
 589                 {
 590                         bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
 591
 592                         if (dst_reg != BPF_REG_0)
 593                                 EMIT1(0x50); /* push rax */
 594                         if (dst_reg != BPF_REG_3)
 595                                 EMIT1(0x52); /* push rdx */
 596
 597                         /* mov r11, dst_reg */
 598                         EMIT_mov(AUX_REG, dst_reg);
 599
 600                         if (BPF_SRC(insn->code) == BPF_X)
 601                                 emit_mov_reg(&prog, is64, BPF_REG_0, src_reg);
 602                         else
 603                                 emit_mov_imm32(&prog, is64, BPF_REG_0, imm32);
 604
 605                         if (is64)
 606                                 EMIT1(add_1mod(0x48, AUX_REG));
 607                         else if (is_ereg(AUX_REG))
 608                                 EMIT1(add_1mod(0x40, AUX_REG));
 609                         /* mul(q) r11 */
 610                         EMIT2(0xF7, add_1reg(0xE0, AUX_REG));
 611
 612                         if (dst_reg != BPF_REG_3)
 613                                 EMIT1(0x5A); /* pop rdx */
 614                         if (dst_reg != BPF_REG_0) {
 615                                 /* mov dst_reg, rax */
 616                                 EMIT_mov(dst_reg, BPF_REG_0);
 617                                 EMIT1(0x58); /* pop rax */
 618                         }
 619                         break;
 620                 }
 621                         /* Shifts */
 622                 case BPF_ALU | BPF_LSH | BPF_K:
 623                 case BPF_ALU | BPF_RSH | BPF_K:
 624                 case BPF_ALU | BPF_ARSH | BPF_K:
 625                 case BPF_ALU64 | BPF_LSH | BPF_K:
 626                 case BPF_ALU64 | BPF_RSH | BPF_K:
 627                 case BPF_ALU64 | BPF_ARSH | BPF_K:
 628                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 629                                 EMIT1(add_1mod(0x48, dst_reg));
 630                         else if (is_ereg(dst_reg))
 631                                 EMIT1(add_1mod(0x40, dst_reg));
 632
 633                         switch (BPF_OP(insn->code)) {
 634                         case BPF_LSH: b3 = 0xE0; break;
 635                         case BPF_RSH: b3 = 0xE8; break;
 636                         case BPF_ARSH: b3 = 0xF8; break;
 637                         }
 638
 639                         if (imm32 == 1)
 640                                 EMIT2(0xD1, add_1reg(b3, dst_reg));
 641                         else
 642                                 EMIT3(0xC1, add_1reg(b3, dst_reg), imm32);
 643                         break;
 644
 645                 case BPF_ALU | BPF_LSH | BPF_X:
 646                 case BPF_ALU | BPF_RSH | BPF_X:
 647                 case BPF_ALU | BPF_ARSH | BPF_X:
 648                 case BPF_ALU64 | BPF_LSH | BPF_X:
 649                 case BPF_ALU64 | BPF_RSH | BPF_X:
 650                 case BPF_ALU64 | BPF_ARSH | BPF_X:
 651
 652                         /* Check for bad case when dst_reg == rcx */
 653                         if (dst_reg == BPF_REG_4) {
 654                                 /* mov r11, dst_reg */
 655                                 EMIT_mov(AUX_REG, dst_reg);
 656                                 dst_reg = AUX_REG;
 657                         }
 658
 659                         if (src_reg != BPF_REG_4) { /* common case */
 660                                 EMIT1(0x51); /* push rcx */
 661
 662                                 /* mov rcx, src_reg */
 663                                 EMIT_mov(BPF_REG_4, src_reg);
 664                         }
 665
 666                         /* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
 667                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 668                                 EMIT1(add_1mod(0x48, dst_reg));
 669                         else if (is_ereg(dst_reg))
 670                                 EMIT1(add_1mod(0x40, dst_reg));
 671
 672                         switch (BPF_OP(insn->code)) {
 673                         case BPF_LSH: b3 = 0xE0; break;
 674                         case BPF_RSH: b3 = 0xE8; break;
 675                         case BPF_ARSH: b3 = 0xF8; break;
 676                         }
 677                         EMIT2(0xD3, add_1reg(b3, dst_reg));
 678
 679                         if (src_reg != BPF_REG_4)
 680                                 EMIT1(0x59); /* pop rcx */
 681
 682                         if (insn->dst_reg == BPF_REG_4)
 683                                 /* mov dst_reg, r11 */
 684                                 EMIT_mov(insn->dst_reg, AUX_REG);
 685                         break;
 686
 687                 case BPF_ALU | BPF_END | BPF_FROM_BE:
 688                         switch (imm32) {
 689                         case 16:
 690                                 /* Emit 'ror %ax, 8' to swap lower 2 bytes */
 691                                 EMIT1(0x66);
 692                                 if (is_ereg(dst_reg))
 693                                         EMIT1(0x41);
 694                                 EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8);
 695
 696                                 /* Emit 'movzwl eax, ax' */
 697                                 if (is_ereg(dst_reg))
 698                                         EMIT3(0x45, 0x0F, 0xB7);
 699                                 else
 700                                         EMIT2(0x0F, 0xB7);
 701                                 EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
 702                                 break;
 703                         case 32:
 704                                 /* Emit 'bswap eax' to swap lower 4 bytes */
 705                                 if (is_ereg(dst_reg))
 706                                         EMIT2(0x41, 0x0F);
 707                                 else
 708                                         EMIT1(0x0F);
 709                                 EMIT1(add_1reg(0xC8, dst_reg));
 710                                 break;
 711                         case 64:
 712                                 /* Emit 'bswap rax' to swap 8 bytes */
 713                                 EMIT3(add_1mod(0x48, dst_reg), 0x0F,
 714                                       add_1reg(0xC8, dst_reg));
 715                                 break;
 716                         }
 717                         break;
 718
 719                 case BPF_ALU | BPF_END | BPF_FROM_LE:
 720                         switch (imm32) {
 721                         case 16:
 722                                 /*
 723                                  * Emit 'movzwl eax, ax' to zero extend 16-bit
 724                                  * into 64 bit
 725                                  */
 726                                 if (is_ereg(dst_reg))
 727                                         EMIT3(0x45, 0x0F, 0xB7);
 728                                 else
 729                                         EMIT2(0x0F, 0xB7);
 730                                 EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
 731                                 break;
 732                         case 32:
 733                                 /* Emit 'mov eax, eax' to clear upper 32-bits */
 734                                 if (is_ereg(dst_reg))
 735                                         EMIT1(0x45);
 736                                 EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg));
 737                                 break;
 738                         case 64:
 739                                 /* nop */
 740                                 break;
 741                         }
 742                         break;
 743
 744                         /* ST: *(u8*)(dst_reg + off) = imm */
 745                 case BPF_ST | BPF_MEM | BPF_B:
 746                         if (is_ereg(dst_reg))
 747                                 EMIT2(0x41, 0xC6);
 748                         else
 749                                 EMIT1(0xC6);
 750                         goto st;
 751                 case BPF_ST | BPF_MEM | BPF_H:
 752                         if (is_ereg(dst_reg))
 753                                 EMIT3(0x66, 0x41, 0xC7);
 754                         else
 755                                 EMIT2(0x66, 0xC7);
 756                         goto st;
 757                 case BPF_ST | BPF_MEM | BPF_W:
 758                         if (is_ereg(dst_reg))
 759                                 EMIT2(0x41, 0xC7);
 760                         else
 761                                 EMIT1(0xC7);
 762                         goto st;
 763                 case BPF_ST | BPF_MEM | BPF_DW:
 764                         EMIT2(add_1mod(0x48, dst_reg), 0xC7);
 765
 766 st:                     if (is_imm8(insn->off))
 767                                 EMIT2(add_1reg(0x40, dst_reg), insn->off);
 768                         else
 769                                 EMIT1_off32(add_1reg(0x80, dst_reg), insn->off);
 770
 771                         EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
 772                         break;
 773
 774                         /* STX: *(u8*)(dst_reg + off) = src_reg */
 775                 case BPF_STX | BPF_MEM | BPF_B:
 776                         /* Emit 'mov byte ptr [rax + off], al' */
 777                         if (is_ereg(dst_reg) || is_ereg(src_reg) ||
 778                             /* We have to add extra byte for x86 SIL, DIL regs */
 779                             src_reg == BPF_REG_1 || src_reg == BPF_REG_2)
 780                                 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);
 781                         else
 782                                 EMIT1(0x88);
 783                         goto stx;
 784                 case BPF_STX | BPF_MEM | BPF_H:
 785                         if (is_ereg(dst_reg) || is_ereg(src_reg))
 786                                 EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);
 787                         else
 788                                 EMIT2(0x66, 0x89);
 789                         goto stx;
 790                 case BPF_STX | BPF_MEM | BPF_W:
 791                         if (is_ereg(dst_reg) || is_ereg(src_reg))
 792                                 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);
 793                         else
 794                                 EMIT1(0x89);
 795                         goto stx;
 796                 case BPF_STX | BPF_MEM | BPF_DW:
 797                         EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);
 798 stx:                    if (is_imm8(insn->off))
 799                                 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
 800                         else
 801                                 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
 802                                             insn->off);
 803                         break;
 804
 805                         /* LDX: dst_reg = *(u8*)(src_reg + off) */
 806                 case BPF_LDX | BPF_MEM | BPF_B:
 807                         /* Emit 'movzx rax, byte ptr [rax + off]' */
 808                         EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);
 809                         goto ldx;
 810                 case BPF_LDX | BPF_MEM | BPF_H:
 811                         /* Emit 'movzx rax, word ptr [rax + off]' */
 812                         EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);
 813                         goto ldx;
 814                 case BPF_LDX | BPF_MEM | BPF_W:
 815                         /* Emit 'mov eax, dword ptr [rax+0x14]' */
 816                         if (is_ereg(dst_reg) || is_ereg(src_reg))
 817                                 EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);
 818                         else
 819                                 EMIT1(0x8B);
 820                         goto ldx;
 821                 case BPF_LDX | BPF_MEM | BPF_DW:
 822                         /* Emit 'mov rax, qword ptr [rax+0x14]' */
 823                         EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);
 824 ldx:                    /*
 825                          * If insn->off == 0 we can save one extra byte, but
 826                          * special case of x86 R13 which always needs an offset
 827                          * is not worth the hassle
 828                          */
 829                         if (is_imm8(insn->off))
 830                                 EMIT2(add_2reg(0x40, src_reg, dst_reg), insn->off);
 831                         else
 832                                 EMIT1_off32(add_2reg(0x80, src_reg, dst_reg),
 833                                             insn->off);
 834                         break;
 835
 836                         /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */
 837                 case BPF_STX | BPF_XADD | BPF_W:
 838                         /* Emit 'lock add dword ptr [rax + off], eax' */
 839                         if (is_ereg(dst_reg) || is_ereg(src_reg))
 840                                 EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01);
 841                         else
 842                                 EMIT2(0xF0, 0x01);
 843                         goto xadd;
 844                 case BPF_STX | BPF_XADD | BPF_DW:
 845                         EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01);
 846 xadd:                   if (is_imm8(insn->off))
 847                                 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
 848                         else
 849                                 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
 850                                             insn->off);
 851                         break;
 852
 853                         /* call */
 854                 case BPF_JMP | BPF_CALL:
 855                         func = (u8 *) __bpf_call_base + imm32;
 856                         jmp_offset = func - (image + addrs[i]);
 857                         if (!imm32 || !is_simm32(jmp_offset)) {
 858                                 pr_err("unsupported BPF func %d addr %p image %p\n",
 859                                        imm32, func, image);
 860                                 return -EINVAL;
 861                         }
 862                         EMIT1_off32(0xE8, jmp_offset);
 863                         break;
 864
 865                 case BPF_JMP | BPF_TAIL_CALL:
 866                         emit_bpf_tail_call(&prog);
 867                         break;
 868
 869                         /* cond jump */
 870                 case BPF_JMP | BPF_JEQ | BPF_X:
 871                 case BPF_JMP | BPF_JNE | BPF_X:
 872                 case BPF_JMP | BPF_JGT | BPF_X:
 873                 case BPF_JMP | BPF_JLT | BPF_X:
 874                 case BPF_JMP | BPF_JGE | BPF_X:
 875                 case BPF_JMP | BPF_JLE | BPF_X:
 876                 case BPF_JMP | BPF_JSGT | BPF_X:
 877                 case BPF_JMP | BPF_JSLT | BPF_X:
 878                 case BPF_JMP | BPF_JSGE | BPF_X:
 879                 case BPF_JMP | BPF_JSLE | BPF_X:
 880                 case BPF_JMP32 | BPF_JEQ | BPF_X:
 881                 case BPF_JMP32 | BPF_JNE | BPF_X:
 882                 case BPF_JMP32 | BPF_JGT | BPF_X:
 883                 case BPF_JMP32 | BPF_JLT | BPF_X:
 884                 case BPF_JMP32 | BPF_JGE | BPF_X:
 885                 case BPF_JMP32 | BPF_JLE | BPF_X:
 886                 case BPF_JMP32 | BPF_JSGT | BPF_X:
 887                 case BPF_JMP32 | BPF_JSLT | BPF_X:
 888                 case BPF_JMP32 | BPF_JSGE | BPF_X:
 889                 case BPF_JMP32 | BPF_JSLE | BPF_X:
 890                         /* cmp dst_reg, src_reg */
 891                         if (BPF_CLASS(insn->code) == BPF_JMP)
 892                                 EMIT1(add_2mod(0x48, dst_reg, src_reg));
 893                         else if (is_ereg(dst_reg) || is_ereg(src_reg))
 894                                 EMIT1(add_2mod(0x40, dst_reg, src_reg));
 895                         EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg));
 896                         goto emit_cond_jmp;
 897
 898                 case BPF_JMP | BPF_JSET | BPF_X:
 899                 case BPF_JMP32 | BPF_JSET | BPF_X:
 900                         /* test dst_reg, src_reg */
 901                         if (BPF_CLASS(insn->code) == BPF_JMP)
 902                                 EMIT1(add_2mod(0x48, dst_reg, src_reg));
 903                         else if (is_ereg(dst_reg) || is_ereg(src_reg))
 904                                 EMIT1(add_2mod(0x40, dst_reg, src_reg));
 905                         EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg));
 906                         goto emit_cond_jmp;
 907
 908                 case BPF_JMP | BPF_JSET | BPF_K:
 909                 case BPF_JMP32 | BPF_JSET | BPF_K:
 910                         /* test dst_reg, imm32 */
 911                         if (BPF_CLASS(insn->code) == BPF_JMP)
 912                                 EMIT1(add_1mod(0x48, dst_reg));
 913                         else if (is_ereg(dst_reg))
 914                                 EMIT1(add_1mod(0x40, dst_reg));
 915                         EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32);
 916                         goto emit_cond_jmp;
 917
 918                 case BPF_JMP | BPF_JEQ | BPF_K:
 919                 case BPF_JMP | BPF_JNE | BPF_K:
 920                 case BPF_JMP | BPF_JGT | BPF_K:
 921                 case BPF_JMP | BPF_JLT | BPF_K:
 922                 case BPF_JMP | BPF_JGE | BPF_K:
 923                 case BPF_JMP | BPF_JLE | BPF_K:
 924                 case BPF_JMP | BPF_JSGT | BPF_K:
 925                 case BPF_JMP | BPF_JSLT | BPF_K:
 926                 case BPF_JMP | BPF_JSGE | BPF_K:
 927                 case BPF_JMP | BPF_JSLE | BPF_K:
 928                 case BPF_JMP32 | BPF_JEQ | BPF_K:
 929                 case BPF_JMP32 | BPF_JNE | BPF_K:
 930                 case BPF_JMP32 | BPF_JGT | BPF_K:
 931                 case BPF_JMP32 | BPF_JLT | BPF_K:
 932                 case BPF_JMP32 | BPF_JGE | BPF_K:
 933                 case BPF_JMP32 | BPF_JLE | BPF_K:
 934                 case BPF_JMP32 | BPF_JSGT | BPF_K:
 935                 case BPF_JMP32 | BPF_JSLT | BPF_K:
 936                 case BPF_JMP32 | BPF_JSGE | BPF_K:
 937                 case BPF_JMP32 | BPF_JSLE | BPF_K:
 938                         /* cmp dst_reg, imm8/32 */
 939                         if (BPF_CLASS(insn->code) == BPF_JMP)
 940                                 EMIT1(add_1mod(0x48, dst_reg));
 941                         else if (is_ereg(dst_reg))
 942                                 EMIT1(add_1mod(0x40, dst_reg));
 943
 944                         if (is_imm8(imm32))
 945                                 EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32);
 946                         else
 947                                 EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32);
 948
 949 emit_cond_jmp:          /* Convert BPF opcode to x86 */
 950                         switch (BPF_OP(insn->code)) {
 951                         case BPF_JEQ:
 952                                 jmp_cond = X86_JE;
 953                                 break;
 954                         case BPF_JSET:
 955                         case BPF_JNE:
 956                                 jmp_cond = X86_JNE;
 957                                 break;
 958                         case BPF_JGT:
 959                                 /* GT is unsigned '>', JA in x86 */
 960                                 jmp_cond = X86_JA;
 961                                 break;
 962                         case BPF_JLT:
 963                                 /* LT is unsigned '<', JB in x86 */
 964                                 jmp_cond = X86_JB;
 965                                 break;
 966                         case BPF_JGE:
 967                                 /* GE is unsigned '>=', JAE in x86 */
 968                                 jmp_cond = X86_JAE;
 969                                 break;
 970                         case BPF_JLE:
 971                                 /* LE is unsigned '<=', JBE in x86 */
 972                                 jmp_cond = X86_JBE;
 973                                 break;
 974                         case BPF_JSGT:
 975                                 /* Signed '>', GT in x86 */
 976                                 jmp_cond = X86_JG;
 977                                 break;
 978                         case BPF_JSLT:
 979                                 /* Signed '<', LT in x86 */
 980                                 jmp_cond = X86_JL;
 981                                 break;
 982                         case BPF_JSGE:
 983                                 /* Signed '>=', GE in x86 */
 984                                 jmp_cond = X86_JGE;
 985                                 break;
 986                         case BPF_JSLE:
 987                                 /* Signed '<=', LE in x86 */
 988                                 jmp_cond = X86_JLE;
 989                                 break;
 990                         default: /* to silence GCC warning */
 991                                 return -EFAULT;
 992                         }
 993                         jmp_offset = addrs[i + insn->off] - addrs[i];
 994                         if (is_imm8(jmp_offset)) {
 995                                 EMIT2(jmp_cond, jmp_offset);
 996                         } else if (is_simm32(jmp_offset)) {
 997                                 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
 998                         } else {
 999                                 pr_err("cond_jmp gen bug %llx\n", jmp_offset);
1000                                 return -EFAULT;
1001                         }
1002
1003                         break;
1004
1005                 case BPF_JMP | BPF_JA:
1006                         if (insn->off == -1)
1007                                 /* -1 jmp instructions will always jump
1008                                  * backwards two bytes. Explicitly handling
1009                                  * this case avoids wasting too many passes
1010                                  * when there are long sequences of replaced
1011                                  * dead code.
1012                                  */
1013                                 jmp_offset = -2;
1014                         else
1015                                 jmp_offset = addrs[i + insn->off] - addrs[i];
1016
1017                         if (!jmp_offset)
1018                                 /* Optimize out nop jumps */
1019                                 break;
1020 emit_jmp:
1021                         if (is_imm8(jmp_offset)) {
1022                                 EMIT2(0xEB, jmp_offset);
1023                         } else if (is_simm32(jmp_offset)) {
1024                                 EMIT1_off32(0xE9, jmp_offset);
1025                         } else {
1026                                 pr_err("jmp gen bug %llx\n", jmp_offset);
1027                                 return -EFAULT;
1028                         }
1029                         break;
1030
1031                 case BPF_JMP | BPF_EXIT:
1032                         if (seen_exit) {
1033                                 jmp_offset = ctx->cleanup_addr - addrs[i];
1034                                 goto emit_jmp;
1035                         }
1036                         seen_exit = true;
1037                         /* Update cleanup_addr */
1038                         ctx->cleanup_addr = proglen;
1039                         /* mov rbx, qword ptr [rbp+0] */
1040                         EMIT4(0x48, 0x8B, 0x5D, 0);
1041                         /* mov r13, qword ptr [rbp+8] */
1042                         EMIT4(0x4C, 0x8B, 0x6D, 8);
1043                         /* mov r14, qword ptr [rbp+16] */
1044                         EMIT4(0x4C, 0x8B, 0x75, 16);
1045                         /* mov r15, qword ptr [rbp+24] */
1046                         EMIT4(0x4C, 0x8B, 0x7D, 24);
1047
1048                         /* add rbp, AUX_STACK_SPACE */
1049                         EMIT4(0x48, 0x83, 0xC5, AUX_STACK_SPACE);
1050                         EMIT1(0xC9); /* leave */
1051                         EMIT1(0xC3); /* ret */
1052                         break;
1053
1054                 default:
1055                         /*
1056                          * By design x86-64 JIT should support all BPF instructions.
1057                          * This error will be seen if new instruction was added
1058                          * to the interpreter, but not to the JIT, or if there is
1059                          * junk in bpf_prog.
1060                          */
1061                         pr_err("bpf_jit: unknown opcode %02x\n", insn->code);
1062                         return -EINVAL;
1063                 }
1064
1065                 ilen = prog - temp;
1066                 if (ilen > BPF_MAX_INSN_SIZE) {
1067                         pr_err("bpf_jit: fatal insn size error\n");
1068                         return -EFAULT;
1069                 }
1070
1071                 if (image) {
1072                         if (unlikely(proglen + ilen > oldproglen)) {
1073                                 pr_err("bpf_jit: fatal error\n");
1074                                 return -EFAULT;
1075                         }
1076                         memcpy(image + proglen, temp, ilen);
1077                 }
1078                 proglen += ilen;
1079                 addrs[i] = proglen;
1080                 prog = temp;
1081         }
1082         return proglen;
1083 }
1084
1085 struct x64_jit_data {
1086         struct bpf_binary_header *header;
1087         int *addrs;
1088         u8 *image;
1089         int proglen;
1090         struct jit_context ctx;
1091 };
1092
1093 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1094 {
1095         struct bpf_binary_header *header = NULL;
1096         struct bpf_prog *tmp, *orig_prog = prog;
1097         struct x64_jit_data *jit_data;
1098         int proglen, oldproglen = 0;
1099         struct jit_context ctx = {};
1100         bool tmp_blinded = false;
1101         bool extra_pass = false;
1102         u8 *image = NULL;
1103         int *addrs;
1104         int pass;
1105         int i;
1106
1107         if (!prog->jit_requested)
1108                 return orig_prog;
1109
1110         tmp = bpf_jit_blind_constants(prog);
1111         /*
1112          * If blinding was requested and we failed during blinding,
1113          * we must fall back to the interpreter.
1114          */
1115         if (IS_ERR(tmp))
1116                 return orig_prog;
1117         if (tmp != prog) {
1118                 tmp_blinded = true;
1119                 prog = tmp;
1120         }
1121
1122         jit_data = prog->aux->jit_data;
1123         if (!jit_data) {
1124                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1125                 if (!jit_data) {
1126                         prog = orig_prog;
1127                         goto out;
1128                 }
1129                 prog->aux->jit_data = jit_data;
1130         }
1131         addrs = jit_data->addrs;
1132         if (addrs) {
1133                 ctx = jit_data->ctx;
1134                 oldproglen = jit_data->proglen;
1135                 image = jit_data->image;
1136                 header = jit_data->header;
1137                 extra_pass = true;
1138                 goto skip_init_addrs;
1139         }
1140         addrs = kmalloc_array(prog->len, sizeof(*addrs), GFP_KERNEL);
1141         if (!addrs) {
1142                 prog = orig_prog;
1143                 goto out_addrs;
1144         }
1145
1146         /*
1147          * Before first pass, make a rough estimation of addrs[]
1148          * each BPF instruction is translated to less than 64 bytes
1149          */
1150         for (proglen = 0, i = 0; i < prog->len; i++) {
1151                 proglen += 64;
1152                 addrs[i] = proglen;
1153         }
1154         ctx.cleanup_addr = proglen;
1155 skip_init_addrs:
1156
1157         /*
1158          * JITed image shrinks with every pass and the loop iterates
1159          * until the image stops shrinking. Very large BPF programs
1160          * may converge on the last pass. In such case do one more
1161          * pass to emit the final image.
1162          */
1163         for (pass = 0; pass < 20 || image; pass++) {
1164                 proglen = do_jit(prog, addrs, image, oldproglen, &ctx);
1165                 if (proglen <= 0) {
1166 out_image:
1167                         image = NULL;
1168                         if (header)
1169                                 bpf_jit_binary_free(header);
1170                         prog = orig_prog;
1171                         goto out_addrs;
1172                 }
1173                 if (image) {
1174                         if (proglen != oldproglen) {
1175                                 pr_err("bpf_jit: proglen=%d != oldproglen=%d\n",
1176                                        proglen, oldproglen);
1177                                 goto out_image;
1178                         }
1179                         break;
1180                 }
1181                 if (proglen == oldproglen) {
1182                         header = bpf_jit_binary_alloc(proglen, &image,
1183                                                       1, jit_fill_hole);
1184                         if (!header) {
1185                                 prog = orig_prog;
1186                                 goto out_addrs;
1187                         }
1188                 }
1189                 oldproglen = proglen;
1190                 cond_resched();
1191         }
1192
1193         if (bpf_jit_enable > 1)
1194                 bpf_jit_dump(prog->len, proglen, pass + 1, image);
1195
1196         if (image) {
1197                 if (!prog->is_func || extra_pass) {
1198                         bpf_jit_binary_lock_ro(header);
1199                 } else {
1200                         jit_data->addrs = addrs;
1201                         jit_data->ctx = ctx;
1202                         jit_data->proglen = proglen;
1203                         jit_data->image = image;
1204                         jit_data->header = header;
1205                 }
1206                 prog->bpf_func = (void *)image;
1207                 prog->jited = 1;
1208                 prog->jited_len = proglen;
1209         } else {
1210                 prog = orig_prog;
1211         }
1212
1213         if (!image || !prog->is_func || extra_pass) {
1214                 if (image)
1215                         bpf_prog_fill_jited_linfo(prog, addrs);
1216 out_addrs:
1217                 kfree(addrs);
1218                 kfree(jit_data);
1219                 prog->aux->jit_data = NULL;
1220         }
1221 out:
1222         if (tmp_blinded)
1223                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1224                                            tmp : orig_prog);
1225         return prog;
1226 }