arch/x86/include/asm/jump_label.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_JUMP_LABEL_H
   3 #define _ASM_X86_JUMP_LABEL_H
   4
   5 #ifndef HAVE_JUMP_LABEL
   6 /*
   7  * For better or for worse, if jump labels (the gcc extension) are missing,
   8  * then the entire static branch patching infrastructure is compiled out.
   9  * If that happens, the code in here will malfunction.  Raise a compiler
  10  * error instead.
  11  *
  12  * In theory, jump labels and the static branch patching infrastructure
  13  * could be decoupled to fix this.
  14  */
  15 #error asm/jump_label.h included on a non-jump-label kernel
  16 #endif
  17
  18 #define JUMP_LABEL_NOP_SIZE 5
  19
  20 #ifdef CONFIG_X86_64
  21 # define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
  22 #else
  23 # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
  24 #endif
  25
  26 #include <asm/asm.h>
  27 #include <asm/nops.h>
  28
  29 #ifndef __ASSEMBLY__
  30
  31 #include <linux/stringify.h>
  32 #include <linux/types.h>
  33
  34 static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
  35 {
  36         asm_volatile_goto("1:"
  37                 ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
  38                 ".pushsection __jump_table,  \"aw\" \n\t"
  39                 _ASM_ALIGN "\n\t"
  40                 ".long 1b - ., %l[l_yes] - . \n\t"
  41                 _ASM_PTR "%c0 + %c1 - .\n\t"
  42                 ".popsection \n\t"
  43                 : :  "i" (key), "i" (branch) : : l_yes);
  44
  45         return false;
  46 l_yes:
  47         return true;
  48 }
  49
  50 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
  51 {
  52         asm_volatile_goto("1:"
  53                 ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
  54                 "2:\n\t"
  55                 ".pushsection __jump_table,  \"aw\" \n\t"
  56                 _ASM_ALIGN "\n\t"
  57                 ".long 1b - ., %l[l_yes] - . \n\t"
  58                 _ASM_PTR "%c0 + %c1 - .\n\t"
  59                 ".popsection \n\t"
  60                 : :  "i" (key), "i" (branch) : : l_yes);
  61
  62         return false;
  63 l_yes:
  64         return true;
  65 }
  66
  67 #else   /* __ASSEMBLY__ */
  68
  69 .macro STATIC_JUMP_IF_TRUE target, key, def
  70 .Lstatic_jump_\@:
  71         .if \def
  72         /* Equivalent to "jmp.d32 \target" */
  73         .byte           0xe9
  74         .long           \target - .Lstatic_jump_after_\@
  75 .Lstatic_jump_after_\@:
  76         .else
  77         .byte           STATIC_KEY_INIT_NOP
  78         .endif
  79         .pushsection __jump_table, "aw"
  80         _ASM_ALIGN
  81         .long           .Lstatic_jump_\@ - ., \target - .
  82         _ASM_PTR        \key - .
  83         .popsection
  84 .endm
  85
  86 .macro STATIC_JUMP_IF_FALSE target, key, def
  87 .Lstatic_jump_\@:
  88         .if \def
  89         .byte           STATIC_KEY_INIT_NOP
  90         .else
  91         /* Equivalent to "jmp.d32 \target" */
  92         .byte           0xe9
  93         .long           \target - .Lstatic_jump_after_\@
  94 .Lstatic_jump_after_\@:
  95         .endif
  96         .pushsection __jump_table, "aw"
  97         _ASM_ALIGN
  98         .long           .Lstatic_jump_\@ - ., \target - .
  99         _ASM_PTR        \key + 1 - .
 100         .popsection
 101 .endm
 102
 103 #endif  /* __ASSEMBLY__ */
 104
 105 #endif