arch/x86/kernel/jump_label.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * jump label x86 support
   4  *
   5  * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
   6  *
   7  */
   8 #include <linux/jump_label.h>
   9 #include <linux/memory.h>
  10 #include <linux/uaccess.h>
  11 #include <linux/module.h>
  12 #include <linux/list.h>
  13 #include <linux/jhash.h>
  14 #include <linux/cpu.h>
  15 #include <asm/kprobes.h>
  16 #include <asm/alternative.h>
  17 #include <asm/text-patching.h>
  18
  19 static const void *
  20 __jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type)
  21 {
  22         const void *expect, *code;
  23         const void *addr, *dest;
  24
  25         addr = (void *)jump_entry_code(entry);
  26         dest = (void *)jump_entry_target(entry);
  27
  28         code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest);
  29
  30         if (type == JUMP_LABEL_JMP)
  31                 expect = x86_nops[5];
  32         else
  33                 expect = code;
  34
  35         if (memcmp(addr, expect, JUMP_LABEL_NOP_SIZE)) {
  36                 /*
  37                  * The location is not an op that we were expecting.
  38                  * Something went wrong. Crash the box, as something could be
  39                  * corrupting the kernel.
  40                  */
  41                 pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph != %5ph)) type:%d\n",
  42                                 addr, addr, addr, expect, type);
  43                 BUG();
  44         }
  45
  46         if (type == JUMP_LABEL_NOP)
  47                 code = x86_nops[5];
  48
  49         return code;
  50 }
  51
  52 static inline void __jump_label_transform(struct jump_entry *entry,
  53                                           enum jump_label_type type,
  54                                           int init)
  55 {
  56         const void *opcode = __jump_label_set_jump_code(entry, type);
  57
  58         /*
  59          * As long as only a single processor is running and the code is still
  60          * not marked as RO, text_poke_early() can be used; Checking that
  61          * system_state is SYSTEM_BOOTING guarantees it. It will be set to
  62          * SYSTEM_SCHEDULING before other cores are awaken and before the
  63          * code is write-protected.
  64          *
  65          * At the time the change is being done, just ignore whether we
  66          * are doing nop -> jump or jump -> nop transition, and assume
  67          * always nop being the 'currently valid' instruction
  68          */
  69         if (init || system_state == SYSTEM_BOOTING) {
  70                 text_poke_early((void *)jump_entry_code(entry), opcode,
  71                                 JUMP_LABEL_NOP_SIZE);
  72                 return;
  73         }
  74
  75         text_poke_bp((void *)jump_entry_code(entry), opcode, JUMP_LABEL_NOP_SIZE, NULL);
  76 }
  77
  78 static void __ref jump_label_transform(struct jump_entry *entry,
  79                                        enum jump_label_type type,
  80                                        int init)
  81 {
  82         mutex_lock(&text_mutex);
  83         __jump_label_transform(entry, type, init);
  84         mutex_unlock(&text_mutex);
  85 }
  86
  87 void arch_jump_label_transform(struct jump_entry *entry,
  88                                enum jump_label_type type)
  89 {
  90         jump_label_transform(entry, type, 0);
  91 }
  92
  93 bool arch_jump_label_transform_queue(struct jump_entry *entry,
  94                                      enum jump_label_type type)
  95 {
  96         const void *opcode;
  97
  98         if (system_state == SYSTEM_BOOTING) {
  99                 /*
 100                  * Fallback to the non-batching mode.
 101                  */
 102                 arch_jump_label_transform(entry, type);
 103                 return true;
 104         }
 105
 106         mutex_lock(&text_mutex);
 107         opcode = __jump_label_set_jump_code(entry, type);
 108         text_poke_queue((void *)jump_entry_code(entry),
 109                         opcode, JUMP_LABEL_NOP_SIZE, NULL);
 110         mutex_unlock(&text_mutex);
 111         return true;
 112 }
 113
 114 void arch_jump_label_transform_apply(void)
 115 {
 116         mutex_lock(&text_mutex);
 117         text_poke_finish();
 118         mutex_unlock(&text_mutex);
 119 }
 120
 121 static enum {
 122         JL_STATE_START,
 123         JL_STATE_NO_UPDATE,
 124         JL_STATE_UPDATE,
 125 } jlstate __initdata_or_module = JL_STATE_START;
 126
 127 __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
 128                                       enum jump_label_type type)
 129 {
 130         if (jlstate == JL_STATE_UPDATE)
 131                 jump_label_transform(entry, type, 1);
 132 }