arch/xtensa/kernel/jump_label.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (C) 2018 Cadence Design Systems Inc.
   3
   4 #include <linux/cpu.h>
   5 #include <linux/jump_label.h>
   6 #include <linux/kernel.h>
   7 #include <linux/memory.h>
   8 #include <linux/stop_machine.h>
   9 #include <linux/types.h>
  10
  11 #include <asm/cacheflush.h>
  12
  13 #ifdef HAVE_JUMP_LABEL
  14
  15 #define J_OFFSET_MASK 0x0003ffff
  16 #define J_SIGN_MASK (~(J_OFFSET_MASK >> 1))
  17
  18 #if defined(__XTENSA_EL__)
  19 #define J_INSN 0x6
  20 #define NOP_INSN 0x0020f0
  21 #elif defined(__XTENSA_EB__)
  22 #define J_INSN 0x60000000
  23 #define NOP_INSN 0x0f020000
  24 #else
  25 #error Unsupported endianness.
  26 #endif
  27
  28 struct patch {
  29         atomic_t cpu_count;
  30         unsigned long addr;
  31         size_t sz;
  32         const void *data;
  33 };
  34
  35 static void local_patch_text(unsigned long addr, const void *data, size_t sz)
  36 {
  37         memcpy((void *)addr, data, sz);
  38         local_flush_icache_range(addr, addr + sz);
  39 }
  40
  41 static int patch_text_stop_machine(void *data)
  42 {
  43         struct patch *patch = data;
  44
  45         if (atomic_inc_return(&patch->cpu_count) == 1) {
  46                 local_patch_text(patch->addr, patch->data, patch->sz);
  47                 atomic_inc(&patch->cpu_count);
  48         } else {
  49                 while (atomic_read(&patch->cpu_count) <= num_online_cpus())
  50                         cpu_relax();
  51                 __invalidate_icache_range(patch->addr, patch->sz);
  52         }
  53         return 0;
  54 }
  55
  56 static void patch_text(unsigned long addr, const void *data, size_t sz)
  57 {
  58         if (IS_ENABLED(CONFIG_SMP)) {
  59                 struct patch patch = {
  60                         .cpu_count = ATOMIC_INIT(0),
  61                         .addr = addr,
  62                         .sz = sz,
  63                         .data = data,
  64                 };
  65                 stop_machine_cpuslocked(patch_text_stop_machine,
  66                                         &patch, NULL);
  67         } else {
  68                 unsigned long flags;
  69
  70                 local_irq_save(flags);
  71                 local_patch_text(addr, data, sz);
  72                 local_irq_restore(flags);
  73         }
  74 }
  75
  76 void arch_jump_label_transform(struct jump_entry *e,
  77                                enum jump_label_type type)
  78 {
  79         u32 d = (jump_entry_target(e) - (jump_entry_code(e) + 4));
  80         u32 insn;
  81
  82         /* Jump only works within 128K of the J instruction. */
  83         BUG_ON(!((d & J_SIGN_MASK) == 0 ||
  84                  (d & J_SIGN_MASK) == J_SIGN_MASK));
  85
  86         if (type == JUMP_LABEL_JMP) {
  87 #if defined(__XTENSA_EL__)
  88                 insn = ((d & J_OFFSET_MASK) << 6) | J_INSN;
  89 #elif defined(__XTENSA_EB__)
  90                 insn = ((d & J_OFFSET_MASK) << 8) | J_INSN;
  91 #endif
  92         } else {
  93                 insn = NOP_INSN;
  94         }
  95
  96         patch_text(jump_entry_code(e), &insn, JUMP_LABEL_NOP_SIZE);
  97 }
  98
  99 #endif /* HAVE_JUMP_LABEL */