riscv: Add jump-label implementation

Add jump-label implementation based on the ARM64 version
and add CONFIG_JUMP_LABEL=y to the defconfigs.

Signed-off-by: Emil Renner Berthing <kernel@esmil.dk>
Reviewed-by: Björn Töpel <bjorn.topel@gmail.com>
Tested-by: Björn Töpel <bjorn.topel@gmail.com>
Signed-off-by: Palmer Dabbelt <palmerdabbelt@google.com>

diff --git a/Documentation/features/core/jump-labels/arch-support.txt b/Documentation/features/core/jump-labels/arch-support.txt
index 632a1c7..760243d 100644 (file)
--- a/Documentation/features/core/jump-labels/arch-support.txt
+++ b/Documentation/features/core/jump-labels/arch-support.txt
@@ -23,7 +23,7 @@
     |    openrisc: | TODO |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: |  ok  |

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index b55be09..bf6a9fe 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -47,6 +47,8 @@ config RISCV
        select GENERIC_TIME_VSYSCALL if MMU && 64BIT
        select HANDLE_DOMAIN_IRQ
        select HAVE_ARCH_AUDITSYSCALL
+       select HAVE_ARCH_JUMP_LABEL
+       select HAVE_ARCH_JUMP_LABEL_RELATIVE
        select HAVE_ARCH_KASAN if MMU && 64BIT
        select HAVE_ARCH_KGDB
        select HAVE_ARCH_KGDB_QXFER_PKT

diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig
index 4da4886..d58c93e 100644 (file)
--- a/arch/riscv/configs/defconfig
+++ b/arch/riscv/configs/defconfig
@@ -17,6 +17,7 @@ CONFIG_BPF_SYSCALL=y
 CONFIG_SOC_SIFIVE=y
 CONFIG_SOC_VIRT=y
 CONFIG_SMP=y
+CONFIG_JUMP_LABEL=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_NET=y

diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig
index b48138e..cd1df62 100644 (file)
--- a/arch/riscv/configs/nommu_k210_defconfig
+++ b/arch/riscv/configs/nommu_k210_defconfig
@@ -33,6 +33,7 @@ CONFIG_SMP=y
 CONFIG_NR_CPUS=2
 CONFIG_CMDLINE="earlycon console=ttySIF0"
 CONFIG_CMDLINE_FORCE=y
+CONFIG_JUMP_LABEL=y
 # CONFIG_BLOCK is not set
 CONFIG_BINFMT_FLAT=y
 # CONFIG_COREDUMP is not set

diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig
index cf74e17..f27596e 100644 (file)
--- a/arch/riscv/configs/nommu_virt_defconfig
+++ b/arch/riscv/configs/nommu_virt_defconfig
@@ -30,6 +30,7 @@ CONFIG_MAXPHYSMEM_2GB=y
 CONFIG_SMP=y
 CONFIG_CMDLINE="root=/dev/vda rw earlycon=uart8250,mmio,0x10000000,115200n8 console=ttyS0"
 CONFIG_CMDLINE_FORCE=y
+CONFIG_JUMP_LABEL=y
 # CONFIG_BLK_DEV_BSG is not set
 CONFIG_PARTITION_ADVANCED=y
 # CONFIG_MSDOS_PARTITION is not set

diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig
index 05bbf52..3a55f0e 100644 (file)
--- a/arch/riscv/configs/rv32_defconfig
+++ b/arch/riscv/configs/rv32_defconfig
@@ -17,6 +17,7 @@ CONFIG_BPF_SYSCALL=y
 CONFIG_SOC_VIRT=y
 CONFIG_ARCH_RV32I=y
 CONFIG_SMP=y
+CONFIG_JUMP_LABEL=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_NET=y

diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h
new file mode 100644 (file)
index 0000000..38af2ec
--- /dev/null
+++ b/arch/riscv/include/asm/jump_label.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/include/asm/jump_label.h
+ */
+#ifndef __ASM_JUMP_LABEL_H
+#define __ASM_JUMP_LABEL_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+#include <asm/asm.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+static __always_inline bool arch_static_branch(struct static_key *key,
+                                              bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     nop                                     \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key,
+                                                   bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     jal             zero, %l[label]         \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+#endif  /* __ASSEMBLY__ */
+#endif /* __ASM_JUMP_LABEL_H */

diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index b355cf4..a5287ab 100644 (file)
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -53,4 +53,6 @@ endif
 obj-$(CONFIG_HOTPLUG_CPU)      += cpu-hotplug.o
 obj-$(CONFIG_KGDB)             += kgdb.o
 
+obj-$(CONFIG_JUMP_LABEL)       += jump_label.o
+
 clean:

diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c
new file mode 100644 (file)
index 0000000..20e0905
--- /dev/null
+++ b/arch/riscv/kernel/jump_label.c
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/kernel/jump_label.c
+ */
+#include <linux/jump_label.h>
+#include <linux/kernel.h>
+#include <linux/memory.h>
+#include <linux/mutex.h>
+#include <asm/bug.h>
+#include <asm/patch.h>
+
+#define RISCV_INSN_NOP 0x00000013U
+#define RISCV_INSN_JAL 0x0000006fU
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                              enum jump_label_type type)
+{
+       void *addr = (void *)jump_entry_code(entry);
+       u32 insn;
+
+       if (type == JUMP_LABEL_JMP) {
+               long offset = jump_entry_target(entry) - jump_entry_code(entry);
+
+               if (WARN_ON(offset & 1 || offset < -524288 || offset >= 524288))
+                       return;
+
+               insn = RISCV_INSN_JAL |
+                       (((u32)offset & GENMASK(19, 12)) << (12 - 12)) |
+                       (((u32)offset & GENMASK(11, 11)) << (20 - 11)) |
+                       (((u32)offset & GENMASK(10,  1)) << (21 -  1)) |
+                       (((u32)offset & GENMASK(20, 20)) << (31 - 20));
+       } else {
+               insn = RISCV_INSN_NOP;
+       }
+
+       mutex_lock(&text_mutex);
+       patch_text_nosync(addr, &insn, sizeof(insn));
+       mutex_unlock(&text_mutex);
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+                                     enum jump_label_type type)
+{
+       /*
+        * We use the same instructions in the arch_static_branch and
+        * arch_static_branch_jump inline functions, so there's no
+        * need to patch them up here.
+        * The core will call arch_jump_label_transform  when those
+        * instructions need to be replaced.
+        */
+}