riscv: ftrace: support direct call using call_ops

jump to FTRACE_ADDR if distance is out of reach

Co-developed-by: Björn Töpel <bjorn@rivosinc.com>
Signed-off-by: Björn Töpel <bjorn@rivosinc.com>
Signed-off-by: Andy Chiu <andybnac@gmail.com>
Link: https://lore.kernel.org/r/20250407180838.42877-11-andybnac@gmail.com
Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com>
Signed-off-by: Palmer Dabbelt <palmer@dabbelt.com>

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index ec986c9..8fdca63 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -152,7 +152,7 @@ config RISCV
        select HAVE_DMA_CONTIGUOUS if MMU
        select HAVE_DYNAMIC_FTRACE if !XIP_KERNEL && MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE)
        select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C
-       select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+       select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
        select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI_CLANG)
        select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE
        select HAVE_FTRACE_GRAPH_FUNC

diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h
index 6a5c0a7..22ebea3 100644 (file)
--- a/arch/riscv/include/asm/ftrace.h
+++ b/arch/riscv/include/asm/ftrace.h
@@ -130,6 +130,9 @@ struct __arch_ftrace_regs {
        unsigned long sp;
        unsigned long s0;
        unsigned long t1;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+       unsigned long direct_tramp;
+#endif
        union {
                unsigned long args[8];
                struct {
@@ -223,10 +226,13 @@ void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
                       struct ftrace_ops *op, struct ftrace_regs *fregs);
 #define ftrace_graph_func ftrace_graph_func
 
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs, unsigned long addr)
 {
        arch_ftrace_regs(fregs)->t1 = addr;
 }
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
+
 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
 
 #endif /* __ASSEMBLY__ */

diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index 2d96197..b263340 100644 (file)
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -495,6 +495,9 @@ void asm_offsets(void)
        OFFSET(STACKFRAME_RA, stackframe, ra);
 #ifdef CONFIG_FUNCTION_TRACER
        DEFINE(FTRACE_OPS_FUNC,         offsetof(struct ftrace_ops, func));
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+       DEFINE(FTRACE_OPS_DIRECT_CALL,  offsetof(struct ftrace_ops, direct_call));
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
 #endif
 
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS

diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c
index d56fc6e..4c6c243 100644 (file)
--- a/arch/riscv/kernel/ftrace.c
+++ b/arch/riscv/kernel/ftrace.c
@@ -17,7 +17,7 @@
 unsigned long ftrace_call_adjust(unsigned long addr)
 {
        if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
-               return addr + 8;
+               return addr + 8 + MCOUNT_AUIPC_SIZE;
 
        return addr + MCOUNT_AUIPC_SIZE;
 }
@@ -83,10 +83,9 @@ static const struct ftrace_ops *riscv64_rec_get_ops(struct dyn_ftrace *rec)
        return ops;
 }
 
-static int ftrace_rec_set_ops(const struct dyn_ftrace *rec,
-                             const struct ftrace_ops *ops)
+static int ftrace_rec_set_ops(const struct dyn_ftrace *rec, const struct ftrace_ops *ops)
 {
-       unsigned long literal = rec->ip - 8;
+       unsigned long literal = ALIGN_DOWN(rec->ip - 12, 8);
 
        return patch_text_nosync((void *)literal, &ops, sizeof(ops));
 }
@@ -117,7 +116,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
        orig_addr = (unsigned long)&ftrace_caller;
        distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr;
        if (distance > JALR_RANGE)
-               return -EINVAL;
+               addr = FTRACE_ADDR;
 
        return __ftrace_modify_call(pc, addr, false);
 }
@@ -204,15 +203,13 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
                       unsigned long addr)
 {
        unsigned long caller = rec->ip - MCOUNT_AUIPC_SIZE;
-       unsigned int call[2];
        int ret;
 
-       make_call_t0(caller, old_addr, call);
        ret = ftrace_rec_update_ops(rec);
        if (ret)
                return ret;
 
-       return __ftrace_modify_call(caller, addr, true);
+       return __ftrace_modify_call(caller, FTRACE_ADDR, true);
 }
 #endif
 

diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S
index 699684e..48f6c4f 100644 (file)
--- a/arch/riscv/kernel/mcount-dyn.S
+++ b/arch/riscv/kernel/mcount-dyn.S
@@ -82,12 +82,9 @@
 *                       +++++++++
 **/
        .macro SAVE_ABI_REGS
-       mv      t4, sp                  // Save original SP in T4
        addi    sp, sp, -FREGS_SIZE_ON_STACK
-
        REG_S   t0,  FREGS_EPC(sp)
        REG_S   x1,  FREGS_RA(sp)
-       REG_S   t4,  FREGS_SP(sp)       // Put original SP on stack
 #ifdef HAVE_FUNCTION_GRAPH_FP_TEST
        REG_S   x8,  FREGS_S0(sp)
 #endif
@@ -108,9 +105,12 @@
        REG_S   x15, FREGS_A5(sp)
        REG_S   x16, FREGS_A6(sp)
        REG_S   x17, FREGS_A7(sp)
+       mv      a0, sp
+       addi    a0, a0, FREGS_SIZE_ON_STACK
+       REG_S   a0, FREGS_SP(sp)        // Put original SP on stack
        .endm
 
-       .macro RESTORE_ABI_REGS, all=0
+       .macro RESTORE_ABI_REGS
        REG_L   t0, FREGS_EPC(sp)
        REG_L   x1, FREGS_RA(sp)
 #ifdef HAVE_FUNCTION_GRAPH_FP_TEST
@@ -139,6 +139,19 @@
 
        .macro PREPARE_ARGS
        addi    a0, t0, -MCOUNT_JALR_SIZE       // ip (callsite's jalr insn)
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
+       mv      a1, ra                          // parent_ip
+       REG_L   a2, -16(t0)                     // op
+       REG_L   ra, FTRACE_OPS_FUNC(a2)         // op->func
+#else
+       la      a1, function_trace_op
+       REG_L   a2, 0(a1)                       // op
+       mv      a1, ra                          // parent_ip
+#endif
+       mv      a3, sp                          // regs
+       .endm
+
+SYM_FUNC_START(ftrace_caller)
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
        /*
         * When CALL_OPS is enabled (2 or 4) nops [8B] are placed before the
@@ -158,19 +171,17 @@
         * t0 is set to ip+8 after the jalr is executed at the callsite,
         * so we find the associated op at t0-16.
         */
-       mv      a1, ra                          // parent_ip
-       REG_L   a2, -16(t0)                     // op
-       REG_L   ra, FTRACE_OPS_FUNC(a2)         // op->func
-#else
-       la      a1, function_trace_op
-       REG_L   a2, 0(a1)                       // op
-       mv      a1, ra                          // parent_ip
-#endif
-       mv      a3, sp                          // regs
-       .endm
+       REG_L   t1, -16(t0) // op Should be SZ_REG instead of 16
 
-SYM_FUNC_START(ftrace_caller)
-       mv      t1, zero
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+       /*
+        * If the op has a direct call, handle it immediately without
+        * saving/restoring registers.
+        */
+       REG_L   t1, FTRACE_OPS_DIRECT_CALL(t1)
+       bnez    t1, ftrace_caller_direct
+#endif
+#endif
        SAVE_ABI_REGS
        PREPARE_ARGS
 
@@ -182,10 +193,14 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
        jalr    ra, 0(ra)
 #endif
        RESTORE_ABI_REGS
-       bnez    t1, .Ldirect
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+       bnez    t1, ftrace_caller_direct
+#endif
        jr      t0
-.Ldirect:
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+SYM_INNER_LABEL(ftrace_caller_direct, SYM_L_LOCAL)
        jr      t1
+#endif
 SYM_FUNC_END(ftrace_caller)
 
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS