Enable the function graph tracer in combination with the EABI unwinder,
so that Thumb2 builds or Clang ARM builds can make use of it.
This involves using the unwinder to locate the return address of an
instrumented function on the stack, so that it can be overridden and
made to refer to the ftrace handling routines that need to be called at
function return.
Given that for these builds, it is not guaranteed that the value of the
link register is stored on the stack, fall back to the stack slot that
will be used by the ftrace exit code to restore LR in the instrumented
function's execution context.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
select HAVE_EXIT_THREAD
select HAVE_FAST_GUP if ARM_LPAE
select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
select HAVE_EXIT_THREAD
select HAVE_FAST_GUP if ARM_LPAE
select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
- select HAVE_FUNCTION_GRAPH_TRACER if !THUMB2_KERNEL && !CC_IS_CLANG
+ select HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !(THUMB2_KERNEL && CC_IS_CLANG)
select HAVE_FUTEX_CMPXCHG if FUTEX
select HAVE_GCC_PLUGINS
select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !(THUMB2_KERNEL && CC_IS_CLANG)
select HAVE_FUTEX_CMPXCHG if FUTEX
select HAVE_GCC_PLUGINS
config UNWINDER_ARM
bool "ARM EABI stack unwinder"
config UNWINDER_ARM
bool "ARM EABI stack unwinder"
- depends on AEABI && !FUNCTION_GRAPH_TRACER
# https://github.com/ClangBuiltLinux/linux/issues/732
depends on !LD_IS_LLD || LLD_VERSION >= 110000
select ARM_UNWIND
# https://github.com/ClangBuiltLinux/linux/issues/732
depends on !LD_IS_LLD || LLD_VERSION >= 110000
select ARM_UNWIND
-#if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND)
-/*
- * return_address uses walk_stackframe to do it's work. If both
- * CONFIG_FRAME_POINTER=y and CONFIG_ARM_UNWIND=y walk_stackframe uses unwind
- * information. For this to work in the function tracer many functions would
- * have to be marked with __notrace. So for now just depend on
- * !CONFIG_ARM_UNWIND.
- */
-
void *return_address(unsigned int);
void *return_address(unsigned int);
-#else
-
-static inline void *return_address(unsigned int level)
-{
- return NULL;
-}
-
-#endif
-
#define ftrace_return_address(n) return_address(n)
#define ARCH_HAS_SYSCALL_MATCH_SYM_NAME
#define ftrace_return_address(n) return_address(n)
#define ARCH_HAS_SYSCALL_MATCH_SYM_NAME
KASAN_SANITIZE_stacktrace.o := n
KASAN_SANITIZE_traps.o := n
KASAN_SANITIZE_stacktrace.o := n
KASAN_SANITIZE_traps.o := n
-ifneq ($(CONFIG_ARM_UNWIND),y)
-obj-$(CONFIG_FRAME_POINTER) += return_address.o
-endif
-
+obj-y += return_address.o
obj-$(CONFIG_ATAGS) += atags_parse.o
obj-$(CONFIG_ATAGS_PROC) += atags_proc.o
obj-$(CONFIG_DEPRECATED_PARAM_STRUCT) += atags_compat.o
obj-$(CONFIG_ATAGS) += atags_parse.o
obj-$(CONFIG_ATAGS_PROC) += atags_proc.o
obj-$(CONFIG_DEPRECATED_PARAM_STRUCT) += atags_compat.o
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.globl ftrace_graph_regs_call
ftrace_graph_regs_call:
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.globl ftrace_graph_regs_call
ftrace_graph_regs_call:
+ARM( mov r0, r0 )
+THUMB( nop.w )
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.macro __ftrace_graph_regs_caller
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.macro __ftrace_graph_regs_caller
- sub r0, fp, #4 @ lr of instrumented routine (parent)
+#ifdef CONFIG_UNWINDER_FRAME_POINTER
+ sub r0, fp, #4 @ lr of instrumented routine (parent)
+#else
+ add r0, sp, #S_LR
+#endif
@ called from __ftrace_regs_caller
@ called from __ftrace_regs_caller
- ldr r1, [sp, #S_PC] @ instrumented routine (func)
+ ldr r1, [sp, #S_PC] @ instrumented routine (func)
mcount_adjust_addr r1, r1
mcount_adjust_addr r1, r1
- mov r2, fp @ frame pointer
+ mov r2, fpreg @ frame pointer
+ add r3, sp, #PT_REGS_SIZE
bl prepare_ftrace_return
@ pop registers saved in ftrace_regs_caller
bl prepare_ftrace_return
@ pop registers saved in ftrace_regs_caller
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.globl ftrace_graph_call\suffix
ftrace_graph_call\suffix:
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.globl ftrace_graph_call\suffix
ftrace_graph_call\suffix:
+ARM( mov r0, r0 )
+THUMB( nop.w )
#endif
mcount_exit
.endm
.macro __ftrace_graph_caller
#endif
mcount_exit
.endm
.macro __ftrace_graph_caller
+#ifdef CONFIG_UNWINDER_FRAME_POINTER
sub r0, fp, #4 @ &lr of instrumented routine (&parent)
sub r0, fp, #4 @ &lr of instrumented routine (&parent)
+#else
+ add r0, sp, #20
+#endif
#ifdef CONFIG_DYNAMIC_FTRACE
@ called from __ftrace_caller, saved in mcount_enter
ldr r1, [sp, #16] @ instrumented routine (func)
#ifdef CONFIG_DYNAMIC_FTRACE
@ called from __ftrace_caller, saved in mcount_enter
ldr r1, [sp, #16] @ instrumented routine (func)
@ called from __mcount, untouched in lr
mcount_adjust_addr r1, lr @ instrumented routine (func)
#endif
@ called from __mcount, untouched in lr
mcount_adjust_addr r1, lr @ instrumented routine (func)
#endif
- mov r2, fp @ frame pointer
+ mov r2, fpreg @ frame pointer
+ add r3, sp, #24
bl prepare_ftrace_return
mcount_exit
.endm
bl prepare_ftrace_return
mcount_exit
.endm
.purgem mcount_exit
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.purgem mcount_exit
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
- .globl return_to_handler
-return_to_handler:
+ENTRY(return_to_handler)
stmdb sp!, {r0-r3}
add r0, sp, #16 @ sp at exit of instrumented routine
bl ftrace_return_to_handler
mov lr, r0 @ r0 has real ret addr
ldmia sp!, {r0-r3}
ret lr
stmdb sp!, {r0-r3}
add r0, sp, #16 @ sp at exit of instrumented routine
bl ftrace_return_to_handler
mov lr, r0 @ r0 has real ret addr
ldmia sp!, {r0-r3}
ret lr
+ENDPROC(return_to_handler)
#endif
ENTRY(ftrace_stub)
#endif
ENTRY(ftrace_stub)
#include <asm/ftrace.h>
#include <asm/insn.h>
#include <asm/set_memory.h>
#include <asm/ftrace.h>
#include <asm/insn.h>
#include <asm/set_memory.h>
+#include <asm/stacktrace.h>
#include <asm/patch.h>
/*
#include <asm/patch.h>
/*
#endif /* CONFIG_DYNAMIC_FTRACE */
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
#endif /* CONFIG_DYNAMIC_FTRACE */
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
- unsigned long frame_pointer)
+ unsigned long frame_pointer,
+ unsigned long stack_pointer)
{
unsigned long return_hooker = (unsigned long) &return_to_handler;
unsigned long old;
{
unsigned long return_hooker = (unsigned long) &return_to_handler;
unsigned long old;
if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) {
/* FP points one word below parent's top of stack */
frame_pointer += 4;
if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) {
/* FP points one word below parent's top of stack */
frame_pointer += 4;
+ } else {
+ struct stackframe frame = {
+ .fp = frame_pointer,
+ .sp = stack_pointer,
+ .lr = self_addr,
+ .pc = self_addr,
+ };
+ if (unwind_frame(&frame) < 0)
+ return;
+ if (frame.lr != self_addr)
+ parent = frame.lr_addr;
+ frame_pointer = frame.sp;
unsigned long caller_fn = (unsigned long) func;
unsigned long pc = (unsigned long) callsite;
unsigned long branch = arm_gen_branch(pc, caller_fn);
unsigned long caller_fn = (unsigned long) func;
unsigned long pc = (unsigned long) callsite;
unsigned long branch = arm_gen_branch(pc, caller_fn);
- unsigned long nop = 0xe1a00000; /* mov r0, r0 */
+ unsigned long nop = arm_gen_nop();
unsigned long old = enable ? nop : branch;
unsigned long new = enable ? branch : nop;
unsigned long old = enable ? nop : branch;
unsigned long new = enable ? branch : nop;