kprobes/ftrace: Add recursion protection to the ftrace callback

If a ftrace callback does not supply its own recursion protection and
does not set the RECURSION_SAFE flag in its ftrace_ops, then ftrace will
make a helper trampoline to do so before calling the callback instead of
just calling the callback directly.

The default for ftrace_ops is going to change. It will expect that handlers
provide their own recursion protection, unless its ftrace_ops states
otherwise.

Link: https://lkml.kernel.org/r/20201028115613.140212174@goodmis.org
Link: https://lkml.kernel.org/r/20201106023546.944907560@goodmis.org
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Josh  Poimboeuf <jpoimboe@redhat.com>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Miroslav Benes <mbenes@suse.cz>
Cc: Petr Mladek <pmladek@suse.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Guo Ren <guoren@kernel.org>
Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
Cc: Helge Deller <deller@gmx.de>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Christian Borntraeger <borntraeger@de.ibm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: x86@kernel.org
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Naveen N. Rao" <naveen.n.rao@linux.ibm.com>
Cc: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: linux-csky@vger.kernel.org
Cc: linux-parisc@vger.kernel.org
Cc: linuxppc-dev@lists.ozlabs.org
Cc: linux-s390@vger.kernel.org
Acked-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>

diff --git a/arch/csky/kernel/probes/ftrace.c b/arch/csky/kernel/probes/ftrace.c
index 5264763..5eb2604 100644 (file)
--- a/arch/csky/kernel/probes/ftrace.c
+++ b/arch/csky/kernel/probes/ftrace.c
@@ -13,16 +13,21 @@ int arch_check_ftrace_location(struct kprobe *p)
 void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
                           struct ftrace_ops *ops, struct pt_regs *regs)
 {
+       int bit;
        bool lr_saver = false;
        struct kprobe *p;
        struct kprobe_ctlblk *kcb;
 
-       /* Preempt is disabled by ftrace */
+       bit = ftrace_test_recursion_trylock();
+       if (bit < 0)
+               return;
+
+       preempt_disable_notrace();
        p = get_kprobe((kprobe_opcode_t *)ip);
        if (!p) {
                p = get_kprobe((kprobe_opcode_t *)(ip - MCOUNT_INSN_SIZE));
                if (unlikely(!p) || kprobe_disabled(p))
-                       return;
+                       goto out;
                lr_saver = true;
        }
 
@@ -56,6 +61,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
                 */
                __this_cpu_write(current_kprobe, NULL);
        }
+out:
+       preempt_enable_notrace();
+       ftrace_test_recursion_unlock(bit);
 }
 NOKPROBE_SYMBOL(kprobe_ftrace_handler);
 

diff --git a/arch/parisc/kernel/ftrace.c b/arch/parisc/kernel/ftrace.c
index 63e3ecb..13d8504 100644 (file)
--- a/arch/parisc/kernel/ftrace.c
+++ b/arch/parisc/kernel/ftrace.c
@@ -207,14 +207,21 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
                           struct ftrace_ops *ops, struct pt_regs *regs)
 {
        struct kprobe_ctlblk *kcb;
-       struct kprobe *p = get_kprobe((kprobe_opcode_t *)ip);
+       struct kprobe *p;
+       int bit;
 
-       if (unlikely(!p) || kprobe_disabled(p))
+       bit = ftrace_test_recursion_trylock();
+       if (bit < 0)
                return;
 
+       preempt_disable_notrace();
+       p = get_kprobe((kprobe_opcode_t *)ip);
+       if (unlikely(!p) || kprobe_disabled(p))
+               goto out;
+
        if (kprobe_running()) {
                kprobes_inc_nmissed_count(p);
-               return;
+               goto out;
        }
 
        __this_cpu_write(current_kprobe, p);
@@ -235,6 +242,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
                }
        }
        __this_cpu_write(current_kprobe, NULL);
+out:
+       preempt_enable_notrace();
+       ftrace_test_recursion_unlock(bit);
 }
 NOKPROBE_SYMBOL(kprobe_ftrace_handler);
 

diff --git a/arch/powerpc/kernel/kprobes-ftrace.c b/arch/powerpc/kernel/kprobes-ftrace.c
index 972cb28..5df8d50 100644 (file)
--- a/arch/powerpc/kernel/kprobes-ftrace.c
+++ b/arch/powerpc/kernel/kprobes-ftrace.c
@@ -18,10 +18,16 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,
 {
        struct kprobe *p;
        struct kprobe_ctlblk *kcb;
+       int bit;
 
+       bit = ftrace_test_recursion_trylock();
+       if (bit < 0)
+               return;
+
+       preempt_disable_notrace();
        p = get_kprobe((kprobe_opcode_t *)nip);
        if (unlikely(!p) || kprobe_disabled(p))
-               return;
+               goto out;
 
        kcb = get_kprobe_ctlblk();
        if (kprobe_running()) {
@@ -52,6 +58,9 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,
                 */
                __this_cpu_write(current_kprobe, NULL);
        }
+out:
+       preempt_enable_notrace();
+       ftrace_test_recursion_unlock(bit);
 }
 NOKPROBE_SYMBOL(kprobe_ftrace_handler);
 

diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c
index b388e87..8f31c72 100644 (file)
--- a/arch/s390/kernel/ftrace.c
+++ b/arch/s390/kernel/ftrace.c
@@ -201,14 +201,21 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
                struct ftrace_ops *ops, struct pt_regs *regs)
 {
        struct kprobe_ctlblk *kcb;
-       struct kprobe *p = get_kprobe((kprobe_opcode_t *)ip);
+       struct kprobe *p;
+       int bit;
 
-       if (unlikely(!p) || kprobe_disabled(p))
+       bit = ftrace_test_recursion_trylock();
+       if (bit < 0)
                return;
 
+       preempt_disable_notrace();
+       p = get_kprobe((kprobe_opcode_t *)ip);
+       if (unlikely(!p) || kprobe_disabled(p))
+               goto out;
+
        if (kprobe_running()) {
                kprobes_inc_nmissed_count(p);
-               return;
+               goto out;
        }
 
        __this_cpu_write(current_kprobe, p);
@@ -228,6 +235,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
                }
        }
        __this_cpu_write(current_kprobe, NULL);
+out:
+       preempt_enable_notrace();
+       ftrace_test_recursion_unlock(bit);
 }
 NOKPROBE_SYMBOL(kprobe_ftrace_handler);
 

diff --git a/arch/x86/kernel/kprobes/ftrace.c b/arch/x86/kernel/kprobes/ftrace.c
index 681a4b3..a40a6cd 100644 (file)
--- a/arch/x86/kernel/kprobes/ftrace.c
+++ b/arch/x86/kernel/kprobes/ftrace.c
@@ -18,11 +18,16 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
 {
        struct kprobe *p;
        struct kprobe_ctlblk *kcb;
+       int bit;
 
-       /* Preempt is disabled by ftrace */
+       bit = ftrace_test_recursion_trylock();
+       if (bit < 0)
+               return;
+
+       preempt_disable_notrace();
        p = get_kprobe((kprobe_opcode_t *)ip);
        if (unlikely(!p) || kprobe_disabled(p))
-               return;
+               goto out;
 
        kcb = get_kprobe_ctlblk();
        if (kprobe_running()) {
@@ -52,6 +57,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
                 */
                __this_cpu_write(current_kprobe, NULL);
        }
+out:
+       preempt_enable_notrace();
+       ftrace_test_recursion_unlock(bit);
 }
 NOKPROBE_SYMBOL(kprobe_ftrace_handler);