arch/x86/kernel/rethook.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * x86 implementation of rethook. Mostly copied from arch/x86/kernel/kprobes/core.c.
   4  */
   5 #include <linux/bug.h>
   6 #include <linux/rethook.h>
   7 #include <linux/kprobes.h>
   8 #include <linux/objtool.h>
   9
  10 #include "kprobes/common.h"
  11
  12 __visible void arch_rethook_trampoline_callback(struct pt_regs *regs);
  13
  14 #ifndef ANNOTATE_NOENDBR
  15 #define ANNOTATE_NOENDBR
  16 #endif
  17
  18 /*
  19  * When a target function returns, this code saves registers and calls
  20  * arch_rethook_trampoline_callback(), which calls the rethook handler.
  21  */
  22 asm(
  23         ".text\n"
  24         ".global arch_rethook_trampoline\n"
  25         ".type arch_rethook_trampoline, @function\n"
  26         "arch_rethook_trampoline:\n"
  27 #ifdef CONFIG_X86_64
  28         ANNOTATE_NOENDBR        /* This is only jumped from ret instruction */
  29         /* Push a fake return address to tell the unwinder it's a rethook. */
  30         "       pushq $arch_rethook_trampoline\n"
  31         UNWIND_HINT_FUNC
  32         "       pushq $" __stringify(__KERNEL_DS) "\n"
  33         /* Save the 'sp - 16', this will be fixed later. */
  34         "       pushq %rsp\n"
  35         "       pushfq\n"
  36         SAVE_REGS_STRING
  37         "       movq %rsp, %rdi\n"
  38         "       call arch_rethook_trampoline_callback\n"
  39         RESTORE_REGS_STRING
  40         /* In the callback function, 'regs->flags' is copied to 'regs->ss'. */
  41         "       addq $16, %rsp\n"
  42         "       popfq\n"
  43 #else
  44         /* Push a fake return address to tell the unwinder it's a rethook. */
  45         "       pushl $arch_rethook_trampoline\n"
  46         UNWIND_HINT_FUNC
  47         "       pushl %ss\n"
  48         /* Save the 'sp - 8', this will be fixed later. */
  49         "       pushl %esp\n"
  50         "       pushfl\n"
  51         SAVE_REGS_STRING
  52         "       movl %esp, %eax\n"
  53         "       call arch_rethook_trampoline_callback\n"
  54         RESTORE_REGS_STRING
  55         /* In the callback function, 'regs->flags' is copied to 'regs->ss'. */
  56         "       addl $8, %esp\n"
  57         "       popfl\n"
  58 #endif
  59         ASM_RET
  60         ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n"
  61 );
  62 NOKPROBE_SYMBOL(arch_rethook_trampoline);
  63
  64 /*
  65  * Called from arch_rethook_trampoline
  66  */
  67 __used __visible void arch_rethook_trampoline_callback(struct pt_regs *regs)
  68 {
  69         unsigned long *frame_pointer;
  70
  71         /* fixup registers */
  72         regs->cs = __KERNEL_CS;
  73 #ifdef CONFIG_X86_32
  74         regs->gs = 0;
  75 #endif
  76         regs->ip = (unsigned long)&arch_rethook_trampoline;
  77         regs->orig_ax = ~0UL;
  78         regs->sp += 2*sizeof(long);
  79         frame_pointer = (long *)(regs + 1);
  80
  81         /*
  82          * The return address at 'frame_pointer' is recovered by the
  83          * arch_rethook_fixup_return() which called from this
  84          * rethook_trampoline_handler().
  85          */
  86         rethook_trampoline_handler(regs, (unsigned long)frame_pointer);
  87
  88         /*
  89          * Copy FLAGS to 'pt_regs::ss' so that arch_rethook_trapmoline()
  90          * can do RET right after POPF.
  91          */
  92         *(unsigned long *)&regs->ss = regs->flags;
  93 }
  94 NOKPROBE_SYMBOL(arch_rethook_trampoline_callback);
  95
  96 /*
  97  * arch_rethook_trampoline() skips updating frame pointer. The frame pointer
  98  * saved in arch_rethook_trampoline_callback() points to the real caller
  99  * function's frame pointer. Thus the arch_rethook_trampoline() doesn't have
 100  * a standard stack frame with CONFIG_FRAME_POINTER=y.
 101  * Let's mark it non-standard function. Anyway, FP unwinder can correctly
 102  * unwind without the hint.
 103  */
 104 STACK_FRAME_NON_STANDARD_FP(arch_rethook_trampoline);
 105
 106 /* This is called from rethook_trampoline_handler(). */
 107 void arch_rethook_fixup_return(struct pt_regs *regs,
 108                                unsigned long correct_ret_addr)
 109 {
 110         unsigned long *frame_pointer = (void *)(regs + 1);
 111
 112         /* Replace fake return address with real one. */
 113         *frame_pointer = correct_ret_addr;
 114 }
 115 NOKPROBE_SYMBOL(arch_rethook_fixup_return);
 116
 117 void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount)
 118 {
 119         unsigned long *stack = (unsigned long *)regs->sp;
 120
 121         rh->ret_addr = stack[0];
 122         rh->frame = regs->sp;
 123
 124         /* Replace the return addr with trampoline addr */
 125         stack[0] = (unsigned long) arch_rethook_trampoline;
 126 }
 127 NOKPROBE_SYMBOL(arch_rethook_prepare);