arch/s390/kernel/kprobes.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  *  Kernel Probes (KProbes)
   4  *
   5  * Copyright IBM Corp. 2002, 2006
   6  *
   7  * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com>
   8  */
   9
  10 #define pr_fmt(fmt) "kprobes: " fmt
  11
  12 #include <linux/moduleloader.h>
  13 #include <linux/kprobes.h>
  14 #include <linux/ptrace.h>
  15 #include <linux/preempt.h>
  16 #include <linux/stop_machine.h>
  17 #include <linux/kdebug.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/extable.h>
  20 #include <linux/module.h>
  21 #include <linux/slab.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/ftrace.h>
  24 #include <asm/set_memory.h>
  25 #include <asm/sections.h>
  26 #include <asm/dis.h>
  27 #include "entry.h"
  28
  29 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
  30 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  31
  32 struct kretprobe_blackpoint kretprobe_blacklist[] = { };
  33
  34 DEFINE_INSN_CACHE_OPS(s390_insn);
  35
  36 static int insn_page_in_use;
  37
  38 void *alloc_insn_page(void)
  39 {
  40         void *page;
  41
  42         page = module_alloc(PAGE_SIZE);
  43         if (!page)
  44                 return NULL;
  45         __set_memory((unsigned long) page, 1, SET_MEMORY_RO | SET_MEMORY_X);
  46         return page;
  47 }
  48
  49 static void *alloc_s390_insn_page(void)
  50 {
  51         if (xchg(&insn_page_in_use, 1) == 1)
  52                 return NULL;
  53         return &kprobes_insn_page;
  54 }
  55
  56 static void free_s390_insn_page(void *page)
  57 {
  58         xchg(&insn_page_in_use, 0);
  59 }
  60
  61 struct kprobe_insn_cache kprobe_s390_insn_slots = {
  62         .mutex = __MUTEX_INITIALIZER(kprobe_s390_insn_slots.mutex),
  63         .alloc = alloc_s390_insn_page,
  64         .free = free_s390_insn_page,
  65         .pages = LIST_HEAD_INIT(kprobe_s390_insn_slots.pages),
  66         .insn_size = MAX_INSN_SIZE,
  67 };
  68
  69 static void copy_instruction(struct kprobe *p)
  70 {
  71         kprobe_opcode_t insn[MAX_INSN_SIZE];
  72         s64 disp, new_disp;
  73         u64 addr, new_addr;
  74         unsigned int len;
  75
  76         len = insn_length(*p->addr >> 8);
  77         memcpy(&insn, p->addr, len);
  78         p->opcode = insn[0];
  79         if (probe_is_insn_relative_long(&insn[0])) {
  80                 /*
  81                  * For pc-relative instructions in RIL-b or RIL-c format patch
  82                  * the RI2 displacement field. We have already made sure that
  83                  * the insn slot for the patched instruction is within the same
  84                  * 2GB area as the original instruction (either kernel image or
  85                  * module area). Therefore the new displacement will always fit.
  86                  */
  87                 disp = *(s32 *)&insn[1];
  88                 addr = (u64)(unsigned long)p->addr;
  89                 new_addr = (u64)(unsigned long)p->ainsn.insn;
  90                 new_disp = ((addr + (disp * 2)) - new_addr) / 2;
  91                 *(s32 *)&insn[1] = new_disp;
  92         }
  93         s390_kernel_write(p->ainsn.insn, &insn, len);
  94 }
  95 NOKPROBE_SYMBOL(copy_instruction);
  96
  97 static int s390_get_insn_slot(struct kprobe *p)
  98 {
  99         /*
 100          * Get an insn slot that is within the same 2GB area like the original
 101          * instruction. That way instructions with a 32bit signed displacement
 102          * field can be patched and executed within the insn slot.
 103          */
 104         p->ainsn.insn = NULL;
 105         if (is_kernel((unsigned long)p->addr))
 106                 p->ainsn.insn = get_s390_insn_slot();
 107         else if (is_module_addr(p->addr))
 108                 p->ainsn.insn = get_insn_slot();
 109         return p->ainsn.insn ? 0 : -ENOMEM;
 110 }
 111 NOKPROBE_SYMBOL(s390_get_insn_slot);
 112
 113 static void s390_free_insn_slot(struct kprobe *p)
 114 {
 115         if (!p->ainsn.insn)
 116                 return;
 117         if (is_kernel((unsigned long)p->addr))
 118                 free_s390_insn_slot(p->ainsn.insn, 0);
 119         else
 120                 free_insn_slot(p->ainsn.insn, 0);
 121         p->ainsn.insn = NULL;
 122 }
 123 NOKPROBE_SYMBOL(s390_free_insn_slot);
 124
 125 /* Check if paddr is at an instruction boundary */
 126 static bool can_probe(unsigned long paddr)
 127 {
 128         unsigned long addr, offset = 0;
 129         kprobe_opcode_t insn;
 130         struct kprobe *kp;
 131
 132         if (paddr & 0x01)
 133                 return false;
 134
 135         if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
 136                 return false;
 137
 138         /* Decode instructions */
 139         addr = paddr - offset;
 140         while (addr < paddr) {
 141                 if (copy_from_kernel_nofault(&insn, (void *)addr, sizeof(insn)))
 142                         return false;
 143
 144                 if (insn >> 8 == 0) {
 145                         if (insn != BREAKPOINT_INSTRUCTION) {
 146                                 /*
 147                                  * Note that QEMU inserts opcode 0x0000 to implement
 148                                  * software breakpoints for guests. Since the size of
 149                                  * the original instruction is unknown, stop following
 150                                  * instructions and prevent setting a kprobe.
 151                                  */
 152                                 return false;
 153                         }
 154                         /*
 155                          * Check if the instruction has been modified by another
 156                          * kprobe, in which case the original instruction is
 157                          * decoded.
 158                          */
 159                         kp = get_kprobe((void *)addr);
 160                         if (!kp) {
 161                                 /* not a kprobe */
 162                                 return false;
 163                         }
 164                         insn = kp->opcode;
 165                 }
 166                 addr += insn_length(insn >> 8);
 167         }
 168         return addr == paddr;
 169 }
 170
 171 int arch_prepare_kprobe(struct kprobe *p)
 172 {
 173         if (!can_probe((unsigned long)p->addr))
 174                 return -EINVAL;
 175         /* Make sure the probe isn't going on a difficult instruction */
 176         if (probe_is_prohibited_opcode(p->addr))
 177                 return -EINVAL;
 178         if (s390_get_insn_slot(p))
 179                 return -ENOMEM;
 180         copy_instruction(p);
 181         return 0;
 182 }
 183 NOKPROBE_SYMBOL(arch_prepare_kprobe);
 184
 185 struct swap_insn_args {
 186         struct kprobe *p;
 187         unsigned int arm_kprobe : 1;
 188 };
 189
 190 static int swap_instruction(void *data)
 191 {
 192         struct swap_insn_args *args = data;
 193         struct kprobe *p = args->p;
 194         u16 opc;
 195
 196         opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
 197         s390_kernel_write(p->addr, &opc, sizeof(opc));
 198         return 0;
 199 }
 200 NOKPROBE_SYMBOL(swap_instruction);
 201
 202 void arch_arm_kprobe(struct kprobe *p)
 203 {
 204         struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
 205
 206         stop_machine_cpuslocked(swap_instruction, &args, NULL);
 207 }
 208 NOKPROBE_SYMBOL(arch_arm_kprobe);
 209
 210 void arch_disarm_kprobe(struct kprobe *p)
 211 {
 212         struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
 213
 214         stop_machine_cpuslocked(swap_instruction, &args, NULL);
 215 }
 216 NOKPROBE_SYMBOL(arch_disarm_kprobe);
 217
 218 void arch_remove_kprobe(struct kprobe *p)
 219 {
 220         s390_free_insn_slot(p);
 221 }
 222 NOKPROBE_SYMBOL(arch_remove_kprobe);
 223
 224 static void enable_singlestep(struct kprobe_ctlblk *kcb,
 225                               struct pt_regs *regs,
 226                               unsigned long ip)
 227 {
 228         struct per_regs per_kprobe;
 229
 230         /* Set up the PER control registers %cr9-%cr11 */
 231         per_kprobe.control = PER_EVENT_IFETCH;
 232         per_kprobe.start = ip;
 233         per_kprobe.end = ip;
 234
 235         /* Save control regs and psw mask */
 236         __ctl_store(kcb->kprobe_saved_ctl, 9, 11);
 237         kcb->kprobe_saved_imask = regs->psw.mask &
 238                 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
 239
 240         /* Set PER control regs, turns on single step for the given address */
 241         __ctl_load(per_kprobe, 9, 11);
 242         regs->psw.mask |= PSW_MASK_PER;
 243         regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
 244         regs->psw.addr = ip;
 245 }
 246 NOKPROBE_SYMBOL(enable_singlestep);
 247
 248 static void disable_singlestep(struct kprobe_ctlblk *kcb,
 249                                struct pt_regs *regs,
 250                                unsigned long ip)
 251 {
 252         /* Restore control regs and psw mask, set new psw address */
 253         __ctl_load(kcb->kprobe_saved_ctl, 9, 11);
 254         regs->psw.mask &= ~PSW_MASK_PER;
 255         regs->psw.mask |= kcb->kprobe_saved_imask;
 256         regs->psw.addr = ip;
 257 }
 258 NOKPROBE_SYMBOL(disable_singlestep);
 259
 260 /*
 261  * Activate a kprobe by storing its pointer to current_kprobe. The
 262  * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
 263  * two kprobes can be active, see KPROBE_REENTER.
 264  */
 265 static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
 266 {
 267         kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
 268         kcb->prev_kprobe.status = kcb->kprobe_status;
 269         __this_cpu_write(current_kprobe, p);
 270 }
 271 NOKPROBE_SYMBOL(push_kprobe);
 272
 273 /*
 274  * Deactivate a kprobe by backing up to the previous state. If the
 275  * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
 276  * for any other state prev_kprobe.kp will be NULL.
 277  */
 278 static void pop_kprobe(struct kprobe_ctlblk *kcb)
 279 {
 280         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
 281         kcb->kprobe_status = kcb->prev_kprobe.status;
 282 }
 283 NOKPROBE_SYMBOL(pop_kprobe);
 284
 285 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
 286 {
 287         ri->ret_addr = (kprobe_opcode_t *)regs->gprs[14];
 288         ri->fp = (void *)regs->gprs[15];
 289
 290         /* Replace the return addr with trampoline addr */
 291         regs->gprs[14] = (unsigned long)&__kretprobe_trampoline;
 292 }
 293 NOKPROBE_SYMBOL(arch_prepare_kretprobe);
 294
 295 static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
 296 {
 297         switch (kcb->kprobe_status) {
 298         case KPROBE_HIT_SSDONE:
 299         case KPROBE_HIT_ACTIVE:
 300                 kprobes_inc_nmissed_count(p);
 301                 break;
 302         case KPROBE_HIT_SS:
 303         case KPROBE_REENTER:
 304         default:
 305                 /*
 306                  * A kprobe on the code path to single step an instruction
 307                  * is a BUG. The code path resides in the .kprobes.text
 308                  * section and is executed with interrupts disabled.
 309                  */
 310                 pr_err("Failed to recover from reentered kprobes.\n");
 311                 dump_kprobe(p);
 312                 BUG();
 313         }
 314 }
 315 NOKPROBE_SYMBOL(kprobe_reenter_check);
 316
 317 static int kprobe_handler(struct pt_regs *regs)
 318 {
 319         struct kprobe_ctlblk *kcb;
 320         struct kprobe *p;
 321
 322         /*
 323          * We want to disable preemption for the entire duration of kprobe
 324          * processing. That includes the calls to the pre/post handlers
 325          * and single stepping the kprobe instruction.
 326          */
 327         preempt_disable();
 328         kcb = get_kprobe_ctlblk();
 329         p = get_kprobe((void *)(regs->psw.addr - 2));
 330
 331         if (p) {
 332                 if (kprobe_running()) {
 333                         /*
 334                          * We have hit a kprobe while another is still
 335                          * active. This can happen in the pre and post
 336                          * handler. Single step the instruction of the
 337                          * new probe but do not call any handler function
 338                          * of this secondary kprobe.
 339                          * push_kprobe and pop_kprobe saves and restores
 340                          * the currently active kprobe.
 341                          */
 342                         kprobe_reenter_check(kcb, p);
 343                         push_kprobe(kcb, p);
 344                         kcb->kprobe_status = KPROBE_REENTER;
 345                 } else {
 346                         /*
 347                          * If we have no pre-handler or it returned 0, we
 348                          * continue with single stepping. If we have a
 349                          * pre-handler and it returned non-zero, it prepped
 350                          * for changing execution path, so get out doing
 351                          * nothing more here.
 352                          */
 353                         push_kprobe(kcb, p);
 354                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 355                         if (p->pre_handler && p->pre_handler(p, regs)) {
 356                                 pop_kprobe(kcb);
 357                                 preempt_enable_no_resched();
 358                                 return 1;
 359                         }
 360                         kcb->kprobe_status = KPROBE_HIT_SS;
 361                 }
 362                 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
 363                 return 1;
 364         } /* else:
 365            * No kprobe at this address and no active kprobe. The trap has
 366            * not been caused by a kprobe breakpoint. The race of breakpoint
 367            * vs. kprobe remove does not exist because on s390 as we use
 368            * stop_machine to arm/disarm the breakpoints.
 369            */
 370         preempt_enable_no_resched();
 371         return 0;
 372 }
 373 NOKPROBE_SYMBOL(kprobe_handler);
 374
 375 void arch_kretprobe_fixup_return(struct pt_regs *regs,
 376                                  kprobe_opcode_t *correct_ret_addr)
 377 {
 378         /* Replace fake return address with real one. */
 379         regs->gprs[14] = (unsigned long)correct_ret_addr;
 380 }
 381 NOKPROBE_SYMBOL(arch_kretprobe_fixup_return);
 382
 383 /*
 384  * Called from __kretprobe_trampoline
 385  */
 386 void trampoline_probe_handler(struct pt_regs *regs)
 387 {
 388         kretprobe_trampoline_handler(regs, (void *)regs->gprs[15]);
 389 }
 390 NOKPROBE_SYMBOL(trampoline_probe_handler);
 391
 392 /* assembler function that handles the kretprobes must not be probed itself */
 393 NOKPROBE_SYMBOL(__kretprobe_trampoline);
 394
 395 /*
 396  * Called after single-stepping.  p->addr is the address of the
 397  * instruction whose first byte has been replaced by the "breakpoint"
 398  * instruction.  To avoid the SMP problems that can occur when we
 399  * temporarily put back the original opcode to single-step, we
 400  * single-stepped a copy of the instruction.  The address of this
 401  * copy is p->ainsn.insn.
 402  */
 403 static void resume_execution(struct kprobe *p, struct pt_regs *regs)
 404 {
 405         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 406         unsigned long ip = regs->psw.addr;
 407         int fixup = probe_get_fixup_type(p->ainsn.insn);
 408
 409         if (fixup & FIXUP_PSW_NORMAL)
 410                 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
 411
 412         if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
 413                 int ilen = insn_length(p->ainsn.insn[0] >> 8);
 414                 if (ip - (unsigned long) p->ainsn.insn == ilen)
 415                         ip = (unsigned long) p->addr + ilen;
 416         }
 417
 418         if (fixup & FIXUP_RETURN_REGISTER) {
 419                 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
 420                 regs->gprs[reg] += (unsigned long) p->addr -
 421                                    (unsigned long) p->ainsn.insn;
 422         }
 423
 424         disable_singlestep(kcb, regs, ip);
 425 }
 426 NOKPROBE_SYMBOL(resume_execution);
 427
 428 static int post_kprobe_handler(struct pt_regs *regs)
 429 {
 430         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 431         struct kprobe *p = kprobe_running();
 432
 433         if (!p)
 434                 return 0;
 435
 436         if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
 437                 kcb->kprobe_status = KPROBE_HIT_SSDONE;
 438                 p->post_handler(p, regs, 0);
 439         }
 440
 441         resume_execution(p, regs);
 442         pop_kprobe(kcb);
 443         preempt_enable_no_resched();
 444
 445         /*
 446          * if somebody else is singlestepping across a probe point, psw mask
 447          * will have PER set, in which case, continue the remaining processing
 448          * of do_single_step, as if this is not a probe hit.
 449          */
 450         if (regs->psw.mask & PSW_MASK_PER)
 451                 return 0;
 452
 453         return 1;
 454 }
 455 NOKPROBE_SYMBOL(post_kprobe_handler);
 456
 457 static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
 458 {
 459         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 460         struct kprobe *p = kprobe_running();
 461
 462         switch(kcb->kprobe_status) {
 463         case KPROBE_HIT_SS:
 464         case KPROBE_REENTER:
 465                 /*
 466                  * We are here because the instruction being single
 467                  * stepped caused a page fault. We reset the current
 468                  * kprobe and the nip points back to the probe address
 469                  * and allow the page fault handler to continue as a
 470                  * normal page fault.
 471                  */
 472                 disable_singlestep(kcb, regs, (unsigned long) p->addr);
 473                 pop_kprobe(kcb);
 474                 preempt_enable_no_resched();
 475                 break;
 476         case KPROBE_HIT_ACTIVE:
 477         case KPROBE_HIT_SSDONE:
 478                 /*
 479                  * In case the user-specified fault handler returned
 480                  * zero, try to fix up.
 481                  */
 482                 if (fixup_exception(regs))
 483                         return 1;
 484                 /*
 485                  * fixup_exception() could not handle it,
 486                  * Let do_page_fault() fix it.
 487                  */
 488                 break;
 489         default:
 490                 break;
 491         }
 492         return 0;
 493 }
 494 NOKPROBE_SYMBOL(kprobe_trap_handler);
 495
 496 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 497 {
 498         int ret;
 499
 500         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 501                 local_irq_disable();
 502         ret = kprobe_trap_handler(regs, trapnr);
 503         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 504                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 505         return ret;
 506 }
 507 NOKPROBE_SYMBOL(kprobe_fault_handler);
 508
 509 /*
 510  * Wrapper routine to for handling exceptions.
 511  */
 512 int kprobe_exceptions_notify(struct notifier_block *self,
 513                              unsigned long val, void *data)
 514 {
 515         struct die_args *args = (struct die_args *) data;
 516         struct pt_regs *regs = args->regs;
 517         int ret = NOTIFY_DONE;
 518
 519         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 520                 local_irq_disable();
 521
 522         switch (val) {
 523         case DIE_BPT:
 524                 if (kprobe_handler(regs))
 525                         ret = NOTIFY_STOP;
 526                 break;
 527         case DIE_SSTEP:
 528                 if (post_kprobe_handler(regs))
 529                         ret = NOTIFY_STOP;
 530                 break;
 531         case DIE_TRAP:
 532                 if (!preemptible() && kprobe_running() &&
 533                     kprobe_trap_handler(regs, args->trapnr))
 534                         ret = NOTIFY_STOP;
 535                 break;
 536         default:
 537                 break;
 538         }
 539
 540         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 541                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 542
 543         return ret;
 544 }
 545 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
 546
 547 int __init arch_init_kprobes(void)
 548 {
 549         return 0;
 550 }
 551
 552 int arch_trampoline_kprobe(struct kprobe *p)
 553 {
 554         return 0;
 555 }
 556 NOKPROBE_SYMBOL(arch_trampoline_kprobe);