arch/arm64/kernel/traps.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Based on arch/arm/kernel/traps.c
   4  *
   5  * Copyright (C) 1995-2009 Russell King
   6  * Copyright (C) 2012 ARM Ltd.
   7  */
   8
   9 #include <linux/bug.h>
  10 #include <linux/context_tracking.h>
  11 #include <linux/signal.h>
  12 #include <linux/personality.h>
  13 #include <linux/kallsyms.h>
  14 #include <linux/kprobes.h>
  15 #include <linux/spinlock.h>
  16 #include <linux/uaccess.h>
  17 #include <linux/hardirq.h>
  18 #include <linux/kdebug.h>
  19 #include <linux/module.h>
  20 #include <linux/kexec.h>
  21 #include <linux/delay.h>
  22 #include <linux/init.h>
  23 #include <linux/sched/signal.h>
  24 #include <linux/sched/debug.h>
  25 #include <linux/sched/task_stack.h>
  26 #include <linux/sizes.h>
  27 #include <linux/syscalls.h>
  28 #include <linux/mm_types.h>
  29 #include <linux/kasan.h>
  30
  31 #include <asm/atomic.h>
  32 #include <asm/bug.h>
  33 #include <asm/cpufeature.h>
  34 #include <asm/daifflags.h>
  35 #include <asm/debug-monitors.h>
  36 #include <asm/esr.h>
  37 #include <asm/insn.h>
  38 #include <asm/kprobes.h>
  39 #include <asm/traps.h>
  40 #include <asm/smp.h>
  41 #include <asm/stack_pointer.h>
  42 #include <asm/stacktrace.h>
  43 #include <asm/exception.h>
  44 #include <asm/system_misc.h>
  45 #include <asm/sysreg.h>
  46
  47 static const char *handler[]= {
  48         "Synchronous Abort",
  49         "IRQ",
  50         "FIQ",
  51         "Error"
  52 };
  53
  54 int show_unhandled_signals = 0;
  55
  56 static void dump_backtrace_entry(unsigned long where, const char *loglvl)
  57 {
  58         printk("%s %pS\n", loglvl, (void *)where);
  59 }
  60
  61 static void dump_kernel_instr(const char *lvl, struct pt_regs *regs)
  62 {
  63         unsigned long addr = instruction_pointer(regs);
  64         char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str;
  65         int i;
  66
  67         if (user_mode(regs))
  68                 return;
  69
  70         for (i = -4; i < 1; i++) {
  71                 unsigned int val, bad;
  72
  73                 bad = aarch64_insn_read(&((u32 *)addr)[i], &val);
  74
  75                 if (!bad)
  76                         p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val);
  77                 else {
  78                         p += sprintf(p, "bad PC value");
  79                         break;
  80                 }
  81         }
  82
  83         printk("%sCode: %s\n", lvl, str);
  84 }
  85
  86 void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
  87                     const char *loglvl)
  88 {
  89         struct stackframe frame;
  90         int skip = 0;
  91
  92         pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
  93
  94         if (regs) {
  95                 if (user_mode(regs))
  96                         return;
  97                 skip = 1;
  98         }
  99
 100         if (!tsk)
 101                 tsk = current;
 102
 103         if (!try_get_task_stack(tsk))
 104                 return;
 105
 106         if (tsk == current) {
 107                 start_backtrace(&frame,
 108                                 (unsigned long)__builtin_frame_address(0),
 109                                 (unsigned long)dump_backtrace);
 110         } else {
 111                 /*
 112                  * task blocked in __switch_to
 113                  */
 114                 start_backtrace(&frame,
 115                                 thread_saved_fp(tsk),
 116                                 thread_saved_pc(tsk));
 117         }
 118
 119         printk("%sCall trace:\n", loglvl);
 120         do {
 121                 /* skip until specified stack frame */
 122                 if (!skip) {
 123                         dump_backtrace_entry(frame.pc, loglvl);
 124                 } else if (frame.fp == regs->regs[29]) {
 125                         skip = 0;
 126                         /*
 127                          * Mostly, this is the case where this function is
 128                          * called in panic/abort. As exception handler's
 129                          * stack frame does not contain the corresponding pc
 130                          * at which an exception has taken place, use regs->pc
 131                          * instead.
 132                          */
 133                         dump_backtrace_entry(regs->pc, loglvl);
 134                 }
 135         } while (!unwind_frame(tsk, &frame));
 136
 137         put_task_stack(tsk);
 138 }
 139
 140 void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
 141 {
 142         dump_backtrace(NULL, tsk, loglvl);
 143         barrier();
 144 }
 145
 146 #ifdef CONFIG_PREEMPT
 147 #define S_PREEMPT " PREEMPT"
 148 #elif defined(CONFIG_PREEMPT_RT)
 149 #define S_PREEMPT " PREEMPT_RT"
 150 #else
 151 #define S_PREEMPT ""
 152 #endif
 153
 154 #define S_SMP " SMP"
 155
 156 static int __die(const char *str, int err, struct pt_regs *regs)
 157 {
 158         static int die_counter;
 159         int ret;
 160
 161         pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n",
 162                  str, err, ++die_counter);
 163
 164         /* trap and error numbers are mostly meaningless on ARM */
 165         ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV);
 166         if (ret == NOTIFY_STOP)
 167                 return ret;
 168
 169         print_modules();
 170         show_regs(regs);
 171
 172         dump_kernel_instr(KERN_EMERG, regs);
 173
 174         return ret;
 175 }
 176
 177 static DEFINE_RAW_SPINLOCK(die_lock);
 178
 179 /*
 180  * This function is protected against re-entrancy.
 181  */
 182 void die(const char *str, struct pt_regs *regs, int err)
 183 {
 184         int ret;
 185         unsigned long flags;
 186
 187         raw_spin_lock_irqsave(&die_lock, flags);
 188
 189         oops_enter();
 190
 191         console_verbose();
 192         bust_spinlocks(1);
 193         ret = __die(str, err, regs);
 194
 195         if (regs && kexec_should_crash(current))
 196                 crash_kexec(regs);
 197
 198         bust_spinlocks(0);
 199         add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
 200         oops_exit();
 201
 202         if (in_interrupt())
 203                 panic("Fatal exception in interrupt");
 204         if (panic_on_oops)
 205                 panic("Fatal exception");
 206
 207         raw_spin_unlock_irqrestore(&die_lock, flags);
 208
 209         if (ret != NOTIFY_STOP)
 210                 do_exit(SIGSEGV);
 211 }
 212
 213 static void arm64_show_signal(int signo, const char *str)
 214 {
 215         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 216                                       DEFAULT_RATELIMIT_BURST);
 217         struct task_struct *tsk = current;
 218         unsigned int esr = tsk->thread.fault_code;
 219         struct pt_regs *regs = task_pt_regs(tsk);
 220
 221         /* Leave if the signal won't be shown */
 222         if (!show_unhandled_signals ||
 223             !unhandled_signal(tsk, signo) ||
 224             !__ratelimit(&rs))
 225                 return;
 226
 227         pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk));
 228         if (esr)
 229                 pr_cont("%s, ESR 0x%08x, ", esr_get_class_string(esr), esr);
 230
 231         pr_cont("%s", str);
 232         print_vma_addr(KERN_CONT " in ", regs->pc);
 233         pr_cont("\n");
 234         __show_regs(regs);
 235 }
 236
 237 void arm64_force_sig_fault(int signo, int code, void __user *addr,
 238                            const char *str)
 239 {
 240         arm64_show_signal(signo, str);
 241         if (signo == SIGKILL)
 242                 force_sig(SIGKILL);
 243         else
 244                 force_sig_fault(signo, code, addr);
 245 }
 246
 247 void arm64_force_sig_mceerr(int code, void __user *addr, short lsb,
 248                             const char *str)
 249 {
 250         arm64_show_signal(SIGBUS, str);
 251         force_sig_mceerr(code, addr, lsb);
 252 }
 253
 254 void arm64_force_sig_ptrace_errno_trap(int errno, void __user *addr,
 255                                        const char *str)
 256 {
 257         arm64_show_signal(SIGTRAP, str);
 258         force_sig_ptrace_errno_trap(errno, addr);
 259 }
 260
 261 void arm64_notify_die(const char *str, struct pt_regs *regs,
 262                       int signo, int sicode, void __user *addr,
 263                       int err)
 264 {
 265         if (user_mode(regs)) {
 266                 WARN_ON(regs != current_pt_regs());
 267                 current->thread.fault_address = 0;
 268                 current->thread.fault_code = err;
 269
 270                 arm64_force_sig_fault(signo, sicode, addr, str);
 271         } else {
 272                 die(str, regs, err);
 273         }
 274 }
 275
 276 #ifdef CONFIG_COMPAT
 277 #define PSTATE_IT_1_0_SHIFT     25
 278 #define PSTATE_IT_1_0_MASK      (0x3 << PSTATE_IT_1_0_SHIFT)
 279 #define PSTATE_IT_7_2_SHIFT     10
 280 #define PSTATE_IT_7_2_MASK      (0x3f << PSTATE_IT_7_2_SHIFT)
 281
 282 static u32 compat_get_it_state(struct pt_regs *regs)
 283 {
 284         u32 it, pstate = regs->pstate;
 285
 286         it  = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT;
 287         it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2;
 288
 289         return it;
 290 }
 291
 292 static void compat_set_it_state(struct pt_regs *regs, u32 it)
 293 {
 294         u32 pstate_it;
 295
 296         pstate_it  = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK;
 297         pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK;
 298
 299         regs->pstate &= ~PSR_AA32_IT_MASK;
 300         regs->pstate |= pstate_it;
 301 }
 302
 303 static void advance_itstate(struct pt_regs *regs)
 304 {
 305         u32 it;
 306
 307         /* ARM mode */
 308         if (!(regs->pstate & PSR_AA32_T_BIT) ||
 309             !(regs->pstate & PSR_AA32_IT_MASK))
 310                 return;
 311
 312         it  = compat_get_it_state(regs);
 313
 314         /*
 315          * If this is the last instruction of the block, wipe the IT
 316          * state. Otherwise advance it.
 317          */
 318         if (!(it & 7))
 319                 it = 0;
 320         else
 321                 it = (it & 0xe0) | ((it << 1) & 0x1f);
 322
 323         compat_set_it_state(regs, it);
 324 }
 325 #else
 326 static void advance_itstate(struct pt_regs *regs)
 327 {
 328 }
 329 #endif
 330
 331 void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
 332 {
 333         regs->pc += size;
 334
 335         /*
 336          * If we were single stepping, we want to get the step exception after
 337          * we return from the trap.
 338          */
 339         if (user_mode(regs))
 340                 user_fastforward_single_step(current);
 341
 342         if (compat_user_mode(regs))
 343                 advance_itstate(regs);
 344         else
 345                 regs->pstate &= ~PSR_BTYPE_MASK;
 346 }
 347
 348 static LIST_HEAD(undef_hook);
 349 static DEFINE_RAW_SPINLOCK(undef_lock);
 350
 351 void register_undef_hook(struct undef_hook *hook)
 352 {
 353         unsigned long flags;
 354
 355         raw_spin_lock_irqsave(&undef_lock, flags);
 356         list_add(&hook->node, &undef_hook);
 357         raw_spin_unlock_irqrestore(&undef_lock, flags);
 358 }
 359
 360 void unregister_undef_hook(struct undef_hook *hook)
 361 {
 362         unsigned long flags;
 363
 364         raw_spin_lock_irqsave(&undef_lock, flags);
 365         list_del(&hook->node);
 366         raw_spin_unlock_irqrestore(&undef_lock, flags);
 367 }
 368
 369 static int call_undef_hook(struct pt_regs *regs)
 370 {
 371         struct undef_hook *hook;
 372         unsigned long flags;
 373         u32 instr;
 374         int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
 375         void __user *pc = (void __user *)instruction_pointer(regs);
 376
 377         if (!user_mode(regs)) {
 378                 __le32 instr_le;
 379                 if (probe_kernel_address((__force __le32 *)pc, instr_le))
 380                         goto exit;
 381                 instr = le32_to_cpu(instr_le);
 382         } else if (compat_thumb_mode(regs)) {
 383                 /* 16-bit Thumb instruction */
 384                 __le16 instr_le;
 385                 if (get_user(instr_le, (__le16 __user *)pc))
 386                         goto exit;
 387                 instr = le16_to_cpu(instr_le);
 388                 if (aarch32_insn_is_wide(instr)) {
 389                         u32 instr2;
 390
 391                         if (get_user(instr_le, (__le16 __user *)(pc + 2)))
 392                                 goto exit;
 393                         instr2 = le16_to_cpu(instr_le);
 394                         instr = (instr << 16) | instr2;
 395                 }
 396         } else {
 397                 /* 32-bit ARM instruction */
 398                 __le32 instr_le;
 399                 if (get_user(instr_le, (__le32 __user *)pc))
 400                         goto exit;
 401                 instr = le32_to_cpu(instr_le);
 402         }
 403
 404         raw_spin_lock_irqsave(&undef_lock, flags);
 405         list_for_each_entry(hook, &undef_hook, node)
 406                 if ((instr & hook->instr_mask) == hook->instr_val &&
 407                         (regs->pstate & hook->pstate_mask) == hook->pstate_val)
 408                         fn = hook->fn;
 409
 410         raw_spin_unlock_irqrestore(&undef_lock, flags);
 411 exit:
 412         return fn ? fn(regs, instr) : 1;
 413 }
 414
 415 void force_signal_inject(int signal, int code, unsigned long address)
 416 {
 417         const char *desc;
 418         struct pt_regs *regs = current_pt_regs();
 419
 420         if (WARN_ON(!user_mode(regs)))
 421                 return;
 422
 423         switch (signal) {
 424         case SIGILL:
 425                 desc = "undefined instruction";
 426                 break;
 427         case SIGSEGV:
 428                 desc = "illegal memory access";
 429                 break;
 430         default:
 431                 desc = "unknown or unrecoverable error";
 432                 break;
 433         }
 434
 435         /* Force signals we don't understand to SIGKILL */
 436         if (WARN_ON(signal != SIGKILL &&
 437                     siginfo_layout(signal, code) != SIL_FAULT)) {
 438                 signal = SIGKILL;
 439         }
 440
 441         arm64_notify_die(desc, regs, signal, code, (void __user *)address, 0);
 442 }
 443
 444 /*
 445  * Set up process info to signal segmentation fault - called on access error.
 446  */
 447 void arm64_notify_segfault(unsigned long addr)
 448 {
 449         int code;
 450
 451         mmap_read_lock(current->mm);
 452         if (find_vma(current->mm, addr) == NULL)
 453                 code = SEGV_MAPERR;
 454         else
 455                 code = SEGV_ACCERR;
 456         mmap_read_unlock(current->mm);
 457
 458         force_signal_inject(SIGSEGV, code, addr);
 459 }
 460
 461 void do_undefinstr(struct pt_regs *regs)
 462 {
 463         /* check for AArch32 breakpoint instructions */
 464         if (!aarch32_break_handler(regs))
 465                 return;
 466
 467         if (call_undef_hook(regs) == 0)
 468                 return;
 469
 470         BUG_ON(!user_mode(regs));
 471         force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
 472 }
 473 NOKPROBE_SYMBOL(do_undefinstr);
 474
 475 void do_bti(struct pt_regs *regs)
 476 {
 477         BUG_ON(!user_mode(regs));
 478         force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
 479 }
 480 NOKPROBE_SYMBOL(do_bti);
 481
 482 #define __user_cache_maint(insn, address, res)                  \
 483         if (address >= user_addr_max()) {                       \
 484                 res = -EFAULT;                                  \
 485         } else {                                                \
 486                 uaccess_ttbr0_enable();                         \
 487                 asm volatile (                                  \
 488                         "1:     " insn ", %1\n"                 \
 489                         "       mov     %w0, #0\n"              \
 490                         "2:\n"                                  \
 491                         "       .pushsection .fixup,\"ax\"\n"   \
 492                         "       .align  2\n"                    \
 493                         "3:     mov     %w0, %w2\n"             \
 494                         "       b       2b\n"                   \
 495                         "       .popsection\n"                  \
 496                         _ASM_EXTABLE(1b, 3b)                    \
 497                         : "=r" (res)                            \
 498                         : "r" (address), "i" (-EFAULT));        \
 499                 uaccess_ttbr0_disable();                        \
 500         }
 501
 502 static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs)
 503 {
 504         unsigned long address;
 505         int rt = ESR_ELx_SYS64_ISS_RT(esr);
 506         int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
 507         int ret = 0;
 508
 509         address = untagged_addr(pt_regs_read_reg(regs, rt));
 510
 511         switch (crm) {
 512         case ESR_ELx_SYS64_ISS_CRM_DC_CVAU:     /* DC CVAU, gets promoted */
 513                 __user_cache_maint("dc civac", address, ret);
 514                 break;
 515         case ESR_ELx_SYS64_ISS_CRM_DC_CVAC:     /* DC CVAC, gets promoted */
 516                 __user_cache_maint("dc civac", address, ret);
 517                 break;
 518         case ESR_ELx_SYS64_ISS_CRM_DC_CVADP:    /* DC CVADP */
 519                 __user_cache_maint("sys 3, c7, c13, 1", address, ret);
 520                 break;
 521         case ESR_ELx_SYS64_ISS_CRM_DC_CVAP:     /* DC CVAP */
 522                 __user_cache_maint("sys 3, c7, c12, 1", address, ret);
 523                 break;
 524         case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC:    /* DC CIVAC */
 525                 __user_cache_maint("dc civac", address, ret);
 526                 break;
 527         case ESR_ELx_SYS64_ISS_CRM_IC_IVAU:     /* IC IVAU */
 528                 __user_cache_maint("ic ivau", address, ret);
 529                 break;
 530         default:
 531                 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
 532                 return;
 533         }
 534
 535         if (ret)
 536                 arm64_notify_segfault(address);
 537         else
 538                 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 539 }
 540
 541 static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)
 542 {
 543         int rt = ESR_ELx_SYS64_ISS_RT(esr);
 544         unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0);
 545
 546         if (cpus_have_const_cap(ARM64_WORKAROUND_1542419)) {
 547                 /* Hide DIC so that we can trap the unnecessary maintenance...*/
 548                 val &= ~BIT(CTR_DIC_SHIFT);
 549
 550                 /* ... and fake IminLine to reduce the number of traps. */
 551                 val &= ~CTR_IMINLINE_MASK;
 552                 val |= (PAGE_SHIFT - 2) & CTR_IMINLINE_MASK;
 553         }
 554
 555         pt_regs_write_reg(regs, rt, val);
 556
 557         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 558 }
 559
 560 static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
 561 {
 562         int rt = ESR_ELx_SYS64_ISS_RT(esr);
 563
 564         pt_regs_write_reg(regs, rt, arch_timer_read_counter());
 565         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 566 }
 567
 568 static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
 569 {
 570         int rt = ESR_ELx_SYS64_ISS_RT(esr);
 571
 572         pt_regs_write_reg(regs, rt, arch_timer_get_rate());
 573         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 574 }
 575
 576 static void mrs_handler(unsigned int esr, struct pt_regs *regs)
 577 {
 578         u32 sysreg, rt;
 579
 580         rt = ESR_ELx_SYS64_ISS_RT(esr);
 581         sysreg = esr_sys64_to_sysreg(esr);
 582
 583         if (do_emulate_mrs(regs, sysreg, rt) != 0)
 584                 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
 585 }
 586
 587 static void wfi_handler(unsigned int esr, struct pt_regs *regs)
 588 {
 589         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 590 }
 591
 592 struct sys64_hook {
 593         unsigned int esr_mask;
 594         unsigned int esr_val;
 595         void (*handler)(unsigned int esr, struct pt_regs *regs);
 596 };
 597
 598 static const struct sys64_hook sys64_hooks[] = {
 599         {
 600                 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK,
 601                 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL,
 602                 .handler = user_cache_maint_handler,
 603         },
 604         {
 605                 /* Trap read access to CTR_EL0 */
 606                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
 607                 .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ,
 608                 .handler = ctr_read_handler,
 609         },
 610         {
 611                 /* Trap read access to CNTVCT_EL0 */
 612                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
 613                 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT,
 614                 .handler = cntvct_read_handler,
 615         },
 616         {
 617                 /* Trap read access to CNTFRQ_EL0 */
 618                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
 619                 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ,
 620                 .handler = cntfrq_read_handler,
 621         },
 622         {
 623                 /* Trap read access to CPUID registers */
 624                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK,
 625                 .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL,
 626                 .handler = mrs_handler,
 627         },
 628         {
 629                 /* Trap WFI instructions executed in userspace */
 630                 .esr_mask = ESR_ELx_WFx_MASK,
 631                 .esr_val = ESR_ELx_WFx_WFI_VAL,
 632                 .handler = wfi_handler,
 633         },
 634         {},
 635 };
 636
 637 #ifdef CONFIG_COMPAT
 638 static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs)
 639 {
 640         int cond;
 641
 642         /* Only a T32 instruction can trap without CV being set */
 643         if (!(esr & ESR_ELx_CV)) {
 644                 u32 it;
 645
 646                 it = compat_get_it_state(regs);
 647                 if (!it)
 648                         return true;
 649
 650                 cond = it >> 4;
 651         } else {
 652                 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
 653         }
 654
 655         return aarch32_opcode_cond_checks[cond](regs->pstate);
 656 }
 657
 658 static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
 659 {
 660         int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT;
 661
 662         pt_regs_write_reg(regs, reg, arch_timer_get_rate());
 663         arm64_skip_faulting_instruction(regs, 4);
 664 }
 665
 666 static const struct sys64_hook cp15_32_hooks[] = {
 667         {
 668                 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK,
 669                 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ,
 670                 .handler = compat_cntfrq_read_handler,
 671         },
 672         {},
 673 };
 674
 675 static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
 676 {
 677         int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT;
 678         int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT;
 679         u64 val = arch_timer_read_counter();
 680
 681         pt_regs_write_reg(regs, rt, lower_32_bits(val));
 682         pt_regs_write_reg(regs, rt2, upper_32_bits(val));
 683         arm64_skip_faulting_instruction(regs, 4);
 684 }
 685
 686 static const struct sys64_hook cp15_64_hooks[] = {
 687         {
 688                 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK,
 689                 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT,
 690                 .handler = compat_cntvct_read_handler,
 691         },
 692         {},
 693 };
 694
 695 void do_cp15instr(unsigned int esr, struct pt_regs *regs)
 696 {
 697         const struct sys64_hook *hook, *hook_base;
 698
 699         if (!cp15_cond_valid(esr, regs)) {
 700                 /*
 701                  * There is no T16 variant of a CP access, so we
 702                  * always advance PC by 4 bytes.
 703                  */
 704                 arm64_skip_faulting_instruction(regs, 4);
 705                 return;
 706         }
 707
 708         switch (ESR_ELx_EC(esr)) {
 709         case ESR_ELx_EC_CP15_32:
 710                 hook_base = cp15_32_hooks;
 711                 break;
 712         case ESR_ELx_EC_CP15_64:
 713                 hook_base = cp15_64_hooks;
 714                 break;
 715         default:
 716                 do_undefinstr(regs);
 717                 return;
 718         }
 719
 720         for (hook = hook_base; hook->handler; hook++)
 721                 if ((hook->esr_mask & esr) == hook->esr_val) {
 722                         hook->handler(esr, regs);
 723                         return;
 724                 }
 725
 726         /*
 727          * New cp15 instructions may previously have been undefined at
 728          * EL0. Fall back to our usual undefined instruction handler
 729          * so that we handle these consistently.
 730          */
 731         do_undefinstr(regs);
 732 }
 733 NOKPROBE_SYMBOL(do_cp15instr);
 734 #endif
 735
 736 void do_sysinstr(unsigned int esr, struct pt_regs *regs)
 737 {
 738         const struct sys64_hook *hook;
 739
 740         for (hook = sys64_hooks; hook->handler; hook++)
 741                 if ((hook->esr_mask & esr) == hook->esr_val) {
 742                         hook->handler(esr, regs);
 743                         return;
 744                 }
 745
 746         /*
 747          * New SYS instructions may previously have been undefined at EL0. Fall
 748          * back to our usual undefined instruction handler so that we handle
 749          * these consistently.
 750          */
 751         do_undefinstr(regs);
 752 }
 753 NOKPROBE_SYMBOL(do_sysinstr);
 754
 755 static const char *esr_class_str[] = {
 756         [0 ... ESR_ELx_EC_MAX]          = "UNRECOGNIZED EC",
 757         [ESR_ELx_EC_UNKNOWN]            = "Unknown/Uncategorized",
 758         [ESR_ELx_EC_WFx]                = "WFI/WFE",
 759         [ESR_ELx_EC_CP15_32]            = "CP15 MCR/MRC",
 760         [ESR_ELx_EC_CP15_64]            = "CP15 MCRR/MRRC",
 761         [ESR_ELx_EC_CP14_MR]            = "CP14 MCR/MRC",
 762         [ESR_ELx_EC_CP14_LS]            = "CP14 LDC/STC",
 763         [ESR_ELx_EC_FP_ASIMD]           = "ASIMD",
 764         [ESR_ELx_EC_CP10_ID]            = "CP10 MRC/VMRS",
 765         [ESR_ELx_EC_PAC]                = "PAC",
 766         [ESR_ELx_EC_CP14_64]            = "CP14 MCRR/MRRC",
 767         [ESR_ELx_EC_BTI]                = "BTI",
 768         [ESR_ELx_EC_ILL]                = "PSTATE.IL",
 769         [ESR_ELx_EC_SVC32]              = "SVC (AArch32)",
 770         [ESR_ELx_EC_HVC32]              = "HVC (AArch32)",
 771         [ESR_ELx_EC_SMC32]              = "SMC (AArch32)",
 772         [ESR_ELx_EC_SVC64]              = "SVC (AArch64)",
 773         [ESR_ELx_EC_HVC64]              = "HVC (AArch64)",
 774         [ESR_ELx_EC_SMC64]              = "SMC (AArch64)",
 775         [ESR_ELx_EC_SYS64]              = "MSR/MRS (AArch64)",
 776         [ESR_ELx_EC_SVE]                = "SVE",
 777         [ESR_ELx_EC_ERET]               = "ERET/ERETAA/ERETAB",
 778         [ESR_ELx_EC_IMP_DEF]            = "EL3 IMP DEF",
 779         [ESR_ELx_EC_IABT_LOW]           = "IABT (lower EL)",
 780         [ESR_ELx_EC_IABT_CUR]           = "IABT (current EL)",
 781         [ESR_ELx_EC_PC_ALIGN]           = "PC Alignment",
 782         [ESR_ELx_EC_DABT_LOW]           = "DABT (lower EL)",
 783         [ESR_ELx_EC_DABT_CUR]           = "DABT (current EL)",
 784         [ESR_ELx_EC_SP_ALIGN]           = "SP Alignment",
 785         [ESR_ELx_EC_FP_EXC32]           = "FP (AArch32)",
 786         [ESR_ELx_EC_FP_EXC64]           = "FP (AArch64)",
 787         [ESR_ELx_EC_SERROR]             = "SError",
 788         [ESR_ELx_EC_BREAKPT_LOW]        = "Breakpoint (lower EL)",
 789         [ESR_ELx_EC_BREAKPT_CUR]        = "Breakpoint (current EL)",
 790         [ESR_ELx_EC_SOFTSTP_LOW]        = "Software Step (lower EL)",
 791         [ESR_ELx_EC_SOFTSTP_CUR]        = "Software Step (current EL)",
 792         [ESR_ELx_EC_WATCHPT_LOW]        = "Watchpoint (lower EL)",
 793         [ESR_ELx_EC_WATCHPT_CUR]        = "Watchpoint (current EL)",
 794         [ESR_ELx_EC_BKPT32]             = "BKPT (AArch32)",
 795         [ESR_ELx_EC_VECTOR32]           = "Vector catch (AArch32)",
 796         [ESR_ELx_EC_BRK64]              = "BRK (AArch64)",
 797 };
 798
 799 const char *esr_get_class_string(u32 esr)
 800 {
 801         return esr_class_str[ESR_ELx_EC(esr)];
 802 }
 803
 804 /*
 805  * bad_mode handles the impossible case in the exception vector. This is always
 806  * fatal.
 807  */
 808 asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
 809 {
 810         console_verbose();
 811
 812         pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n",
 813                 handler[reason], smp_processor_id(), esr,
 814                 esr_get_class_string(esr));
 815
 816         __show_regs(regs);
 817         local_daif_mask();
 818         panic("bad mode");
 819 }
 820
 821 /*
 822  * bad_el0_sync handles unexpected, but potentially recoverable synchronous
 823  * exceptions taken from EL0. Unlike bad_mode, this returns.
 824  */
 825 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
 826 {
 827         void __user *pc = (void __user *)instruction_pointer(regs);
 828
 829         current->thread.fault_address = 0;
 830         current->thread.fault_code = esr;
 831
 832         arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc,
 833                               "Bad EL0 synchronous exception");
 834 }
 835
 836 #ifdef CONFIG_VMAP_STACK
 837
 838 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
 839         __aligned(16);
 840
 841 asmlinkage void handle_bad_stack(struct pt_regs *regs)
 842 {
 843         unsigned long tsk_stk = (unsigned long)current->stack;
 844         unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
 845         unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
 846         unsigned int esr = read_sysreg(esr_el1);
 847         unsigned long far = read_sysreg(far_el1);
 848
 849         console_verbose();
 850         pr_emerg("Insufficient stack space to handle exception!");
 851
 852         pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr));
 853         pr_emerg("FAR: 0x%016lx\n", far);
 854
 855         pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
 856                  tsk_stk, tsk_stk + THREAD_SIZE);
 857         pr_emerg("IRQ stack:      [0x%016lx..0x%016lx]\n",
 858                  irq_stk, irq_stk + THREAD_SIZE);
 859         pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
 860                  ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
 861
 862         __show_regs(regs);
 863
 864         /*
 865          * We use nmi_panic to limit the potential for recusive overflows, and
 866          * to get a better stack trace.
 867          */
 868         nmi_panic(NULL, "kernel stack overflow");
 869         cpu_park_loop();
 870 }
 871 #endif
 872
 873 void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr)
 874 {
 875         console_verbose();
 876
 877         pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n",
 878                 smp_processor_id(), esr, esr_get_class_string(esr));
 879         if (regs)
 880                 __show_regs(regs);
 881
 882         nmi_panic(regs, "Asynchronous SError Interrupt");
 883
 884         cpu_park_loop();
 885         unreachable();
 886 }
 887
 888 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr)
 889 {
 890         u32 aet = arm64_ras_serror_get_severity(esr);
 891
 892         switch (aet) {
 893         case ESR_ELx_AET_CE:    /* corrected error */
 894         case ESR_ELx_AET_UEO:   /* restartable, not yet consumed */
 895                 /*
 896                  * The CPU can make progress. We may take UEO again as
 897                  * a more severe error.
 898                  */
 899                 return false;
 900
 901         case ESR_ELx_AET_UEU:   /* Uncorrected Unrecoverable */
 902         case ESR_ELx_AET_UER:   /* Uncorrected Recoverable */
 903                 /*
 904                  * The CPU can't make progress. The exception may have
 905                  * been imprecise.
 906                  *
 907                  * Neoverse-N1 #1349291 means a non-KVM SError reported as
 908                  * Unrecoverable should be treated as Uncontainable. We
 909                  * call arm64_serror_panic() in both cases.
 910                  */
 911                 return true;
 912
 913         case ESR_ELx_AET_UC:    /* Uncontainable or Uncategorized error */
 914         default:
 915                 /* Error has been silently propagated */
 916                 arm64_serror_panic(regs, esr);
 917         }
 918 }
 919
 920 asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr)
 921 {
 922         nmi_enter();
 923
 924         /* non-RAS errors are not containable */
 925         if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr))
 926                 arm64_serror_panic(regs, esr);
 927
 928         nmi_exit();
 929 }
 930
 931 asmlinkage void enter_from_user_mode(void)
 932 {
 933         CT_WARN_ON(ct_state() != CONTEXT_USER);
 934         user_exit_irqoff();
 935 }
 936 NOKPROBE_SYMBOL(enter_from_user_mode);
 937
 938 void __pte_error(const char *file, int line, unsigned long val)
 939 {
 940         pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
 941 }
 942
 943 void __pmd_error(const char *file, int line, unsigned long val)
 944 {
 945         pr_err("%s:%d: bad pmd %016lx.\n", file, line, val);
 946 }
 947
 948 void __pud_error(const char *file, int line, unsigned long val)
 949 {
 950         pr_err("%s:%d: bad pud %016lx.\n", file, line, val);
 951 }
 952
 953 void __pgd_error(const char *file, int line, unsigned long val)
 954 {
 955         pr_err("%s:%d: bad pgd %016lx.\n", file, line, val);
 956 }
 957
 958 /* GENERIC_BUG traps */
 959
 960 int is_valid_bugaddr(unsigned long addr)
 961 {
 962         /*
 963          * bug_handler() only called for BRK #BUG_BRK_IMM.
 964          * So the answer is trivial -- any spurious instances with no
 965          * bug table entry will be rejected by report_bug() and passed
 966          * back to the debug-monitors code and handled as a fatal
 967          * unexpected debug exception.
 968          */
 969         return 1;
 970 }
 971
 972 static int bug_handler(struct pt_regs *regs, unsigned int esr)
 973 {
 974         switch (report_bug(regs->pc, regs)) {
 975         case BUG_TRAP_TYPE_BUG:
 976                 die("Oops - BUG", regs, 0);
 977                 break;
 978
 979         case BUG_TRAP_TYPE_WARN:
 980                 break;
 981
 982         default:
 983                 /* unknown/unrecognised bug trap type */
 984                 return DBG_HOOK_ERROR;
 985         }
 986
 987         /* If thread survives, skip over the BUG instruction and continue: */
 988         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 989         return DBG_HOOK_HANDLED;
 990 }
 991
 992 static struct break_hook bug_break_hook = {
 993         .fn = bug_handler,
 994         .imm = BUG_BRK_IMM,
 995 };
 996
 997 #ifdef CONFIG_KASAN_SW_TAGS
 998
 999 #define KASAN_ESR_RECOVER       0x20
1000 #define KASAN_ESR_WRITE 0x10
1001 #define KASAN_ESR_SIZE_MASK     0x0f
1002 #define KASAN_ESR_SIZE(esr)     (1 << ((esr) & KASAN_ESR_SIZE_MASK))
1003
1004 static int kasan_handler(struct pt_regs *regs, unsigned int esr)
1005 {
1006         bool recover = esr & KASAN_ESR_RECOVER;
1007         bool write = esr & KASAN_ESR_WRITE;
1008         size_t size = KASAN_ESR_SIZE(esr);
1009         u64 addr = regs->regs[0];
1010         u64 pc = regs->pc;
1011
1012         kasan_report(addr, size, write, pc);
1013
1014         /*
1015          * The instrumentation allows to control whether we can proceed after
1016          * a crash was detected. This is done by passing the -recover flag to
1017          * the compiler. Disabling recovery allows to generate more compact
1018          * code.
1019          *
1020          * Unfortunately disabling recovery doesn't work for the kernel right
1021          * now. KASAN reporting is disabled in some contexts (for example when
1022          * the allocator accesses slab object metadata; this is controlled by
1023          * current->kasan_depth). All these accesses are detected by the tool,
1024          * even though the reports for them are not printed.
1025          *
1026          * This is something that might be fixed at some point in the future.
1027          */
1028         if (!recover)
1029                 die("Oops - KASAN", regs, 0);
1030
1031         /* If thread survives, skip over the brk instruction and continue: */
1032         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
1033         return DBG_HOOK_HANDLED;
1034 }
1035
1036 static struct break_hook kasan_break_hook = {
1037         .fn     = kasan_handler,
1038         .imm    = KASAN_BRK_IMM,
1039         .mask   = KASAN_BRK_MASK,
1040 };
1041 #endif
1042
1043 /*
1044  * Initial handler for AArch64 BRK exceptions
1045  * This handler only used until debug_traps_init().
1046  */
1047 int __init early_brk64(unsigned long addr, unsigned int esr,
1048                 struct pt_regs *regs)
1049 {
1050 #ifdef CONFIG_KASAN_SW_TAGS
1051         unsigned int comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
1052
1053         if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
1054                 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
1055 #endif
1056         return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
1057 }
1058
1059 void __init trap_init(void)
1060 {
1061         register_kernel_break_hook(&bug_break_hook);
1062 #ifdef CONFIG_KASAN_SW_TAGS
1063         register_kernel_break_hook(&kasan_break_hook);
1064 #endif
1065         debug_traps_init();
1066 }