arch/powerpc/kernel/interrupt.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2
   3 #include <linux/context_tracking.h>
   4 #include <linux/err.h>
   5 #include <linux/compat.h>
   6
   7 #include <asm/asm-prototypes.h>
   8 #include <asm/kup.h>
   9 #include <asm/cputime.h>
  10 #include <asm/interrupt.h>
  11 #include <asm/hw_irq.h>
  12 #include <asm/interrupt.h>
  13 #include <asm/kprobes.h>
  14 #include <asm/paca.h>
  15 #include <asm/ptrace.h>
  16 #include <asm/reg.h>
  17 #include <asm/signal.h>
  18 #include <asm/switch_to.h>
  19 #include <asm/syscall.h>
  20 #include <asm/time.h>
  21 #include <asm/unistd.h>
  22
  23 #if defined(CONFIG_PPC_ADV_DEBUG_REGS) && defined(CONFIG_PPC32)
  24 unsigned long global_dbcr0[NR_CPUS];
  25 #endif
  26
  27 typedef long (*syscall_fn)(long, long, long, long, long, long);
  28
  29 /* Has to run notrace because it is entered not completely "reconciled" */
  30 notrace long system_call_exception(long r3, long r4, long r5,
  31                                    long r6, long r7, long r8,
  32                                    unsigned long r0, struct pt_regs *regs)
  33 {
  34         syscall_fn f;
  35
  36         kuep_lock();
  37 #ifdef CONFIG_PPC32
  38         kuap_save_and_lock(regs);
  39 #endif
  40
  41         regs->orig_gpr3 = r3;
  42
  43         if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
  44                 BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
  45
  46         trace_hardirqs_off(); /* finish reconciling */
  47
  48         CT_WARN_ON(ct_state() == CONTEXT_KERNEL);
  49         user_exit_irqoff();
  50
  51         if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))
  52                 BUG_ON(!(regs->msr & MSR_RI));
  53         BUG_ON(!(regs->msr & MSR_PR));
  54         BUG_ON(arch_irq_disabled_regs(regs));
  55
  56 #ifdef CONFIG_PPC_PKEY
  57         if (mmu_has_feature(MMU_FTR_PKEY)) {
  58                 unsigned long amr, iamr;
  59                 bool flush_needed = false;
  60                 /*
  61                  * When entering from userspace we mostly have the AMR/IAMR
  62                  * different from kernel default values. Hence don't compare.
  63                  */
  64                 amr = mfspr(SPRN_AMR);
  65                 iamr = mfspr(SPRN_IAMR);
  66                 regs->amr  = amr;
  67                 regs->iamr = iamr;
  68                 if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
  69                         mtspr(SPRN_AMR, AMR_KUAP_BLOCKED);
  70                         flush_needed = true;
  71                 }
  72                 if (mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) {
  73                         mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED);
  74                         flush_needed = true;
  75                 }
  76                 if (flush_needed)
  77                         isync();
  78         } else
  79 #endif
  80                 kuap_assert_locked();
  81
  82         booke_restore_dbcr0();
  83
  84         account_cpu_user_entry();
  85
  86         account_stolen_time();
  87
  88         /*
  89          * This is not required for the syscall exit path, but makes the
  90          * stack frame look nicer. If this was initialised in the first stack
  91          * frame, or if the unwinder was taught the first stack frame always
  92          * returns to user with IRQS_ENABLED, this store could be avoided!
  93          */
  94         irq_soft_mask_regs_set_state(regs, IRQS_ENABLED);
  95
  96         local_irq_enable();
  97
  98         if (unlikely(current_thread_info()->flags & _TIF_SYSCALL_DOTRACE)) {
  99                 if (unlikely(trap_is_unsupported_scv(regs))) {
 100                         /* Unsupported scv vector */
 101                         _exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
 102                         return regs->gpr[3];
 103                 }
 104                 /*
 105                  * We use the return value of do_syscall_trace_enter() as the
 106                  * syscall number. If the syscall was rejected for any reason
 107                  * do_syscall_trace_enter() returns an invalid syscall number
 108                  * and the test against NR_syscalls will fail and the return
 109                  * value to be used is in regs->gpr[3].
 110                  */
 111                 r0 = do_syscall_trace_enter(regs);
 112                 if (unlikely(r0 >= NR_syscalls))
 113                         return regs->gpr[3];
 114                 r3 = regs->gpr[3];
 115                 r4 = regs->gpr[4];
 116                 r5 = regs->gpr[5];
 117                 r6 = regs->gpr[6];
 118                 r7 = regs->gpr[7];
 119                 r8 = regs->gpr[8];
 120
 121         } else if (unlikely(r0 >= NR_syscalls)) {
 122                 if (unlikely(trap_is_unsupported_scv(regs))) {
 123                         /* Unsupported scv vector */
 124                         _exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
 125                         return regs->gpr[3];
 126                 }
 127                 return -ENOSYS;
 128         }
 129
 130         /* May be faster to do array_index_nospec? */
 131         barrier_nospec();
 132
 133         if (unlikely(is_compat_task())) {
 134                 f = (void *)compat_sys_call_table[r0];
 135
 136                 r3 &= 0x00000000ffffffffULL;
 137                 r4 &= 0x00000000ffffffffULL;
 138                 r5 &= 0x00000000ffffffffULL;
 139                 r6 &= 0x00000000ffffffffULL;
 140                 r7 &= 0x00000000ffffffffULL;
 141                 r8 &= 0x00000000ffffffffULL;
 142
 143         } else {
 144                 f = (void *)sys_call_table[r0];
 145         }
 146
 147         return f(r3, r4, r5, r6, r7, r8);
 148 }
 149
 150 /*
 151  * local irqs must be disabled. Returns false if the caller must re-enable
 152  * them, check for new work, and try again.
 153  *
 154  * This should be called with local irqs disabled, but if they were previously
 155  * enabled when the interrupt handler returns (indicating a process-context /
 156  * synchronous interrupt) then irqs_enabled should be true.
 157  */
 158 static notrace __always_inline bool __prep_irq_for_enabled_exit(bool clear_ri)
 159 {
 160         /* This must be done with RI=1 because tracing may touch vmaps */
 161         trace_hardirqs_on();
 162
 163         /* This pattern matches prep_irq_for_idle */
 164         if (clear_ri)
 165                 __hard_EE_RI_disable();
 166         else
 167                 __hard_irq_disable();
 168 #ifdef CONFIG_PPC64
 169         if (unlikely(lazy_irq_pending_nocheck())) {
 170                 /* Took an interrupt, may have more exit work to do. */
 171                 if (clear_ri)
 172                         __hard_RI_enable();
 173                 trace_hardirqs_off();
 174                 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
 175
 176                 return false;
 177         }
 178         local_paca->irq_happened = 0;
 179         irq_soft_mask_set(IRQS_ENABLED);
 180 #endif
 181         return true;
 182 }
 183
 184 static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri, bool irqs_enabled)
 185 {
 186         if (__prep_irq_for_enabled_exit(clear_ri))
 187                 return true;
 188
 189         /*
 190          * Must replay pending soft-masked interrupts now. Don't just
 191          * local_irq_enabe(); local_irq_disable(); because if we are
 192          * returning from an asynchronous interrupt here, another one
 193          * might hit after irqs are enabled, and it would exit via this
 194          * same path allowing another to fire, and so on unbounded.
 195          *
 196          * If interrupts were enabled when this interrupt exited,
 197          * indicating a process context (synchronous) interrupt,
 198          * local_irq_enable/disable can be used, which will enable
 199          * interrupts rather than keeping them masked (unclear how
 200          * much benefit this is over just replaying for all cases,
 201          * because we immediately disable again, so all we're really
 202          * doing is allowing hard interrupts to execute directly for
 203          * a very small time, rather than being masked and replayed).
 204          */
 205         if (irqs_enabled) {
 206                 local_irq_enable();
 207                 local_irq_disable();
 208         } else {
 209                 replay_soft_interrupts();
 210         }
 211
 212         return false;
 213 }
 214
 215 static notrace void booke_load_dbcr0(void)
 216 {
 217 #ifdef CONFIG_PPC_ADV_DEBUG_REGS
 218         unsigned long dbcr0 = current->thread.debug.dbcr0;
 219
 220         if (likely(!(dbcr0 & DBCR0_IDM)))
 221                 return;
 222
 223         /*
 224          * Check to see if the dbcr0 register is set up to debug.
 225          * Use the internal debug mode bit to do this.
 226          */
 227         mtmsr(mfmsr() & ~MSR_DE);
 228         if (IS_ENABLED(CONFIG_PPC32)) {
 229                 isync();
 230                 global_dbcr0[smp_processor_id()] = mfspr(SPRN_DBCR0);
 231         }
 232         mtspr(SPRN_DBCR0, dbcr0);
 233         mtspr(SPRN_DBSR, -1);
 234 #endif
 235 }
 236
 237 /*
 238  * This should be called after a syscall returns, with r3 the return value
 239  * from the syscall. If this function returns non-zero, the system call
 240  * exit assembly should additionally load all GPR registers and CTR and XER
 241  * from the interrupt frame.
 242  *
 243  * The function graph tracer can not trace the return side of this function,
 244  * because RI=0 and soft mask state is "unreconciled", so it is marked notrace.
 245  */
 246 notrace unsigned long syscall_exit_prepare(unsigned long r3,
 247                                            struct pt_regs *regs,
 248                                            long scv)
 249 {
 250         unsigned long ti_flags;
 251         unsigned long ret = 0;
 252         bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv;
 253
 254         CT_WARN_ON(ct_state() == CONTEXT_USER);
 255
 256         kuap_assert_locked();
 257
 258         regs->result = r3;
 259
 260         /* Check whether the syscall is issued inside a restartable sequence */
 261         rseq_syscall(regs);
 262
 263         ti_flags = current_thread_info()->flags;
 264
 265         if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) {
 266                 if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) {
 267                         r3 = -r3;
 268                         regs->ccr |= 0x10000000; /* Set SO bit in CR */
 269                 }
 270         }
 271
 272         if (unlikely(ti_flags & _TIF_PERSYSCALL_MASK)) {
 273                 if (ti_flags & _TIF_RESTOREALL)
 274                         ret = _TIF_RESTOREALL;
 275                 else
 276                         regs->gpr[3] = r3;
 277                 clear_bits(_TIF_PERSYSCALL_MASK, &current_thread_info()->flags);
 278         } else {
 279                 regs->gpr[3] = r3;
 280         }
 281
 282         if (unlikely(ti_flags & _TIF_SYSCALL_DOTRACE)) {
 283                 do_syscall_trace_leave(regs);
 284                 ret |= _TIF_RESTOREALL;
 285         }
 286
 287         local_irq_disable();
 288
 289 again:
 290         ti_flags = READ_ONCE(current_thread_info()->flags);
 291         while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {
 292                 local_irq_enable();
 293                 if (ti_flags & _TIF_NEED_RESCHED) {
 294                         schedule();
 295                 } else {
 296                         /*
 297                          * SIGPENDING must restore signal handler function
 298                          * argument GPRs, and some non-volatiles (e.g., r1).
 299                          * Restore all for now. This could be made lighter.
 300                          */
 301                         if (ti_flags & _TIF_SIGPENDING)
 302                                 ret |= _TIF_RESTOREALL;
 303                         do_notify_resume(regs, ti_flags);
 304                 }
 305                 local_irq_disable();
 306                 ti_flags = READ_ONCE(current_thread_info()->flags);
 307         }
 308
 309         if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) {
 310                 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
 311                                 unlikely((ti_flags & _TIF_RESTORE_TM))) {
 312                         restore_tm_state(regs);
 313                 } else {
 314                         unsigned long mathflags = MSR_FP;
 315
 316                         if (cpu_has_feature(CPU_FTR_VSX))
 317                                 mathflags |= MSR_VEC | MSR_VSX;
 318                         else if (cpu_has_feature(CPU_FTR_ALTIVEC))
 319                                 mathflags |= MSR_VEC;
 320
 321                         /*
 322                          * If userspace MSR has all available FP bits set,
 323                          * then they are live and no need to restore. If not,
 324                          * it means the regs were given up and restore_math
 325                          * may decide to restore them (to avoid taking an FP
 326                          * fault).
 327                          */
 328                         if ((regs->msr & mathflags) != mathflags)
 329                                 restore_math(regs);
 330                 }
 331         }
 332
 333         user_enter_irqoff();
 334
 335         /* scv need not set RI=0 because SRRs are not used */
 336         if (unlikely(!__prep_irq_for_enabled_exit(is_not_scv))) {
 337                 user_exit_irqoff();
 338                 local_irq_enable();
 339                 local_irq_disable();
 340                 goto again;
 341         }
 342
 343 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 344         local_paca->tm_scratch = regs->msr;
 345 #endif
 346
 347         booke_load_dbcr0();
 348
 349         account_cpu_user_exit();
 350
 351         /* Restore user access locks last */
 352         kuap_user_restore(regs);
 353         kuep_unlock();
 354
 355         return ret;
 356 }
 357
 358 notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr)
 359 {
 360         unsigned long ti_flags;
 361         unsigned long flags;
 362         unsigned long ret = 0;
 363
 364         if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))
 365                 BUG_ON(!(regs->msr & MSR_RI));
 366         BUG_ON(!(regs->msr & MSR_PR));
 367         BUG_ON(arch_irq_disabled_regs(regs));
 368         CT_WARN_ON(ct_state() == CONTEXT_USER);
 369
 370         /*
 371          * We don't need to restore AMR on the way back to userspace for KUAP.
 372          * AMR can only have been unlocked if we interrupted the kernel.
 373          */
 374         kuap_assert_locked();
 375
 376         local_irq_save(flags);
 377
 378 again:
 379         ti_flags = READ_ONCE(current_thread_info()->flags);
 380         while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {
 381                 local_irq_enable(); /* returning to user: may enable */
 382                 if (ti_flags & _TIF_NEED_RESCHED) {
 383                         schedule();
 384                 } else {
 385                         if (ti_flags & _TIF_SIGPENDING)
 386                                 ret |= _TIF_RESTOREALL;
 387                         do_notify_resume(regs, ti_flags);
 388                 }
 389                 local_irq_disable();
 390                 ti_flags = READ_ONCE(current_thread_info()->flags);
 391         }
 392
 393         if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && IS_ENABLED(CONFIG_PPC_FPU)) {
 394                 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
 395                                 unlikely((ti_flags & _TIF_RESTORE_TM))) {
 396                         restore_tm_state(regs);
 397                 } else {
 398                         unsigned long mathflags = MSR_FP;
 399
 400                         if (cpu_has_feature(CPU_FTR_VSX))
 401                                 mathflags |= MSR_VEC | MSR_VSX;
 402                         else if (cpu_has_feature(CPU_FTR_ALTIVEC))
 403                                 mathflags |= MSR_VEC;
 404
 405                         /* See above restore_math comment */
 406                         if ((regs->msr & mathflags) != mathflags)
 407                                 restore_math(regs);
 408                 }
 409         }
 410
 411         user_enter_irqoff();
 412
 413         if (unlikely(!__prep_irq_for_enabled_exit(true))) {
 414                 user_exit_irqoff();
 415                 local_irq_enable();
 416                 local_irq_disable();
 417                 goto again;
 418         }
 419
 420         booke_load_dbcr0();
 421
 422 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 423         local_paca->tm_scratch = regs->msr;
 424 #endif
 425
 426         account_cpu_user_exit();
 427
 428         /* Restore user access locks last */
 429         kuap_user_restore(regs);
 430
 431         return ret;
 432 }
 433
 434 void preempt_schedule_irq(void);
 435
 436 notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr)
 437 {
 438         unsigned long flags;
 439         unsigned long ret = 0;
 440         unsigned long kuap;
 441
 442         if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) &&
 443             unlikely(!(regs->msr & MSR_RI)))
 444                 unrecoverable_exception(regs);
 445         BUG_ON(regs->msr & MSR_PR);
 446         /*
 447          * CT_WARN_ON comes here via program_check_exception,
 448          * so avoid recursion.
 449          */
 450         if (TRAP(regs) != INTERRUPT_PROGRAM)
 451                 CT_WARN_ON(ct_state() == CONTEXT_USER);
 452
 453         kuap = kuap_get_and_assert_locked();
 454
 455         if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) {
 456                 clear_bits(_TIF_EMULATE_STACK_STORE, &current_thread_info()->flags);
 457                 ret = 1;
 458         }
 459
 460         local_irq_save(flags);
 461
 462         if (!arch_irq_disabled_regs(regs)) {
 463                 /* Returning to a kernel context with local irqs enabled. */
 464                 WARN_ON_ONCE(!(regs->msr & MSR_EE));
 465 again:
 466                 if (IS_ENABLED(CONFIG_PREEMPT)) {
 467                         /* Return to preemptible kernel context */
 468                         if (unlikely(current_thread_info()->flags & _TIF_NEED_RESCHED)) {
 469                                 if (preempt_count() == 0)
 470                                         preempt_schedule_irq();
 471                         }
 472                 }
 473
 474                 if (unlikely(!prep_irq_for_enabled_exit(true, !irqs_disabled_flags(flags))))
 475                         goto again;
 476         } else {
 477                 /* Returning to a kernel context with local irqs disabled. */
 478                 __hard_EE_RI_disable();
 479 #ifdef CONFIG_PPC64
 480                 if (regs->msr & MSR_EE)
 481                         local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
 482 #endif
 483         }
 484
 485
 486 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 487         local_paca->tm_scratch = regs->msr;
 488 #endif
 489
 490         /*
 491          * 64s does not want to mfspr(SPRN_AMR) here, because this comes after
 492          * mtmsr, which would cause Read-After-Write stalls. Hence, take the
 493          * AMR value from the check above.
 494          */
 495         kuap_kernel_restore(regs, kuap);
 496
 497         return ret;
 498 }