arch/powerpc/kernel/process.c

   1 /*
   2  *  Derived from "arch/i386/kernel/process.c"
   3  *    Copyright (C) 1995  Linus Torvalds
   4  *
   5  *  Updated and modified by Cort Dougan (cort@cs.nmt.edu) and
   6  *  Paul Mackerras (paulus@cs.anu.edu.au)
   7  *
   8  *  PowerPC version
   9  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  10  *
  11  *  This program is free software; you can redistribute it and/or
  12  *  modify it under the terms of the GNU General Public License
  13  *  as published by the Free Software Foundation; either version
  14  *  2 of the License, or (at your option) any later version.
  15  */
  16
  17 #include <linux/errno.h>
  18 #include <linux/sched.h>
  19 #include <linux/sched/debug.h>
  20 #include <linux/sched/task.h>
  21 #include <linux/sched/task_stack.h>
  22 #include <linux/kernel.h>
  23 #include <linux/mm.h>
  24 #include <linux/smp.h>
  25 #include <linux/stddef.h>
  26 #include <linux/unistd.h>
  27 #include <linux/ptrace.h>
  28 #include <linux/slab.h>
  29 #include <linux/user.h>
  30 #include <linux/elf.h>
  31 #include <linux/prctl.h>
  32 #include <linux/init_task.h>
  33 #include <linux/export.h>
  34 #include <linux/kallsyms.h>
  35 #include <linux/mqueue.h>
  36 #include <linux/hardirq.h>
  37 #include <linux/utsname.h>
  38 #include <linux/ftrace.h>
  39 #include <linux/kernel_stat.h>
  40 #include <linux/personality.h>
  41 #include <linux/random.h>
  42 #include <linux/hw_breakpoint.h>
  43 #include <linux/uaccess.h>
  44 #include <linux/elf-randomize.h>
  45
  46 #include <asm/pgtable.h>
  47 #include <asm/io.h>
  48 #include <asm/processor.h>
  49 #include <asm/mmu.h>
  50 #include <asm/prom.h>
  51 #include <asm/machdep.h>
  52 #include <asm/time.h>
  53 #include <asm/runlatch.h>
  54 #include <asm/syscalls.h>
  55 #include <asm/switch_to.h>
  56 #include <asm/tm.h>
  57 #include <asm/debug.h>
  58 #ifdef CONFIG_PPC64
  59 #include <asm/firmware.h>
  60 #endif
  61 #include <asm/code-patching.h>
  62 #include <asm/exec.h>
  63 #include <asm/livepatch.h>
  64 #include <asm/cpu_has_feature.h>
  65 #include <asm/asm-prototypes.h>
  66
  67 #include <linux/kprobes.h>
  68 #include <linux/kdebug.h>
  69
  70 /* Transactional Memory debug */
  71 #ifdef TM_DEBUG_SW
  72 #define TM_DEBUG(x...) printk(KERN_INFO x)
  73 #else
  74 #define TM_DEBUG(x...) do { } while(0)
  75 #endif
  76
  77 extern unsigned long _get_SP(void);
  78
  79 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  80 static void check_if_tm_restore_required(struct task_struct *tsk)
  81 {
  82         /*
  83          * If we are saving the current thread's registers, and the
  84          * thread is in a transactional state, set the TIF_RESTORE_TM
  85          * bit so that we know to restore the registers before
  86          * returning to userspace.
  87          */
  88         if (tsk == current && tsk->thread.regs &&
  89             MSR_TM_ACTIVE(tsk->thread.regs->msr) &&
  90             !test_thread_flag(TIF_RESTORE_TM)) {
  91                 tsk->thread.ckpt_regs.msr = tsk->thread.regs->msr;
  92                 set_thread_flag(TIF_RESTORE_TM);
  93         }
  94 }
  95
  96 static inline bool msr_tm_active(unsigned long msr)
  97 {
  98         return MSR_TM_ACTIVE(msr);
  99 }
 100 #else
 101 static inline bool msr_tm_active(unsigned long msr) { return false; }
 102 static inline void check_if_tm_restore_required(struct task_struct *tsk) { }
 103 #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
 104
 105 bool strict_msr_control;
 106 EXPORT_SYMBOL(strict_msr_control);
 107
 108 static int __init enable_strict_msr_control(char *str)
 109 {
 110         strict_msr_control = true;
 111         pr_info("Enabling strict facility control\n");
 112
 113         return 0;
 114 }
 115 early_param("ppc_strict_facility_enable", enable_strict_msr_control);
 116
 117 unsigned long msr_check_and_set(unsigned long bits)
 118 {
 119         unsigned long oldmsr = mfmsr();
 120         unsigned long newmsr;
 121
 122         newmsr = oldmsr | bits;
 123
 124 #ifdef CONFIG_VSX
 125         if (cpu_has_feature(CPU_FTR_VSX) && (bits & MSR_FP))
 126                 newmsr |= MSR_VSX;
 127 #endif
 128
 129         if (oldmsr != newmsr)
 130                 mtmsr_isync(newmsr);
 131
 132         return newmsr;
 133 }
 134
 135 void __msr_check_and_clear(unsigned long bits)
 136 {
 137         unsigned long oldmsr = mfmsr();
 138         unsigned long newmsr;
 139
 140         newmsr = oldmsr & ~bits;
 141
 142 #ifdef CONFIG_VSX
 143         if (cpu_has_feature(CPU_FTR_VSX) && (bits & MSR_FP))
 144                 newmsr &= ~MSR_VSX;
 145 #endif
 146
 147         if (oldmsr != newmsr)
 148                 mtmsr_isync(newmsr);
 149 }
 150 EXPORT_SYMBOL(__msr_check_and_clear);
 151
 152 #ifdef CONFIG_PPC_FPU
 153 void __giveup_fpu(struct task_struct *tsk)
 154 {
 155         unsigned long msr;
 156
 157         save_fpu(tsk);
 158         msr = tsk->thread.regs->msr;
 159         msr &= ~MSR_FP;
 160 #ifdef CONFIG_VSX
 161         if (cpu_has_feature(CPU_FTR_VSX))
 162                 msr &= ~MSR_VSX;
 163 #endif
 164         tsk->thread.regs->msr = msr;
 165 }
 166
 167 void giveup_fpu(struct task_struct *tsk)
 168 {
 169         check_if_tm_restore_required(tsk);
 170
 171         msr_check_and_set(MSR_FP);
 172         __giveup_fpu(tsk);
 173         msr_check_and_clear(MSR_FP);
 174 }
 175 EXPORT_SYMBOL(giveup_fpu);
 176
 177 /*
 178  * Make sure the floating-point register state in the
 179  * the thread_struct is up to date for task tsk.
 180  */
 181 void flush_fp_to_thread(struct task_struct *tsk)
 182 {
 183         if (tsk->thread.regs) {
 184                 /*
 185                  * We need to disable preemption here because if we didn't,
 186                  * another process could get scheduled after the regs->msr
 187                  * test but before we have finished saving the FP registers
 188                  * to the thread_struct.  That process could take over the
 189                  * FPU, and then when we get scheduled again we would store
 190                  * bogus values for the remaining FP registers.
 191                  */
 192                 preempt_disable();
 193                 if (tsk->thread.regs->msr & MSR_FP) {
 194                         /*
 195                          * This should only ever be called for current or
 196                          * for a stopped child process.  Since we save away
 197                          * the FP register state on context switch,
 198                          * there is something wrong if a stopped child appears
 199                          * to still have its FP state in the CPU registers.
 200                          */
 201                         BUG_ON(tsk != current);
 202                         giveup_fpu(tsk);
 203                 }
 204                 preempt_enable();
 205         }
 206 }
 207 EXPORT_SYMBOL_GPL(flush_fp_to_thread);
 208
 209 void enable_kernel_fp(void)
 210 {
 211         unsigned long cpumsr;
 212
 213         WARN_ON(preemptible());
 214
 215         cpumsr = msr_check_and_set(MSR_FP);
 216
 217         if (current->thread.regs && (current->thread.regs->msr & MSR_FP)) {
 218                 check_if_tm_restore_required(current);
 219                 /*
 220                  * If a thread has already been reclaimed then the
 221                  * checkpointed registers are on the CPU but have definitely
 222                  * been saved by the reclaim code. Don't need to and *cannot*
 223                  * giveup as this would save  to the 'live' structure not the
 224                  * checkpointed structure.
 225                  */
 226                 if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
 227                         return;
 228                 __giveup_fpu(current);
 229         }
 230 }
 231 EXPORT_SYMBOL(enable_kernel_fp);
 232
 233 static int restore_fp(struct task_struct *tsk) {
 234         if (tsk->thread.load_fp || msr_tm_active(tsk->thread.regs->msr)) {
 235                 load_fp_state(&current->thread.fp_state);
 236                 current->thread.load_fp++;
 237                 return 1;
 238         }
 239         return 0;
 240 }
 241 #else
 242 static int restore_fp(struct task_struct *tsk) { return 0; }
 243 #endif /* CONFIG_PPC_FPU */
 244
 245 #ifdef CONFIG_ALTIVEC
 246 #define loadvec(thr) ((thr).load_vec)
 247
 248 static void __giveup_altivec(struct task_struct *tsk)
 249 {
 250         unsigned long msr;
 251
 252         save_altivec(tsk);
 253         msr = tsk->thread.regs->msr;
 254         msr &= ~MSR_VEC;
 255 #ifdef CONFIG_VSX
 256         if (cpu_has_feature(CPU_FTR_VSX))
 257                 msr &= ~MSR_VSX;
 258 #endif
 259         tsk->thread.regs->msr = msr;
 260 }
 261
 262 void giveup_altivec(struct task_struct *tsk)
 263 {
 264         check_if_tm_restore_required(tsk);
 265
 266         msr_check_and_set(MSR_VEC);
 267         __giveup_altivec(tsk);
 268         msr_check_and_clear(MSR_VEC);
 269 }
 270 EXPORT_SYMBOL(giveup_altivec);
 271
 272 void enable_kernel_altivec(void)
 273 {
 274         unsigned long cpumsr;
 275
 276         WARN_ON(preemptible());
 277
 278         cpumsr = msr_check_and_set(MSR_VEC);
 279
 280         if (current->thread.regs && (current->thread.regs->msr & MSR_VEC)) {
 281                 check_if_tm_restore_required(current);
 282                 /*
 283                  * If a thread has already been reclaimed then the
 284                  * checkpointed registers are on the CPU but have definitely
 285                  * been saved by the reclaim code. Don't need to and *cannot*
 286                  * giveup as this would save  to the 'live' structure not the
 287                  * checkpointed structure.
 288                  */
 289                 if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
 290                         return;
 291                 __giveup_altivec(current);
 292         }
 293 }
 294 EXPORT_SYMBOL(enable_kernel_altivec);
 295
 296 /*
 297  * Make sure the VMX/Altivec register state in the
 298  * the thread_struct is up to date for task tsk.
 299  */
 300 void flush_altivec_to_thread(struct task_struct *tsk)
 301 {
 302         if (tsk->thread.regs) {
 303                 preempt_disable();
 304                 if (tsk->thread.regs->msr & MSR_VEC) {
 305                         BUG_ON(tsk != current);
 306                         giveup_altivec(tsk);
 307                 }
 308                 preempt_enable();
 309         }
 310 }
 311 EXPORT_SYMBOL_GPL(flush_altivec_to_thread);
 312
 313 static int restore_altivec(struct task_struct *tsk)
 314 {
 315         if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
 316                 (tsk->thread.load_vec || msr_tm_active(tsk->thread.regs->msr))) {
 317                 load_vr_state(&tsk->thread.vr_state);
 318                 tsk->thread.used_vr = 1;
 319                 tsk->thread.load_vec++;
 320
 321                 return 1;
 322         }
 323         return 0;
 324 }
 325 #else
 326 #define loadvec(thr) 0
 327 static inline int restore_altivec(struct task_struct *tsk) { return 0; }
 328 #endif /* CONFIG_ALTIVEC */
 329
 330 #ifdef CONFIG_VSX
 331 static void __giveup_vsx(struct task_struct *tsk)
 332 {
 333         if (tsk->thread.regs->msr & MSR_FP)
 334                 __giveup_fpu(tsk);
 335         if (tsk->thread.regs->msr & MSR_VEC)
 336                 __giveup_altivec(tsk);
 337         tsk->thread.regs->msr &= ~MSR_VSX;
 338 }
 339
 340 static void giveup_vsx(struct task_struct *tsk)
 341 {
 342         check_if_tm_restore_required(tsk);
 343
 344         msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
 345         __giveup_vsx(tsk);
 346         msr_check_and_clear(MSR_FP|MSR_VEC|MSR_VSX);
 347 }
 348
 349 static void save_vsx(struct task_struct *tsk)
 350 {
 351         if (tsk->thread.regs->msr & MSR_FP)
 352                 save_fpu(tsk);
 353         if (tsk->thread.regs->msr & MSR_VEC)
 354                 save_altivec(tsk);
 355 }
 356
 357 void enable_kernel_vsx(void)
 358 {
 359         unsigned long cpumsr;
 360
 361         WARN_ON(preemptible());
 362
 363         cpumsr = msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
 364
 365         if (current->thread.regs &&
 366             (current->thread.regs->msr & (MSR_VSX|MSR_VEC|MSR_FP))) {
 367                 check_if_tm_restore_required(current);
 368                 /*
 369                  * If a thread has already been reclaimed then the
 370                  * checkpointed registers are on the CPU but have definitely
 371                  * been saved by the reclaim code. Don't need to and *cannot*
 372                  * giveup as this would save  to the 'live' structure not the
 373                  * checkpointed structure.
 374                  */
 375                 if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
 376                         return;
 377                 if (current->thread.regs->msr & MSR_FP)
 378                         __giveup_fpu(current);
 379                 if (current->thread.regs->msr & MSR_VEC)
 380                         __giveup_altivec(current);
 381                 __giveup_vsx(current);
 382         }
 383 }
 384 EXPORT_SYMBOL(enable_kernel_vsx);
 385
 386 void flush_vsx_to_thread(struct task_struct *tsk)
 387 {
 388         if (tsk->thread.regs) {
 389                 preempt_disable();
 390                 if (tsk->thread.regs->msr & (MSR_VSX|MSR_VEC|MSR_FP)) {
 391                         BUG_ON(tsk != current);
 392                         giveup_vsx(tsk);
 393                 }
 394                 preempt_enable();
 395         }
 396 }
 397 EXPORT_SYMBOL_GPL(flush_vsx_to_thread);
 398
 399 static int restore_vsx(struct task_struct *tsk)
 400 {
 401         if (cpu_has_feature(CPU_FTR_VSX)) {
 402                 tsk->thread.used_vsr = 1;
 403                 return 1;
 404         }
 405
 406         return 0;
 407 }
 408 #else
 409 static inline int restore_vsx(struct task_struct *tsk) { return 0; }
 410 static inline void save_vsx(struct task_struct *tsk) { }
 411 #endif /* CONFIG_VSX */
 412
 413 #ifdef CONFIG_SPE
 414 void giveup_spe(struct task_struct *tsk)
 415 {
 416         check_if_tm_restore_required(tsk);
 417
 418         msr_check_and_set(MSR_SPE);
 419         __giveup_spe(tsk);
 420         msr_check_and_clear(MSR_SPE);
 421 }
 422 EXPORT_SYMBOL(giveup_spe);
 423
 424 void enable_kernel_spe(void)
 425 {
 426         WARN_ON(preemptible());
 427
 428         msr_check_and_set(MSR_SPE);
 429
 430         if (current->thread.regs && (current->thread.regs->msr & MSR_SPE)) {
 431                 check_if_tm_restore_required(current);
 432                 __giveup_spe(current);
 433         }
 434 }
 435 EXPORT_SYMBOL(enable_kernel_spe);
 436
 437 void flush_spe_to_thread(struct task_struct *tsk)
 438 {
 439         if (tsk->thread.regs) {
 440                 preempt_disable();
 441                 if (tsk->thread.regs->msr & MSR_SPE) {
 442                         BUG_ON(tsk != current);
 443                         tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
 444                         giveup_spe(tsk);
 445                 }
 446                 preempt_enable();
 447         }
 448 }
 449 #endif /* CONFIG_SPE */
 450
 451 static unsigned long msr_all_available;
 452
 453 static int __init init_msr_all_available(void)
 454 {
 455 #ifdef CONFIG_PPC_FPU
 456         msr_all_available |= MSR_FP;
 457 #endif
 458 #ifdef CONFIG_ALTIVEC
 459         if (cpu_has_feature(CPU_FTR_ALTIVEC))
 460                 msr_all_available |= MSR_VEC;
 461 #endif
 462 #ifdef CONFIG_VSX
 463         if (cpu_has_feature(CPU_FTR_VSX))
 464                 msr_all_available |= MSR_VSX;
 465 #endif
 466 #ifdef CONFIG_SPE
 467         if (cpu_has_feature(CPU_FTR_SPE))
 468                 msr_all_available |= MSR_SPE;
 469 #endif
 470
 471         return 0;
 472 }
 473 early_initcall(init_msr_all_available);
 474
 475 void giveup_all(struct task_struct *tsk)
 476 {
 477         unsigned long usermsr;
 478
 479         if (!tsk->thread.regs)
 480                 return;
 481
 482         usermsr = tsk->thread.regs->msr;
 483
 484         if ((usermsr & msr_all_available) == 0)
 485                 return;
 486
 487         msr_check_and_set(msr_all_available);
 488         check_if_tm_restore_required(tsk);
 489
 490 #ifdef CONFIG_PPC_FPU
 491         if (usermsr & MSR_FP)
 492                 __giveup_fpu(tsk);
 493 #endif
 494 #ifdef CONFIG_ALTIVEC
 495         if (usermsr & MSR_VEC)
 496                 __giveup_altivec(tsk);
 497 #endif
 498 #ifdef CONFIG_VSX
 499         if (usermsr & MSR_VSX)
 500                 __giveup_vsx(tsk);
 501 #endif
 502 #ifdef CONFIG_SPE
 503         if (usermsr & MSR_SPE)
 504                 __giveup_spe(tsk);
 505 #endif
 506
 507         msr_check_and_clear(msr_all_available);
 508 }
 509 EXPORT_SYMBOL(giveup_all);
 510
 511 void restore_math(struct pt_regs *regs)
 512 {
 513         unsigned long msr;
 514
 515         if (!msr_tm_active(regs->msr) &&
 516                 !current->thread.load_fp && !loadvec(current->thread))
 517                 return;
 518
 519         msr = regs->msr;
 520         msr_check_and_set(msr_all_available);
 521
 522         /*
 523          * Only reload if the bit is not set in the user MSR, the bit BEING set
 524          * indicates that the registers are hot
 525          */
 526         if ((!(msr & MSR_FP)) && restore_fp(current))
 527                 msr |= MSR_FP | current->thread.fpexc_mode;
 528
 529         if ((!(msr & MSR_VEC)) && restore_altivec(current))
 530                 msr |= MSR_VEC;
 531
 532         if ((msr & (MSR_FP | MSR_VEC)) == (MSR_FP | MSR_VEC) &&
 533                         restore_vsx(current)) {
 534                 msr |= MSR_VSX;
 535         }
 536
 537         msr_check_and_clear(msr_all_available);
 538
 539         regs->msr = msr;
 540 }
 541
 542 void save_all(struct task_struct *tsk)
 543 {
 544         unsigned long usermsr;
 545
 546         if (!tsk->thread.regs)
 547                 return;
 548
 549         usermsr = tsk->thread.regs->msr;
 550
 551         if ((usermsr & msr_all_available) == 0)
 552                 return;
 553
 554         msr_check_and_set(msr_all_available);
 555
 556         /*
 557          * Saving the way the register space is in hardware, save_vsx boils
 558          * down to a save_fpu() and save_altivec()
 559          */
 560         if (usermsr & MSR_VSX) {
 561                 save_vsx(tsk);
 562         } else {
 563                 if (usermsr & MSR_FP)
 564                         save_fpu(tsk);
 565
 566                 if (usermsr & MSR_VEC)
 567                         save_altivec(tsk);
 568         }
 569
 570         if (usermsr & MSR_SPE)
 571                 __giveup_spe(tsk);
 572
 573         msr_check_and_clear(msr_all_available);
 574 }
 575
 576 void flush_all_to_thread(struct task_struct *tsk)
 577 {
 578         if (tsk->thread.regs) {
 579                 preempt_disable();
 580                 BUG_ON(tsk != current);
 581                 save_all(tsk);
 582
 583 #ifdef CONFIG_SPE
 584                 if (tsk->thread.regs->msr & MSR_SPE)
 585                         tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
 586 #endif
 587
 588                 preempt_enable();
 589         }
 590 }
 591 EXPORT_SYMBOL(flush_all_to_thread);
 592
 593 #ifdef CONFIG_PPC_ADV_DEBUG_REGS
 594 void do_send_trap(struct pt_regs *regs, unsigned long address,
 595                   unsigned long error_code, int signal_code, int breakpt)
 596 {
 597         siginfo_t info;
 598
 599         current->thread.trap_nr = signal_code;
 600         if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
 601                         11, SIGSEGV) == NOTIFY_STOP)
 602                 return;
 603
 604         /* Deliver the signal to userspace */
 605         info.si_signo = SIGTRAP;
 606         info.si_errno = breakpt;        /* breakpoint or watchpoint id */
 607         info.si_code = signal_code;
 608         info.si_addr = (void __user *)address;
 609         force_sig_info(SIGTRAP, &info, current);
 610 }
 611 #else   /* !CONFIG_PPC_ADV_DEBUG_REGS */
 612 void do_break (struct pt_regs *regs, unsigned long address,
 613                     unsigned long error_code)
 614 {
 615         siginfo_t info;
 616
 617         current->thread.trap_nr = TRAP_HWBKPT;
 618         if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
 619                         11, SIGSEGV) == NOTIFY_STOP)
 620                 return;
 621
 622         if (debugger_break_match(regs))
 623                 return;
 624
 625         /* Clear the breakpoint */
 626         hw_breakpoint_disable();
 627
 628         /* Deliver the signal to userspace */
 629         info.si_signo = SIGTRAP;
 630         info.si_errno = 0;
 631         info.si_code = TRAP_HWBKPT;
 632         info.si_addr = (void __user *)address;
 633         force_sig_info(SIGTRAP, &info, current);
 634 }
 635 #endif  /* CONFIG_PPC_ADV_DEBUG_REGS */
 636
 637 static DEFINE_PER_CPU(struct arch_hw_breakpoint, current_brk);
 638
 639 #ifdef CONFIG_PPC_ADV_DEBUG_REGS
 640 /*
 641  * Set the debug registers back to their default "safe" values.
 642  */
 643 static void set_debug_reg_defaults(struct thread_struct *thread)
 644 {
 645         thread->debug.iac1 = thread->debug.iac2 = 0;
 646 #if CONFIG_PPC_ADV_DEBUG_IACS > 2
 647         thread->debug.iac3 = thread->debug.iac4 = 0;
 648 #endif
 649         thread->debug.dac1 = thread->debug.dac2 = 0;
 650 #if CONFIG_PPC_ADV_DEBUG_DVCS > 0
 651         thread->debug.dvc1 = thread->debug.dvc2 = 0;
 652 #endif
 653         thread->debug.dbcr0 = 0;
 654 #ifdef CONFIG_BOOKE
 655         /*
 656          * Force User/Supervisor bits to b11 (user-only MSR[PR]=1)
 657          */
 658         thread->debug.dbcr1 = DBCR1_IAC1US | DBCR1_IAC2US |
 659                         DBCR1_IAC3US | DBCR1_IAC4US;
 660         /*
 661          * Force Data Address Compare User/Supervisor bits to be User-only
 662          * (0b11 MSR[PR]=1) and set all other bits in DBCR2 register to be 0.
 663          */
 664         thread->debug.dbcr2 = DBCR2_DAC1US | DBCR2_DAC2US;
 665 #else
 666         thread->debug.dbcr1 = 0;
 667 #endif
 668 }
 669
 670 static void prime_debug_regs(struct debug_reg *debug)
 671 {
 672         /*
 673          * We could have inherited MSR_DE from userspace, since
 674          * it doesn't get cleared on exception entry.  Make sure
 675          * MSR_DE is clear before we enable any debug events.
 676          */
 677         mtmsr(mfmsr() & ~MSR_DE);
 678
 679         mtspr(SPRN_IAC1, debug->iac1);
 680         mtspr(SPRN_IAC2, debug->iac2);
 681 #if CONFIG_PPC_ADV_DEBUG_IACS > 2
 682         mtspr(SPRN_IAC3, debug->iac3);
 683         mtspr(SPRN_IAC4, debug->iac4);
 684 #endif
 685         mtspr(SPRN_DAC1, debug->dac1);
 686         mtspr(SPRN_DAC2, debug->dac2);
 687 #if CONFIG_PPC_ADV_DEBUG_DVCS > 0
 688         mtspr(SPRN_DVC1, debug->dvc1);
 689         mtspr(SPRN_DVC2, debug->dvc2);
 690 #endif
 691         mtspr(SPRN_DBCR0, debug->dbcr0);
 692         mtspr(SPRN_DBCR1, debug->dbcr1);
 693 #ifdef CONFIG_BOOKE
 694         mtspr(SPRN_DBCR2, debug->dbcr2);
 695 #endif
 696 }
 697 /*
 698  * Unless neither the old or new thread are making use of the
 699  * debug registers, set the debug registers from the values
 700  * stored in the new thread.
 701  */
 702 void switch_booke_debug_regs(struct debug_reg *new_debug)
 703 {
 704         if ((current->thread.debug.dbcr0 & DBCR0_IDM)
 705                 || (new_debug->dbcr0 & DBCR0_IDM))
 706                         prime_debug_regs(new_debug);
 707 }
 708 EXPORT_SYMBOL_GPL(switch_booke_debug_regs);
 709 #else   /* !CONFIG_PPC_ADV_DEBUG_REGS */
 710 #ifndef CONFIG_HAVE_HW_BREAKPOINT
 711 static void set_debug_reg_defaults(struct thread_struct *thread)
 712 {
 713         thread->hw_brk.address = 0;
 714         thread->hw_brk.type = 0;
 715         set_breakpoint(&thread->hw_brk);
 716 }
 717 #endif /* !CONFIG_HAVE_HW_BREAKPOINT */
 718 #endif  /* CONFIG_PPC_ADV_DEBUG_REGS */
 719
 720 #ifdef CONFIG_PPC_ADV_DEBUG_REGS
 721 static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
 722 {
 723         mtspr(SPRN_DAC1, dabr);
 724 #ifdef CONFIG_PPC_47x
 725         isync();
 726 #endif
 727         return 0;
 728 }
 729 #elif defined(CONFIG_PPC_BOOK3S)
 730 static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
 731 {
 732         mtspr(SPRN_DABR, dabr);
 733         if (cpu_has_feature(CPU_FTR_DABRX))
 734                 mtspr(SPRN_DABRX, dabrx);
 735         return 0;
 736 }
 737 #elif defined(CONFIG_PPC_8xx)
 738 static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
 739 {
 740         unsigned long addr = dabr & ~HW_BRK_TYPE_DABR;
 741         unsigned long lctrl1 = 0x90000000; /* compare type: equal on E & F */
 742         unsigned long lctrl2 = 0x8e000002; /* watchpoint 1 on cmp E | F */
 743
 744         if ((dabr & HW_BRK_TYPE_RDWR) == HW_BRK_TYPE_READ)
 745                 lctrl1 |= 0xa0000;
 746         else if ((dabr & HW_BRK_TYPE_RDWR) == HW_BRK_TYPE_WRITE)
 747                 lctrl1 |= 0xf0000;
 748         else if ((dabr & HW_BRK_TYPE_RDWR) == 0)
 749                 lctrl2 = 0;
 750
 751         mtspr(SPRN_LCTRL2, 0);
 752         mtspr(SPRN_CMPE, addr);
 753         mtspr(SPRN_CMPF, addr + 4);
 754         mtspr(SPRN_LCTRL1, lctrl1);
 755         mtspr(SPRN_LCTRL2, lctrl2);
 756
 757         return 0;
 758 }
 759 #else
 760 static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
 761 {
 762         return -EINVAL;
 763 }
 764 #endif
 765
 766 static inline int set_dabr(struct arch_hw_breakpoint *brk)
 767 {
 768         unsigned long dabr, dabrx;
 769
 770         dabr = brk->address | (brk->type & HW_BRK_TYPE_DABR);
 771         dabrx = ((brk->type >> 3) & 0x7);
 772
 773         if (ppc_md.set_dabr)
 774                 return ppc_md.set_dabr(dabr, dabrx);
 775
 776         return __set_dabr(dabr, dabrx);
 777 }
 778
 779 static inline int set_dawr(struct arch_hw_breakpoint *brk)
 780 {
 781         unsigned long dawr, dawrx, mrd;
 782
 783         dawr = brk->address;
 784
 785         dawrx  = (brk->type & (HW_BRK_TYPE_READ | HW_BRK_TYPE_WRITE)) \
 786                                    << (63 - 58); //* read/write bits */
 787         dawrx |= ((brk->type & (HW_BRK_TYPE_TRANSLATE)) >> 2) \
 788                                    << (63 - 59); //* translate */
 789         dawrx |= (brk->type & (HW_BRK_TYPE_PRIV_ALL)) \
 790                                    >> 3; //* PRIM bits */
 791         /* dawr length is stored in field MDR bits 48:53.  Matches range in
 792            doublewords (64 bits) baised by -1 eg. 0b000000=1DW and
 793            0b111111=64DW.
 794            brk->len is in bytes.
 795            This aligns up to double word size, shifts and does the bias.
 796         */
 797         mrd = ((brk->len + 7) >> 3) - 1;
 798         dawrx |= (mrd & 0x3f) << (63 - 53);
 799
 800         if (ppc_md.set_dawr)
 801                 return ppc_md.set_dawr(dawr, dawrx);
 802         mtspr(SPRN_DAWR, dawr);
 803         mtspr(SPRN_DAWRX, dawrx);
 804         return 0;
 805 }
 806
 807 void __set_breakpoint(struct arch_hw_breakpoint *brk)
 808 {
 809         memcpy(this_cpu_ptr(&current_brk), brk, sizeof(*brk));
 810
 811         if (cpu_has_feature(CPU_FTR_DAWR))
 812                 set_dawr(brk);
 813         else
 814                 set_dabr(brk);
 815 }
 816
 817 void set_breakpoint(struct arch_hw_breakpoint *brk)
 818 {
 819         preempt_disable();
 820         __set_breakpoint(brk);
 821         preempt_enable();
 822 }
 823
 824 #ifdef CONFIG_PPC64
 825 DEFINE_PER_CPU(struct cpu_usage, cpu_usage_array);
 826 #endif
 827
 828 static inline bool hw_brk_match(struct arch_hw_breakpoint *a,
 829                               struct arch_hw_breakpoint *b)
 830 {
 831         if (a->address != b->address)
 832                 return false;
 833         if (a->type != b->type)
 834                 return false;
 835         if (a->len != b->len)
 836                 return false;
 837         return true;
 838 }
 839
 840 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 841
 842 static inline bool tm_enabled(struct task_struct *tsk)
 843 {
 844         return tsk && tsk->thread.regs && (tsk->thread.regs->msr & MSR_TM);
 845 }
 846
 847 static void tm_reclaim_thread(struct thread_struct *thr,
 848                               struct thread_info *ti, uint8_t cause)
 849 {
 850         /*
 851          * Use the current MSR TM suspended bit to track if we have
 852          * checkpointed state outstanding.
 853          * On signal delivery, we'd normally reclaim the checkpointed
 854          * state to obtain stack pointer (see:get_tm_stackpointer()).
 855          * This will then directly return to userspace without going
 856          * through __switch_to(). However, if the stack frame is bad,
 857          * we need to exit this thread which calls __switch_to() which
 858          * will again attempt to reclaim the already saved tm state.
 859          * Hence we need to check that we've not already reclaimed
 860          * this state.
 861          * We do this using the current MSR, rather tracking it in
 862          * some specific thread_struct bit, as it has the additional
 863          * benefit of checking for a potential TM bad thing exception.
 864          */
 865         if (!MSR_TM_SUSPENDED(mfmsr()))
 866                 return;
 867
 868         /*
 869          * If we are in a transaction and FP is off then we can't have
 870          * used FP inside that transaction. Hence the checkpointed
 871          * state is the same as the live state. We need to copy the
 872          * live state to the checkpointed state so that when the
 873          * transaction is restored, the checkpointed state is correct
 874          * and the aborted transaction sees the correct state. We use
 875          * ckpt_regs.msr here as that's what tm_reclaim will use to
 876          * determine if it's going to write the checkpointed state or
 877          * not. So either this will write the checkpointed registers,
 878          * or reclaim will. Similarly for VMX.
 879          */
 880         if ((thr->ckpt_regs.msr & MSR_FP) == 0)
 881                 memcpy(&thr->ckfp_state, &thr->fp_state,
 882                        sizeof(struct thread_fp_state));
 883         if ((thr->ckpt_regs.msr & MSR_VEC) == 0)
 884                 memcpy(&thr->ckvr_state, &thr->vr_state,
 885                        sizeof(struct thread_vr_state));
 886
 887         giveup_all(container_of(thr, struct task_struct, thread));
 888
 889         tm_reclaim(thr, thr->ckpt_regs.msr, cause);
 890 }
 891
 892 void tm_reclaim_current(uint8_t cause)
 893 {
 894         tm_enable();
 895         tm_reclaim_thread(&current->thread, current_thread_info(), cause);
 896 }
 897
 898 static inline void tm_reclaim_task(struct task_struct *tsk)
 899 {
 900         /* We have to work out if we're switching from/to a task that's in the
 901          * middle of a transaction.
 902          *
 903          * In switching we need to maintain a 2nd register state as
 904          * oldtask->thread.ckpt_regs.  We tm_reclaim(oldproc); this saves the
 905          * checkpointed (tbegin) state in ckpt_regs, ckfp_state and
 906          * ckvr_state
 907          *
 908          * We also context switch (save) TFHAR/TEXASR/TFIAR in here.
 909          */
 910         struct thread_struct *thr = &tsk->thread;
 911
 912         if (!thr->regs)
 913                 return;
 914
 915         if (!MSR_TM_ACTIVE(thr->regs->msr))
 916                 goto out_and_saveregs;
 917
 918         TM_DEBUG("--- tm_reclaim on pid %d (NIP=%lx, "
 919                  "ccr=%lx, msr=%lx, trap=%lx)\n",
 920                  tsk->pid, thr->regs->nip,
 921                  thr->regs->ccr, thr->regs->msr,
 922                  thr->regs->trap);
 923
 924         tm_reclaim_thread(thr, task_thread_info(tsk), TM_CAUSE_RESCHED);
 925
 926         TM_DEBUG("--- tm_reclaim on pid %d complete\n",
 927                  tsk->pid);
 928
 929 out_and_saveregs:
 930         /* Always save the regs here, even if a transaction's not active.
 931          * This context-switches a thread's TM info SPRs.  We do it here to
 932          * be consistent with the restore path (in recheckpoint) which
 933          * cannot happen later in _switch().
 934          */
 935         tm_save_sprs(thr);
 936 }
 937
 938 extern void __tm_recheckpoint(struct thread_struct *thread,
 939                               unsigned long orig_msr);
 940
 941 void tm_recheckpoint(struct thread_struct *thread,
 942                      unsigned long orig_msr)
 943 {
 944         unsigned long flags;
 945
 946         if (!(thread->regs->msr & MSR_TM))
 947                 return;
 948
 949         /* We really can't be interrupted here as the TEXASR registers can't
 950          * change and later in the trecheckpoint code, we have a userspace R1.
 951          * So let's hard disable over this region.
 952          */
 953         local_irq_save(flags);
 954         hard_irq_disable();
 955
 956         /* The TM SPRs are restored here, so that TEXASR.FS can be set
 957          * before the trecheckpoint and no explosion occurs.
 958          */
 959         tm_restore_sprs(thread);
 960
 961         __tm_recheckpoint(thread, orig_msr);
 962
 963         local_irq_restore(flags);
 964 }
 965
 966 static inline void tm_recheckpoint_new_task(struct task_struct *new)
 967 {
 968         unsigned long msr;
 969
 970         if (!cpu_has_feature(CPU_FTR_TM))
 971                 return;
 972
 973         /* Recheckpoint the registers of the thread we're about to switch to.
 974          *
 975          * If the task was using FP, we non-lazily reload both the original and
 976          * the speculative FP register states.  This is because the kernel
 977          * doesn't see if/when a TM rollback occurs, so if we take an FP
 978          * unavailable later, we are unable to determine which set of FP regs
 979          * need to be restored.
 980          */
 981         if (!tm_enabled(new))
 982                 return;
 983
 984         if (!MSR_TM_ACTIVE(new->thread.regs->msr)){
 985                 tm_restore_sprs(&new->thread);
 986                 return;
 987         }
 988         msr = new->thread.ckpt_regs.msr;
 989         /* Recheckpoint to restore original checkpointed register state. */
 990         TM_DEBUG("*** tm_recheckpoint of pid %d "
 991                  "(new->msr 0x%lx, new->origmsr 0x%lx)\n",
 992                  new->pid, new->thread.regs->msr, msr);
 993
 994         tm_recheckpoint(&new->thread, msr);
 995
 996         /*
 997          * The checkpointed state has been restored but the live state has
 998          * not, ensure all the math functionality is turned off to trigger
 999          * restore_math() to reload.
1000          */
1001         new->thread.regs->msr &= ~(MSR_FP | MSR_VEC | MSR_VSX);
1002
1003         TM_DEBUG("*** tm_recheckpoint of pid %d complete "
1004                  "(kernel msr 0x%lx)\n",
1005                  new->pid, mfmsr());
1006 }
1007
1008 static inline void __switch_to_tm(struct task_struct *prev,
1009                 struct task_struct *new)
1010 {
1011         if (cpu_has_feature(CPU_FTR_TM)) {
1012                 if (tm_enabled(prev) || tm_enabled(new))
1013                         tm_enable();
1014
1015                 if (tm_enabled(prev)) {
1016                         prev->thread.load_tm++;
1017                         tm_reclaim_task(prev);
1018                         if (!MSR_TM_ACTIVE(prev->thread.regs->msr) && prev->thread.load_tm == 0)
1019                                 prev->thread.regs->msr &= ~MSR_TM;
1020                 }
1021
1022                 tm_recheckpoint_new_task(new);
1023         }
1024 }
1025
1026 /*
1027  * This is called if we are on the way out to userspace and the
1028  * TIF_RESTORE_TM flag is set.  It checks if we need to reload
1029  * FP and/or vector state and does so if necessary.
1030  * If userspace is inside a transaction (whether active or
1031  * suspended) and FP/VMX/VSX instructions have ever been enabled
1032  * inside that transaction, then we have to keep them enabled
1033  * and keep the FP/VMX/VSX state loaded while ever the transaction
1034  * continues.  The reason is that if we didn't, and subsequently
1035  * got a FP/VMX/VSX unavailable interrupt inside a transaction,
1036  * we don't know whether it's the same transaction, and thus we
1037  * don't know which of the checkpointed state and the transactional
1038  * state to use.
1039  */
1040 void restore_tm_state(struct pt_regs *regs)
1041 {
1042         unsigned long msr_diff;
1043
1044         /*
1045          * This is the only moment we should clear TIF_RESTORE_TM as
1046          * it is here that ckpt_regs.msr and pt_regs.msr become the same
1047          * again, anything else could lead to an incorrect ckpt_msr being
1048          * saved and therefore incorrect signal contexts.
1049          */
1050         clear_thread_flag(TIF_RESTORE_TM);
1051         if (!MSR_TM_ACTIVE(regs->msr))
1052                 return;
1053
1054         msr_diff = current->thread.ckpt_regs.msr & ~regs->msr;
1055         msr_diff &= MSR_FP | MSR_VEC | MSR_VSX;
1056
1057         /* Ensure that restore_math() will restore */
1058         if (msr_diff & MSR_FP)
1059                 current->thread.load_fp = 1;
1060 #ifdef CONFIG_ALTIVEC
1061         if (cpu_has_feature(CPU_FTR_ALTIVEC) && msr_diff & MSR_VEC)
1062                 current->thread.load_vec = 1;
1063 #endif
1064         restore_math(regs);
1065
1066         regs->msr |= msr_diff;
1067 }
1068
1069 #else
1070 #define tm_recheckpoint_new_task(new)
1071 #define __switch_to_tm(prev, new)
1072 #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
1073
1074 static inline void save_sprs(struct thread_struct *t)
1075 {
1076 #ifdef CONFIG_ALTIVEC
1077         if (cpu_has_feature(CPU_FTR_ALTIVEC))
1078                 t->vrsave = mfspr(SPRN_VRSAVE);
1079 #endif
1080 #ifdef CONFIG_PPC_BOOK3S_64
1081         if (cpu_has_feature(CPU_FTR_DSCR))
1082                 t->dscr = mfspr(SPRN_DSCR);
1083
1084         if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
1085                 t->bescr = mfspr(SPRN_BESCR);
1086                 t->ebbhr = mfspr(SPRN_EBBHR);
1087                 t->ebbrr = mfspr(SPRN_EBBRR);
1088
1089                 t->fscr = mfspr(SPRN_FSCR);
1090
1091                 /*
1092                  * Note that the TAR is not available for use in the kernel.
1093                  * (To provide this, the TAR should be backed up/restored on
1094                  * exception entry/exit instead, and be in pt_regs.  FIXME,
1095                  * this should be in pt_regs anyway (for debug).)
1096                  */
1097                 t->tar = mfspr(SPRN_TAR);
1098         }
1099 #endif
1100 }
1101
1102 static inline void restore_sprs(struct thread_struct *old_thread,
1103                                 struct thread_struct *new_thread)
1104 {
1105 #ifdef CONFIG_ALTIVEC
1106         if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
1107             old_thread->vrsave != new_thread->vrsave)
1108                 mtspr(SPRN_VRSAVE, new_thread->vrsave);
1109 #endif
1110 #ifdef CONFIG_PPC_BOOK3S_64
1111         if (cpu_has_feature(CPU_FTR_DSCR)) {
1112                 u64 dscr = get_paca()->dscr_default;
1113                 if (new_thread->dscr_inherit)
1114                         dscr = new_thread->dscr;
1115
1116                 if (old_thread->dscr != dscr)
1117                         mtspr(SPRN_DSCR, dscr);
1118         }
1119
1120         if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
1121                 if (old_thread->bescr != new_thread->bescr)
1122                         mtspr(SPRN_BESCR, new_thread->bescr);
1123                 if (old_thread->ebbhr != new_thread->ebbhr)
1124                         mtspr(SPRN_EBBHR, new_thread->ebbhr);
1125                 if (old_thread->ebbrr != new_thread->ebbrr)
1126                         mtspr(SPRN_EBBRR, new_thread->ebbrr);
1127
1128                 if (old_thread->fscr != new_thread->fscr)
1129                         mtspr(SPRN_FSCR, new_thread->fscr);
1130
1131                 if (old_thread->tar != new_thread->tar)
1132                         mtspr(SPRN_TAR, new_thread->tar);
1133         }
1134 #endif
1135 }
1136
1137 #ifdef CONFIG_PPC_BOOK3S_64
1138 #define CP_SIZE 128
1139 static const u8 dummy_copy_buffer[CP_SIZE] __attribute__((aligned(CP_SIZE)));
1140 #endif
1141
1142 struct task_struct *__switch_to(struct task_struct *prev,
1143         struct task_struct *new)
1144 {
1145         struct thread_struct *new_thread, *old_thread;
1146         struct task_struct *last;
1147 #ifdef CONFIG_PPC_BOOK3S_64
1148         struct ppc64_tlb_batch *batch;
1149 #endif
1150
1151         new_thread = &new->thread;
1152         old_thread = &current->thread;
1153
1154         WARN_ON(!irqs_disabled());
1155
1156 #ifdef CONFIG_PPC64
1157         /*
1158          * Collect processor utilization data per process
1159          */
1160         if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
1161                 struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
1162                 long unsigned start_tb, current_tb;
1163                 start_tb = old_thread->start_tb;
1164                 cu->current_tb = current_tb = mfspr(SPRN_PURR);
1165                 old_thread->accum_tb += (current_tb - start_tb);
1166                 new_thread->start_tb = current_tb;
1167         }
1168 #endif /* CONFIG_PPC64 */
1169
1170 #ifdef CONFIG_PPC_STD_MMU_64
1171         batch = this_cpu_ptr(&ppc64_tlb_batch);
1172         if (batch->active) {
1173                 current_thread_info()->local_flags |= _TLF_LAZY_MMU;
1174                 if (batch->index)
1175                         __flush_tlb_pending(batch);
1176                 batch->active = 0;
1177         }
1178 #endif /* CONFIG_PPC_STD_MMU_64 */
1179
1180 #ifdef CONFIG_PPC_ADV_DEBUG_REGS
1181         switch_booke_debug_regs(&new->thread.debug);
1182 #else
1183 /*
1184  * For PPC_BOOK3S_64, we use the hw-breakpoint interfaces that would
1185  * schedule DABR
1186  */
1187 #ifndef CONFIG_HAVE_HW_BREAKPOINT
1188         if (unlikely(!hw_brk_match(this_cpu_ptr(&current_brk), &new->thread.hw_brk)))
1189                 __set_breakpoint(&new->thread.hw_brk);
1190 #endif /* CONFIG_HAVE_HW_BREAKPOINT */
1191 #endif
1192
1193         /*
1194          * We need to save SPRs before treclaim/trecheckpoint as these will
1195          * change a number of them.
1196          */
1197         save_sprs(&prev->thread);
1198
1199         /* Save FPU, Altivec, VSX and SPE state */
1200         giveup_all(prev);
1201
1202         __switch_to_tm(prev, new);
1203
1204         if (!radix_enabled()) {
1205                 /*
1206                  * We can't take a PMU exception inside _switch() since there
1207                  * is a window where the kernel stack SLB and the kernel stack
1208                  * are out of sync. Hard disable here.
1209                  */
1210                 hard_irq_disable();
1211         }
1212
1213         /*
1214          * Call restore_sprs() before calling _switch(). If we move it after
1215          * _switch() then we miss out on calling it for new tasks. The reason
1216          * for this is we manually create a stack frame for new tasks that
1217          * directly returns through ret_from_fork() or
1218          * ret_from_kernel_thread(). See copy_thread() for details.
1219          */
1220         restore_sprs(old_thread, new_thread);
1221
1222         last = _switch(old_thread, new_thread);
1223
1224 #ifdef CONFIG_PPC_STD_MMU_64
1225         if (current_thread_info()->local_flags & _TLF_LAZY_MMU) {
1226                 current_thread_info()->local_flags &= ~_TLF_LAZY_MMU;
1227                 batch = this_cpu_ptr(&ppc64_tlb_batch);
1228                 batch->active = 1;
1229         }
1230
1231         if (current_thread_info()->task->thread.regs) {
1232                 restore_math(current_thread_info()->task->thread.regs);
1233
1234                 /*
1235                  * The copy-paste buffer can only store into foreign real
1236                  * addresses, so unprivileged processes can not see the
1237                  * data or use it in any way unless they have foreign real
1238                  * mappings. We don't have a VAS driver that allocates those
1239                  * yet, so no cpabort is required.
1240                  */
1241                 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
1242                         /*
1243                          * DD1 allows paste into normal system memory, so we
1244                          * do an unpaired copy here to clear the buffer and
1245                          * prevent a covert channel being set up.
1246                          *
1247                          * cpabort is not used because it is quite expensive.
1248                          */
1249                         asm volatile(PPC_COPY(%0, %1)
1250                                         : : "r"(dummy_copy_buffer), "r"(0));
1251                 }
1252         }
1253 #endif /* CONFIG_PPC_STD_MMU_64 */
1254
1255         return last;
1256 }
1257
1258 static int instructions_to_print = 16;
1259
1260 static void show_instructions(struct pt_regs *regs)
1261 {
1262         int i;
1263         unsigned long pc = regs->nip - (instructions_to_print * 3 / 4 *
1264                         sizeof(int));
1265
1266         printk("Instruction dump:");
1267
1268         for (i = 0; i < instructions_to_print; i++) {
1269                 int instr;
1270
1271                 if (!(i % 8))
1272                         pr_cont("\n");
1273
1274 #if !defined(CONFIG_BOOKE)
1275                 /* If executing with the IMMU off, adjust pc rather
1276                  * than print XXXXXXXX.
1277                  */
1278                 if (!(regs->msr & MSR_IR))
1279                         pc = (unsigned long)phys_to_virt(pc);
1280 #endif
1281
1282                 if (!__kernel_text_address(pc) ||
1283                      probe_kernel_address((unsigned int __user *)pc, instr)) {
1284                         pr_cont("XXXXXXXX ");
1285                 } else {
1286                         if (regs->nip == pc)
1287                                 pr_cont("<%08x> ", instr);
1288                         else
1289                                 pr_cont("%08x ", instr);
1290                 }
1291
1292                 pc += sizeof(int);
1293         }
1294
1295         pr_cont("\n");
1296 }
1297
1298 struct regbit {
1299         unsigned long bit;
1300         const char *name;
1301 };
1302
1303 static struct regbit msr_bits[] = {
1304 #if defined(CONFIG_PPC64) && !defined(CONFIG_BOOKE)
1305         {MSR_SF,        "SF"},
1306         {MSR_HV,        "HV"},
1307 #endif
1308         {MSR_VEC,       "VEC"},
1309         {MSR_VSX,       "VSX"},
1310 #ifdef CONFIG_BOOKE
1311         {MSR_CE,        "CE"},
1312 #endif
1313         {MSR_EE,        "EE"},
1314         {MSR_PR,        "PR"},
1315         {MSR_FP,        "FP"},
1316         {MSR_ME,        "ME"},
1317 #ifdef CONFIG_BOOKE
1318         {MSR_DE,        "DE"},
1319 #else
1320         {MSR_SE,        "SE"},
1321         {MSR_BE,        "BE"},
1322 #endif
1323         {MSR_IR,        "IR"},
1324         {MSR_DR,        "DR"},
1325         {MSR_PMM,       "PMM"},
1326 #ifndef CONFIG_BOOKE
1327         {MSR_RI,        "RI"},
1328         {MSR_LE,        "LE"},
1329 #endif
1330         {0,             NULL}
1331 };
1332
1333 static void print_bits(unsigned long val, struct regbit *bits, const char *sep)
1334 {
1335         const char *s = "";
1336
1337         for (; bits->bit; ++bits)
1338                 if (val & bits->bit) {
1339                         pr_cont("%s%s", s, bits->name);
1340                         s = sep;
1341                 }
1342 }
1343
1344 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1345 static struct regbit msr_tm_bits[] = {
1346         {MSR_TS_T,      "T"},
1347         {MSR_TS_S,      "S"},
1348         {MSR_TM,        "E"},
1349         {0,             NULL}
1350 };
1351
1352 static void print_tm_bits(unsigned long val)
1353 {
1354 /*
1355  * This only prints something if at least one of the TM bit is set.
1356  * Inside the TM[], the output means:
1357  *   E: Enabled         (bit 32)
1358  *   S: Suspended       (bit 33)
1359  *   T: Transactional   (bit 34)
1360  */
1361         if (val & (MSR_TM | MSR_TS_S | MSR_TS_T)) {
1362                 pr_cont(",TM[");
1363                 print_bits(val, msr_tm_bits, "");
1364                 pr_cont("]");
1365         }
1366 }
1367 #else
1368 static void print_tm_bits(unsigned long val) {}
1369 #endif
1370
1371 static void print_msr_bits(unsigned long val)
1372 {
1373         pr_cont("<");
1374         print_bits(val, msr_bits, ",");
1375         print_tm_bits(val);
1376         pr_cont(">");
1377 }
1378
1379 #ifdef CONFIG_PPC64
1380 #define REG             "%016lx"
1381 #define REGS_PER_LINE   4
1382 #define LAST_VOLATILE   13
1383 #else
1384 #define REG             "%08lx"
1385 #define REGS_PER_LINE   8
1386 #define LAST_VOLATILE   12
1387 #endif
1388
1389 void show_regs(struct pt_regs * regs)
1390 {
1391         int i, trap;
1392
1393         show_regs_print_info(KERN_DEFAULT);
1394
1395         printk("NIP: "REG" LR: "REG" CTR: "REG"\n",
1396                regs->nip, regs->link, regs->ctr);
1397         printk("REGS: %p TRAP: %04lx   %s  (%s)\n",
1398                regs, regs->trap, print_tainted(), init_utsname()->release);
1399         printk("MSR: "REG" ", regs->msr);
1400         print_msr_bits(regs->msr);
1401         printk("  CR: %08lx  XER: %08lx\n", regs->ccr, regs->xer);
1402         trap = TRAP(regs);
1403         if ((regs->trap != 0xc00) && cpu_has_feature(CPU_FTR_CFAR))
1404                 pr_cont("CFAR: "REG" ", regs->orig_gpr3);
1405         if (trap == 0x200 || trap == 0x300 || trap == 0x600)
1406 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
1407                 pr_cont("DEAR: "REG" ESR: "REG" ", regs->dar, regs->dsisr);
1408 #else
1409                 pr_cont("DAR: "REG" DSISR: %08lx ", regs->dar, regs->dsisr);
1410 #endif
1411 #ifdef CONFIG_PPC64
1412         pr_cont("SOFTE: %ld ", regs->softe);
1413 #endif
1414 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1415         if (MSR_TM_ACTIVE(regs->msr))
1416                 pr_cont("\nPACATMSCRATCH: %016llx ", get_paca()->tm_scratch);
1417 #endif
1418
1419         for (i = 0;  i < 32;  i++) {
1420                 if ((i % REGS_PER_LINE) == 0)
1421                         pr_cont("\nGPR%02d: ", i);
1422                 pr_cont(REG " ", regs->gpr[i]);
1423                 if (i == LAST_VOLATILE && !FULL_REGS(regs))
1424                         break;
1425         }
1426         pr_cont("\n");
1427 #ifdef CONFIG_KALLSYMS
1428         /*
1429          * Lookup NIP late so we have the best change of getting the
1430          * above info out without failing
1431          */
1432         printk("NIP ["REG"] %pS\n", regs->nip, (void *)regs->nip);
1433         printk("LR ["REG"] %pS\n", regs->link, (void *)regs->link);
1434 #endif
1435         show_stack(current, (unsigned long *) regs->gpr[1]);
1436         if (!user_mode(regs))
1437                 show_instructions(regs);
1438 }
1439
1440 void flush_thread(void)
1441 {
1442 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1443         flush_ptrace_hw_breakpoint(current);
1444 #else /* CONFIG_HAVE_HW_BREAKPOINT */
1445         set_debug_reg_defaults(&current->thread);
1446 #endif /* CONFIG_HAVE_HW_BREAKPOINT */
1447 }
1448
1449 void
1450 release_thread(struct task_struct *t)
1451 {
1452 }
1453
1454 /*
1455  * this gets called so that we can store coprocessor state into memory and
1456  * copy the current task into the new thread.
1457  */
1458 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
1459 {
1460         flush_all_to_thread(src);
1461         /*
1462          * Flush TM state out so we can copy it.  __switch_to_tm() does this
1463          * flush but it removes the checkpointed state from the current CPU and
1464          * transitions the CPU out of TM mode.  Hence we need to call
1465          * tm_recheckpoint_new_task() (on the same task) to restore the
1466          * checkpointed state back and the TM mode.
1467          *
1468          * Can't pass dst because it isn't ready. Doesn't matter, passing
1469          * dst is only important for __switch_to()
1470          */
1471         __switch_to_tm(src, src);
1472
1473         *dst = *src;
1474
1475         clear_task_ebb(dst);
1476
1477         return 0;
1478 }
1479
1480 static void setup_ksp_vsid(struct task_struct *p, unsigned long sp)
1481 {
1482 #ifdef CONFIG_PPC_STD_MMU_64
1483         unsigned long sp_vsid;
1484         unsigned long llp = mmu_psize_defs[mmu_linear_psize].sllp;
1485
1486         if (radix_enabled())
1487                 return;
1488
1489         if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
1490                 sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_1T)
1491                         << SLB_VSID_SHIFT_1T;
1492         else
1493                 sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_256M)
1494                         << SLB_VSID_SHIFT;
1495         sp_vsid |= SLB_VSID_KERNEL | llp;
1496         p->thread.ksp_vsid = sp_vsid;
1497 #endif
1498 }
1499
1500 /*
1501  * Copy a thread..
1502  */
1503
1504 /*
1505  * Copy architecture-specific thread state
1506  */
1507 int copy_thread(unsigned long clone_flags, unsigned long usp,
1508                 unsigned long kthread_arg, struct task_struct *p)
1509 {
1510         struct pt_regs *childregs, *kregs;
1511         extern void ret_from_fork(void);
1512         extern void ret_from_kernel_thread(void);
1513         void (*f)(void);
1514         unsigned long sp = (unsigned long)task_stack_page(p) + THREAD_SIZE;
1515         struct thread_info *ti = task_thread_info(p);
1516
1517         klp_init_thread_info(ti);
1518
1519         /* Copy registers */
1520         sp -= sizeof(struct pt_regs);
1521         childregs = (struct pt_regs *) sp;
1522         if (unlikely(p->flags & PF_KTHREAD)) {
1523                 /* kernel thread */
1524                 memset(childregs, 0, sizeof(struct pt_regs));
1525                 childregs->gpr[1] = sp + sizeof(struct pt_regs);
1526                 /* function */
1527                 if (usp)
1528                         childregs->gpr[14] = ppc_function_entry((void *)usp);
1529 #ifdef CONFIG_PPC64
1530                 clear_tsk_thread_flag(p, TIF_32BIT);
1531                 childregs->softe = 1;
1532 #endif
1533                 childregs->gpr[15] = kthread_arg;
1534                 p->thread.regs = NULL;  /* no user register state */
1535                 ti->flags |= _TIF_RESTOREALL;
1536                 f = ret_from_kernel_thread;
1537         } else {
1538                 /* user thread */
1539                 struct pt_regs *regs = current_pt_regs();
1540                 CHECK_FULL_REGS(regs);
1541                 *childregs = *regs;
1542                 if (usp)
1543                         childregs->gpr[1] = usp;
1544                 p->thread.regs = childregs;
1545                 childregs->gpr[3] = 0;  /* Result from fork() */
1546                 if (clone_flags & CLONE_SETTLS) {
1547 #ifdef CONFIG_PPC64
1548                         if (!is_32bit_task())
1549                                 childregs->gpr[13] = childregs->gpr[6];
1550                         else
1551 #endif
1552                                 childregs->gpr[2] = childregs->gpr[6];
1553                 }
1554
1555                 f = ret_from_fork;
1556         }
1557         childregs->msr &= ~(MSR_FP|MSR_VEC|MSR_VSX);
1558         sp -= STACK_FRAME_OVERHEAD;
1559
1560         /*
1561          * The way this works is that at some point in the future
1562          * some task will call _switch to switch to the new task.
1563          * That will pop off the stack frame created below and start
1564          * the new task running at ret_from_fork.  The new task will
1565          * do some house keeping and then return from the fork or clone
1566          * system call, using the stack frame created above.
1567          */
1568         ((unsigned long *)sp)[0] = 0;
1569         sp -= sizeof(struct pt_regs);
1570         kregs = (struct pt_regs *) sp;
1571         sp -= STACK_FRAME_OVERHEAD;
1572         p->thread.ksp = sp;
1573 #ifdef CONFIG_PPC32
1574         p->thread.ksp_limit = (unsigned long)task_stack_page(p) +
1575                                 _ALIGN_UP(sizeof(struct thread_info), 16);
1576 #endif
1577 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1578         p->thread.ptrace_bps[0] = NULL;
1579 #endif
1580
1581         p->thread.fp_save_area = NULL;
1582 #ifdef CONFIG_ALTIVEC
1583         p->thread.vr_save_area = NULL;
1584 #endif
1585
1586         setup_ksp_vsid(p, sp);
1587
1588 #ifdef CONFIG_PPC64
1589         if (cpu_has_feature(CPU_FTR_DSCR)) {
1590                 p->thread.dscr_inherit = current->thread.dscr_inherit;
1591                 p->thread.dscr = mfspr(SPRN_DSCR);
1592         }
1593         if (cpu_has_feature(CPU_FTR_HAS_PPR))
1594                 p->thread.ppr = INIT_PPR;
1595 #endif
1596         kregs->nip = ppc_function_entry(f);
1597         return 0;
1598 }
1599
1600 /*
1601  * Set up a thread for executing a new program
1602  */
1603 void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
1604 {
1605 #ifdef CONFIG_PPC64
1606         unsigned long load_addr = regs->gpr[2]; /* saved by ELF_PLAT_INIT */
1607 #endif
1608
1609         /*
1610          * If we exec out of a kernel thread then thread.regs will not be
1611          * set.  Do it now.
1612          */
1613         if (!current->thread.regs) {
1614                 struct pt_regs *regs = task_stack_page(current) + THREAD_SIZE;
1615                 current->thread.regs = regs - 1;
1616         }
1617
1618 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1619         /*
1620          * Clear any transactional state, we're exec()ing. The cause is
1621          * not important as there will never be a recheckpoint so it's not
1622          * user visible.
1623          */
1624         if (MSR_TM_SUSPENDED(mfmsr()))
1625                 tm_reclaim_current(0);
1626 #endif
1627
1628         memset(regs->gpr, 0, sizeof(regs->gpr));
1629         regs->ctr = 0;
1630         regs->link = 0;
1631         regs->xer = 0;
1632         regs->ccr = 0;
1633         regs->gpr[1] = sp;
1634
1635         /*
1636          * We have just cleared all the nonvolatile GPRs, so make
1637          * FULL_REGS(regs) return true.  This is necessary to allow
1638          * ptrace to examine the thread immediately after exec.
1639          */
1640         regs->trap &= ~1UL;
1641
1642 #ifdef CONFIG_PPC32
1643         regs->mq = 0;
1644         regs->nip = start;
1645         regs->msr = MSR_USER;
1646 #else
1647         if (!is_32bit_task()) {
1648                 unsigned long entry;
1649
1650                 if (is_elf2_task()) {
1651                         /* Look ma, no function descriptors! */
1652                         entry = start;
1653
1654                         /*
1655                          * Ulrich says:
1656                          *   The latest iteration of the ABI requires that when
1657                          *   calling a function (at its global entry point),
1658                          *   the caller must ensure r12 holds the entry point
1659                          *   address (so that the function can quickly
1660                          *   establish addressability).
1661                          */
1662                         regs->gpr[12] = start;
1663                         /* Make sure that's restored on entry to userspace. */
1664                         set_thread_flag(TIF_RESTOREALL);
1665                 } else {
1666                         unsigned long toc;
1667
1668                         /* start is a relocated pointer to the function
1669                          * descriptor for the elf _start routine.  The first
1670                          * entry in the function descriptor is the entry
1671                          * address of _start and the second entry is the TOC
1672                          * value we need to use.
1673                          */
1674                         __get_user(entry, (unsigned long __user *)start);
1675                         __get_user(toc, (unsigned long __user *)start+1);
1676
1677                         /* Check whether the e_entry function descriptor entries
1678                          * need to be relocated before we can use them.
1679                          */
1680                         if (load_addr != 0) {
1681                                 entry += load_addr;
1682                                 toc   += load_addr;
1683                         }
1684                         regs->gpr[2] = toc;
1685                 }
1686                 regs->nip = entry;
1687                 regs->msr = MSR_USER64;
1688         } else {
1689                 regs->nip = start;
1690                 regs->gpr[2] = 0;
1691                 regs->msr = MSR_USER32;
1692         }
1693 #endif
1694 #ifdef CONFIG_VSX
1695         current->thread.used_vsr = 0;
1696 #endif
1697         current->thread.load_fp = 0;
1698         memset(&current->thread.fp_state, 0, sizeof(current->thread.fp_state));
1699         current->thread.fp_save_area = NULL;
1700 #ifdef CONFIG_ALTIVEC
1701         memset(&current->thread.vr_state, 0, sizeof(current->thread.vr_state));
1702         current->thread.vr_state.vscr.u[3] = 0x00010000; /* Java mode disabled */
1703         current->thread.vr_save_area = NULL;
1704         current->thread.vrsave = 0;
1705         current->thread.used_vr = 0;
1706         current->thread.load_vec = 0;
1707 #endif /* CONFIG_ALTIVEC */
1708 #ifdef CONFIG_SPE
1709         memset(current->thread.evr, 0, sizeof(current->thread.evr));
1710         current->thread.acc = 0;
1711         current->thread.spefscr = 0;
1712         current->thread.used_spe = 0;
1713 #endif /* CONFIG_SPE */
1714 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1715         current->thread.tm_tfhar = 0;
1716         current->thread.tm_texasr = 0;
1717         current->thread.tm_tfiar = 0;
1718         current->thread.load_tm = 0;
1719 #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
1720 }
1721 EXPORT_SYMBOL(start_thread);
1722
1723 #define PR_FP_ALL_EXCEPT (PR_FP_EXC_DIV | PR_FP_EXC_OVF | PR_FP_EXC_UND \
1724                 | PR_FP_EXC_RES | PR_FP_EXC_INV)
1725
1726 int set_fpexc_mode(struct task_struct *tsk, unsigned int val)
1727 {
1728         struct pt_regs *regs = tsk->thread.regs;
1729
1730         /* This is a bit hairy.  If we are an SPE enabled  processor
1731          * (have embedded fp) we store the IEEE exception enable flags in
1732          * fpexc_mode.  fpexc_mode is also used for setting FP exception
1733          * mode (asyn, precise, disabled) for 'Classic' FP. */
1734         if (val & PR_FP_EXC_SW_ENABLE) {
1735 #ifdef CONFIG_SPE
1736                 if (cpu_has_feature(CPU_FTR_SPE)) {
1737                         /*
1738                          * When the sticky exception bits are set
1739                          * directly by userspace, it must call prctl
1740                          * with PR_GET_FPEXC (with PR_FP_EXC_SW_ENABLE
1741                          * in the existing prctl settings) or
1742                          * PR_SET_FPEXC (with PR_FP_EXC_SW_ENABLE in
1743                          * the bits being set).  <fenv.h> functions
1744                          * saving and restoring the whole
1745                          * floating-point environment need to do so
1746                          * anyway to restore the prctl settings from
1747                          * the saved environment.
1748                          */
1749                         tsk->thread.spefscr_last = mfspr(SPRN_SPEFSCR);
1750                         tsk->thread.fpexc_mode = val &
1751                                 (PR_FP_EXC_SW_ENABLE | PR_FP_ALL_EXCEPT);
1752                         return 0;
1753                 } else {
1754                         return -EINVAL;
1755                 }
1756 #else
1757                 return -EINVAL;
1758 #endif
1759         }
1760
1761         /* on a CONFIG_SPE this does not hurt us.  The bits that
1762          * __pack_fe01 use do not overlap with bits used for
1763          * PR_FP_EXC_SW_ENABLE.  Additionally, the MSR[FE0,FE1] bits
1764          * on CONFIG_SPE implementations are reserved so writing to
1765          * them does not change anything */
1766         if (val > PR_FP_EXC_PRECISE)
1767                 return -EINVAL;
1768         tsk->thread.fpexc_mode = __pack_fe01(val);
1769         if (regs != NULL && (regs->msr & MSR_FP) != 0)
1770                 regs->msr = (regs->msr & ~(MSR_FE0|MSR_FE1))
1771                         | tsk->thread.fpexc_mode;
1772         return 0;
1773 }
1774
1775 int get_fpexc_mode(struct task_struct *tsk, unsigned long adr)
1776 {
1777         unsigned int val;
1778
1779         if (tsk->thread.fpexc_mode & PR_FP_EXC_SW_ENABLE)
1780 #ifdef CONFIG_SPE
1781                 if (cpu_has_feature(CPU_FTR_SPE)) {
1782                         /*
1783                          * When the sticky exception bits are set
1784                          * directly by userspace, it must call prctl
1785                          * with PR_GET_FPEXC (with PR_FP_EXC_SW_ENABLE
1786                          * in the existing prctl settings) or
1787                          * PR_SET_FPEXC (with PR_FP_EXC_SW_ENABLE in
1788                          * the bits being set).  <fenv.h> functions
1789                          * saving and restoring the whole
1790                          * floating-point environment need to do so
1791                          * anyway to restore the prctl settings from
1792                          * the saved environment.
1793                          */
1794                         tsk->thread.spefscr_last = mfspr(SPRN_SPEFSCR);
1795                         val = tsk->thread.fpexc_mode;
1796                 } else
1797                         return -EINVAL;
1798 #else
1799                 return -EINVAL;
1800 #endif
1801         else
1802                 val = __unpack_fe01(tsk->thread.fpexc_mode);
1803         return put_user(val, (unsigned int __user *) adr);
1804 }
1805
1806 int set_endian(struct task_struct *tsk, unsigned int val)
1807 {
1808         struct pt_regs *regs = tsk->thread.regs;
1809
1810         if ((val == PR_ENDIAN_LITTLE && !cpu_has_feature(CPU_FTR_REAL_LE)) ||
1811             (val == PR_ENDIAN_PPC_LITTLE && !cpu_has_feature(CPU_FTR_PPC_LE)))
1812                 return -EINVAL;
1813
1814         if (regs == NULL)
1815                 return -EINVAL;
1816
1817         if (val == PR_ENDIAN_BIG)
1818                 regs->msr &= ~MSR_LE;
1819         else if (val == PR_ENDIAN_LITTLE || val == PR_ENDIAN_PPC_LITTLE)
1820                 regs->msr |= MSR_LE;
1821         else
1822                 return -EINVAL;
1823
1824         return 0;
1825 }
1826
1827 int get_endian(struct task_struct *tsk, unsigned long adr)
1828 {
1829         struct pt_regs *regs = tsk->thread.regs;
1830         unsigned int val;
1831
1832         if (!cpu_has_feature(CPU_FTR_PPC_LE) &&
1833             !cpu_has_feature(CPU_FTR_REAL_LE))
1834                 return -EINVAL;
1835
1836         if (regs == NULL)
1837                 return -EINVAL;
1838
1839         if (regs->msr & MSR_LE) {
1840                 if (cpu_has_feature(CPU_FTR_REAL_LE))
1841                         val = PR_ENDIAN_LITTLE;
1842                 else
1843                         val = PR_ENDIAN_PPC_LITTLE;
1844         } else
1845                 val = PR_ENDIAN_BIG;
1846
1847         return put_user(val, (unsigned int __user *)adr);
1848 }
1849
1850 int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
1851 {
1852         tsk->thread.align_ctl = val;
1853         return 0;
1854 }
1855
1856 int get_unalign_ctl(struct task_struct *tsk, unsigned long adr)
1857 {
1858         return put_user(tsk->thread.align_ctl, (unsigned int __user *)adr);
1859 }
1860
1861 static inline int valid_irq_stack(unsigned long sp, struct task_struct *p,
1862                                   unsigned long nbytes)
1863 {
1864         unsigned long stack_page;
1865         unsigned long cpu = task_cpu(p);
1866
1867         /*
1868          * Avoid crashing if the stack has overflowed and corrupted
1869          * task_cpu(p), which is in the thread_info struct.
1870          */
1871         if (cpu < NR_CPUS && cpu_possible(cpu)) {
1872                 stack_page = (unsigned long) hardirq_ctx[cpu];
1873                 if (sp >= stack_page + sizeof(struct thread_struct)
1874                     && sp <= stack_page + THREAD_SIZE - nbytes)
1875                         return 1;
1876
1877                 stack_page = (unsigned long) softirq_ctx[cpu];
1878                 if (sp >= stack_page + sizeof(struct thread_struct)
1879                     && sp <= stack_page + THREAD_SIZE - nbytes)
1880                         return 1;
1881         }
1882         return 0;
1883 }
1884
1885 int validate_sp(unsigned long sp, struct task_struct *p,
1886                        unsigned long nbytes)
1887 {
1888         unsigned long stack_page = (unsigned long)task_stack_page(p);
1889
1890         if (sp >= stack_page + sizeof(struct thread_struct)
1891             && sp <= stack_page + THREAD_SIZE - nbytes)
1892                 return 1;
1893
1894         return valid_irq_stack(sp, p, nbytes);
1895 }
1896
1897 EXPORT_SYMBOL(validate_sp);
1898
1899 unsigned long get_wchan(struct task_struct *p)
1900 {
1901         unsigned long ip, sp;
1902         int count = 0;
1903
1904         if (!p || p == current || p->state == TASK_RUNNING)
1905                 return 0;
1906
1907         sp = p->thread.ksp;
1908         if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD))
1909                 return 0;
1910
1911         do {
1912                 sp = *(unsigned long *)sp;
1913                 if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD))
1914                         return 0;
1915                 if (count > 0) {
1916                         ip = ((unsigned long *)sp)[STACK_FRAME_LR_SAVE];
1917                         if (!in_sched_functions(ip))
1918                                 return ip;
1919                 }
1920         } while (count++ < 16);
1921         return 0;
1922 }
1923
1924 static int kstack_depth_to_print = CONFIG_PRINT_STACK_DEPTH;
1925
1926 void show_stack(struct task_struct *tsk, unsigned long *stack)
1927 {
1928         unsigned long sp, ip, lr, newsp;
1929         int count = 0;
1930         int firstframe = 1;
1931 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1932         int curr_frame = current->curr_ret_stack;
1933         extern void return_to_handler(void);
1934         unsigned long rth = (unsigned long)return_to_handler;
1935 #endif
1936
1937         sp = (unsigned long) stack;
1938         if (tsk == NULL)
1939                 tsk = current;
1940         if (sp == 0) {
1941                 if (tsk == current)
1942                         sp = current_stack_pointer();
1943                 else
1944                         sp = tsk->thread.ksp;
1945         }
1946
1947         lr = 0;
1948         printk("Call Trace:\n");
1949         do {
1950                 if (!validate_sp(sp, tsk, STACK_FRAME_OVERHEAD))
1951                         return;
1952
1953                 stack = (unsigned long *) sp;
1954                 newsp = stack[0];
1955                 ip = stack[STACK_FRAME_LR_SAVE];
1956                 if (!firstframe || ip != lr) {
1957                         printk("["REG"] ["REG"] %pS", sp, ip, (void *)ip);
1958 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1959                         if ((ip == rth) && curr_frame >= 0) {
1960                                 pr_cont(" (%pS)",
1961                                        (void *)current->ret_stack[curr_frame].ret);
1962                                 curr_frame--;
1963                         }
1964 #endif
1965                         if (firstframe)
1966                                 pr_cont(" (unreliable)");
1967                         pr_cont("\n");
1968                 }
1969                 firstframe = 0;
1970
1971                 /*
1972                  * See if this is an exception frame.
1973                  * We look for the "regshere" marker in the current frame.
1974                  */
1975                 if (validate_sp(sp, tsk, STACK_INT_FRAME_SIZE)
1976                     && stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
1977                         struct pt_regs *regs = (struct pt_regs *)
1978                                 (sp + STACK_FRAME_OVERHEAD);
1979                         lr = regs->link;
1980                         printk("--- interrupt: %lx at %pS\n    LR = %pS\n",
1981                                regs->trap, (void *)regs->nip, (void *)lr);
1982                         firstframe = 1;
1983                 }
1984
1985                 sp = newsp;
1986         } while (count++ < kstack_depth_to_print);
1987 }
1988
1989 #ifdef CONFIG_PPC64
1990 /* Called with hard IRQs off */
1991 void notrace __ppc64_runlatch_on(void)
1992 {
1993         struct thread_info *ti = current_thread_info();
1994         unsigned long ctrl;
1995
1996         ctrl = mfspr(SPRN_CTRLF);
1997         ctrl |= CTRL_RUNLATCH;
1998         mtspr(SPRN_CTRLT, ctrl);
1999
2000         ti->local_flags |= _TLF_RUNLATCH;
2001 }
2002
2003 /* Called with hard IRQs off */
2004 void notrace __ppc64_runlatch_off(void)
2005 {
2006         struct thread_info *ti = current_thread_info();
2007         unsigned long ctrl;
2008
2009         ti->local_flags &= ~_TLF_RUNLATCH;
2010
2011         ctrl = mfspr(SPRN_CTRLF);
2012         ctrl &= ~CTRL_RUNLATCH;
2013         mtspr(SPRN_CTRLT, ctrl);
2014 }
2015 #endif /* CONFIG_PPC64 */
2016
2017 unsigned long arch_align_stack(unsigned long sp)
2018 {
2019         if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
2020                 sp -= get_random_int() & ~PAGE_MASK;
2021         return sp & ~0xf;
2022 }
2023
2024 static inline unsigned long brk_rnd(void)
2025 {
2026         unsigned long rnd = 0;
2027
2028         /* 8MB for 32bit, 1GB for 64bit */
2029         if (is_32bit_task())
2030                 rnd = (get_random_long() % (1UL<<(23-PAGE_SHIFT)));
2031         else
2032                 rnd = (get_random_long() % (1UL<<(30-PAGE_SHIFT)));
2033
2034         return rnd << PAGE_SHIFT;
2035 }
2036
2037 unsigned long arch_randomize_brk(struct mm_struct *mm)
2038 {
2039         unsigned long base = mm->brk;
2040         unsigned long ret;
2041
2042 #ifdef CONFIG_PPC_STD_MMU_64
2043         /*
2044          * If we are using 1TB segments and we are allowed to randomise
2045          * the heap, we can put it above 1TB so it is backed by a 1TB
2046          * segment. Otherwise the heap will be in the bottom 1TB
2047          * which always uses 256MB segments and this may result in a
2048          * performance penalty. We don't need to worry about radix. For
2049          * radix, mmu_highuser_ssize remains unchanged from 256MB.
2050          */
2051         if (!is_32bit_task() && (mmu_highuser_ssize == MMU_SEGSIZE_1T))
2052                 base = max_t(unsigned long, mm->brk, 1UL << SID_SHIFT_1T);
2053 #endif
2054
2055         ret = PAGE_ALIGN(base + brk_rnd());
2056
2057         if (ret < mm->brk)
2058                 return mm->brk;
2059
2060         return ret;
2061 }
2062