arch/arm64/kernel/ptrace.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Based on arch/arm/kernel/ptrace.c
   4  *
   5  * By Ross Biro 1/23/92
   6  * edited by Linus Torvalds
   7  * ARM modifications Copyright (C) 2000 Russell King
   8  * Copyright (C) 2012 ARM Ltd.
   9  */
  10
  11 #include <linux/audit.h>
  12 #include <linux/compat.h>
  13 #include <linux/kernel.h>
  14 #include <linux/sched/signal.h>
  15 #include <linux/sched/task_stack.h>
  16 #include <linux/mm.h>
  17 #include <linux/nospec.h>
  18 #include <linux/smp.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/user.h>
  21 #include <linux/seccomp.h>
  22 #include <linux/security.h>
  23 #include <linux/init.h>
  24 #include <linux/signal.h>
  25 #include <linux/string.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/perf_event.h>
  28 #include <linux/hw_breakpoint.h>
  29 #include <linux/regset.h>
  30 #include <linux/tracehook.h>
  31 #include <linux/elf.h>
  32
  33 #include <asm/compat.h>
  34 #include <asm/cpufeature.h>
  35 #include <asm/debug-monitors.h>
  36 #include <asm/fpsimd.h>
  37 #include <asm/mte.h>
  38 #include <asm/pointer_auth.h>
  39 #include <asm/stacktrace.h>
  40 #include <asm/syscall.h>
  41 #include <asm/traps.h>
  42 #include <asm/system_misc.h>
  43
  44 #define CREATE_TRACE_POINTS
  45 #include <trace/events/syscalls.h>
  46
  47 struct pt_regs_offset {
  48         const char *name;
  49         int offset;
  50 };
  51
  52 #define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)}
  53 #define REG_OFFSET_END {.name = NULL, .offset = 0}
  54 #define GPR_OFFSET_NAME(r) \
  55         {.name = "x" #r, .offset = offsetof(struct pt_regs, regs[r])}
  56
  57 static const struct pt_regs_offset regoffset_table[] = {
  58         GPR_OFFSET_NAME(0),
  59         GPR_OFFSET_NAME(1),
  60         GPR_OFFSET_NAME(2),
  61         GPR_OFFSET_NAME(3),
  62         GPR_OFFSET_NAME(4),
  63         GPR_OFFSET_NAME(5),
  64         GPR_OFFSET_NAME(6),
  65         GPR_OFFSET_NAME(7),
  66         GPR_OFFSET_NAME(8),
  67         GPR_OFFSET_NAME(9),
  68         GPR_OFFSET_NAME(10),
  69         GPR_OFFSET_NAME(11),
  70         GPR_OFFSET_NAME(12),
  71         GPR_OFFSET_NAME(13),
  72         GPR_OFFSET_NAME(14),
  73         GPR_OFFSET_NAME(15),
  74         GPR_OFFSET_NAME(16),
  75         GPR_OFFSET_NAME(17),
  76         GPR_OFFSET_NAME(18),
  77         GPR_OFFSET_NAME(19),
  78         GPR_OFFSET_NAME(20),
  79         GPR_OFFSET_NAME(21),
  80         GPR_OFFSET_NAME(22),
  81         GPR_OFFSET_NAME(23),
  82         GPR_OFFSET_NAME(24),
  83         GPR_OFFSET_NAME(25),
  84         GPR_OFFSET_NAME(26),
  85         GPR_OFFSET_NAME(27),
  86         GPR_OFFSET_NAME(28),
  87         GPR_OFFSET_NAME(29),
  88         GPR_OFFSET_NAME(30),
  89         {.name = "lr", .offset = offsetof(struct pt_regs, regs[30])},
  90         REG_OFFSET_NAME(sp),
  91         REG_OFFSET_NAME(pc),
  92         REG_OFFSET_NAME(pstate),
  93         REG_OFFSET_END,
  94 };
  95
  96 /**
  97  * regs_query_register_offset() - query register offset from its name
  98  * @name:       the name of a register
  99  *
 100  * regs_query_register_offset() returns the offset of a register in struct
 101  * pt_regs from its name. If the name is invalid, this returns -EINVAL;
 102  */
 103 int regs_query_register_offset(const char *name)
 104 {
 105         const struct pt_regs_offset *roff;
 106
 107         for (roff = regoffset_table; roff->name != NULL; roff++)
 108                 if (!strcmp(roff->name, name))
 109                         return roff->offset;
 110         return -EINVAL;
 111 }
 112
 113 /**
 114  * regs_within_kernel_stack() - check the address in the stack
 115  * @regs:      pt_regs which contains kernel stack pointer.
 116  * @addr:      address which is checked.
 117  *
 118  * regs_within_kernel_stack() checks @addr is within the kernel stack page(s).
 119  * If @addr is within the kernel stack, it returns true. If not, returns false.
 120  */
 121 static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr)
 122 {
 123         return ((addr & ~(THREAD_SIZE - 1))  ==
 124                 (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) ||
 125                 on_irq_stack(addr, sizeof(unsigned long), NULL);
 126 }
 127
 128 /**
 129  * regs_get_kernel_stack_nth() - get Nth entry of the stack
 130  * @regs:       pt_regs which contains kernel stack pointer.
 131  * @n:          stack entry number.
 132  *
 133  * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which
 134  * is specified by @regs. If the @n th entry is NOT in the kernel stack,
 135  * this returns 0.
 136  */
 137 unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n)
 138 {
 139         unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs);
 140
 141         addr += n;
 142         if (regs_within_kernel_stack(regs, (unsigned long)addr))
 143                 return *addr;
 144         else
 145                 return 0;
 146 }
 147
 148 /*
 149  * TODO: does not yet catch signals sent when the child dies.
 150  * in exit.c or in signal.c.
 151  */
 152
 153 /*
 154  * Called by kernel/ptrace.c when detaching..
 155  */
 156 void ptrace_disable(struct task_struct *child)
 157 {
 158         /*
 159          * This would be better off in core code, but PTRACE_DETACH has
 160          * grown its fair share of arch-specific worts and changing it
 161          * is likely to cause regressions on obscure architectures.
 162          */
 163         user_disable_single_step(child);
 164 }
 165
 166 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 167 /*
 168  * Handle hitting a HW-breakpoint.
 169  */
 170 static void ptrace_hbptriggered(struct perf_event *bp,
 171                                 struct perf_sample_data *data,
 172                                 struct pt_regs *regs)
 173 {
 174         struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
 175         const char *desc = "Hardware breakpoint trap (ptrace)";
 176
 177 #ifdef CONFIG_COMPAT
 178         if (is_compat_task()) {
 179                 int si_errno = 0;
 180                 int i;
 181
 182                 for (i = 0; i < ARM_MAX_BRP; ++i) {
 183                         if (current->thread.debug.hbp_break[i] == bp) {
 184                                 si_errno = (i << 1) + 1;
 185                                 break;
 186                         }
 187                 }
 188
 189                 for (i = 0; i < ARM_MAX_WRP; ++i) {
 190                         if (current->thread.debug.hbp_watch[i] == bp) {
 191                                 si_errno = -((i << 1) + 1);
 192                                 break;
 193                         }
 194                 }
 195                 arm64_force_sig_ptrace_errno_trap(si_errno, bkpt->trigger,
 196                                                   desc);
 197                 return;
 198         }
 199 #endif
 200         arm64_force_sig_fault(SIGTRAP, TRAP_HWBKPT, bkpt->trigger, desc);
 201 }
 202
 203 /*
 204  * Unregister breakpoints from this task and reset the pointers in
 205  * the thread_struct.
 206  */
 207 void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
 208 {
 209         int i;
 210         struct thread_struct *t = &tsk->thread;
 211
 212         for (i = 0; i < ARM_MAX_BRP; i++) {
 213                 if (t->debug.hbp_break[i]) {
 214                         unregister_hw_breakpoint(t->debug.hbp_break[i]);
 215                         t->debug.hbp_break[i] = NULL;
 216                 }
 217         }
 218
 219         for (i = 0; i < ARM_MAX_WRP; i++) {
 220                 if (t->debug.hbp_watch[i]) {
 221                         unregister_hw_breakpoint(t->debug.hbp_watch[i]);
 222                         t->debug.hbp_watch[i] = NULL;
 223                 }
 224         }
 225 }
 226
 227 void ptrace_hw_copy_thread(struct task_struct *tsk)
 228 {
 229         memset(&tsk->thread.debug, 0, sizeof(struct debug_info));
 230 }
 231
 232 static struct perf_event *ptrace_hbp_get_event(unsigned int note_type,
 233                                                struct task_struct *tsk,
 234                                                unsigned long idx)
 235 {
 236         struct perf_event *bp = ERR_PTR(-EINVAL);
 237
 238         switch (note_type) {
 239         case NT_ARM_HW_BREAK:
 240                 if (idx >= ARM_MAX_BRP)
 241                         goto out;
 242                 idx = array_index_nospec(idx, ARM_MAX_BRP);
 243                 bp = tsk->thread.debug.hbp_break[idx];
 244                 break;
 245         case NT_ARM_HW_WATCH:
 246                 if (idx >= ARM_MAX_WRP)
 247                         goto out;
 248                 idx = array_index_nospec(idx, ARM_MAX_WRP);
 249                 bp = tsk->thread.debug.hbp_watch[idx];
 250                 break;
 251         }
 252
 253 out:
 254         return bp;
 255 }
 256
 257 static int ptrace_hbp_set_event(unsigned int note_type,
 258                                 struct task_struct *tsk,
 259                                 unsigned long idx,
 260                                 struct perf_event *bp)
 261 {
 262         int err = -EINVAL;
 263
 264         switch (note_type) {
 265         case NT_ARM_HW_BREAK:
 266                 if (idx >= ARM_MAX_BRP)
 267                         goto out;
 268                 idx = array_index_nospec(idx, ARM_MAX_BRP);
 269                 tsk->thread.debug.hbp_break[idx] = bp;
 270                 err = 0;
 271                 break;
 272         case NT_ARM_HW_WATCH:
 273                 if (idx >= ARM_MAX_WRP)
 274                         goto out;
 275                 idx = array_index_nospec(idx, ARM_MAX_WRP);
 276                 tsk->thread.debug.hbp_watch[idx] = bp;
 277                 err = 0;
 278                 break;
 279         }
 280
 281 out:
 282         return err;
 283 }
 284
 285 static struct perf_event *ptrace_hbp_create(unsigned int note_type,
 286                                             struct task_struct *tsk,
 287                                             unsigned long idx)
 288 {
 289         struct perf_event *bp;
 290         struct perf_event_attr attr;
 291         int err, type;
 292
 293         switch (note_type) {
 294         case NT_ARM_HW_BREAK:
 295                 type = HW_BREAKPOINT_X;
 296                 break;
 297         case NT_ARM_HW_WATCH:
 298                 type = HW_BREAKPOINT_RW;
 299                 break;
 300         default:
 301                 return ERR_PTR(-EINVAL);
 302         }
 303
 304         ptrace_breakpoint_init(&attr);
 305
 306         /*
 307          * Initialise fields to sane defaults
 308          * (i.e. values that will pass validation).
 309          */
 310         attr.bp_addr    = 0;
 311         attr.bp_len     = HW_BREAKPOINT_LEN_4;
 312         attr.bp_type    = type;
 313         attr.disabled   = 1;
 314
 315         bp = register_user_hw_breakpoint(&attr, ptrace_hbptriggered, NULL, tsk);
 316         if (IS_ERR(bp))
 317                 return bp;
 318
 319         err = ptrace_hbp_set_event(note_type, tsk, idx, bp);
 320         if (err)
 321                 return ERR_PTR(err);
 322
 323         return bp;
 324 }
 325
 326 static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type,
 327                                      struct arch_hw_breakpoint_ctrl ctrl,
 328                                      struct perf_event_attr *attr)
 329 {
 330         int err, len, type, offset, disabled = !ctrl.enabled;
 331
 332         attr->disabled = disabled;
 333         if (disabled)
 334                 return 0;
 335
 336         err = arch_bp_generic_fields(ctrl, &len, &type, &offset);
 337         if (err)
 338                 return err;
 339
 340         switch (note_type) {
 341         case NT_ARM_HW_BREAK:
 342                 if ((type & HW_BREAKPOINT_X) != type)
 343                         return -EINVAL;
 344                 break;
 345         case NT_ARM_HW_WATCH:
 346                 if ((type & HW_BREAKPOINT_RW) != type)
 347                         return -EINVAL;
 348                 break;
 349         default:
 350                 return -EINVAL;
 351         }
 352
 353         attr->bp_len    = len;
 354         attr->bp_type   = type;
 355         attr->bp_addr   += offset;
 356
 357         return 0;
 358 }
 359
 360 static int ptrace_hbp_get_resource_info(unsigned int note_type, u32 *info)
 361 {
 362         u8 num;
 363         u32 reg = 0;
 364
 365         switch (note_type) {
 366         case NT_ARM_HW_BREAK:
 367                 num = hw_breakpoint_slots(TYPE_INST);
 368                 break;
 369         case NT_ARM_HW_WATCH:
 370                 num = hw_breakpoint_slots(TYPE_DATA);
 371                 break;
 372         default:
 373                 return -EINVAL;
 374         }
 375
 376         reg |= debug_monitors_arch();
 377         reg <<= 8;
 378         reg |= num;
 379
 380         *info = reg;
 381         return 0;
 382 }
 383
 384 static int ptrace_hbp_get_ctrl(unsigned int note_type,
 385                                struct task_struct *tsk,
 386                                unsigned long idx,
 387                                u32 *ctrl)
 388 {
 389         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 390
 391         if (IS_ERR(bp))
 392                 return PTR_ERR(bp);
 393
 394         *ctrl = bp ? encode_ctrl_reg(counter_arch_bp(bp)->ctrl) : 0;
 395         return 0;
 396 }
 397
 398 static int ptrace_hbp_get_addr(unsigned int note_type,
 399                                struct task_struct *tsk,
 400                                unsigned long idx,
 401                                u64 *addr)
 402 {
 403         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 404
 405         if (IS_ERR(bp))
 406                 return PTR_ERR(bp);
 407
 408         *addr = bp ? counter_arch_bp(bp)->address : 0;
 409         return 0;
 410 }
 411
 412 static struct perf_event *ptrace_hbp_get_initialised_bp(unsigned int note_type,
 413                                                         struct task_struct *tsk,
 414                                                         unsigned long idx)
 415 {
 416         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 417
 418         if (!bp)
 419                 bp = ptrace_hbp_create(note_type, tsk, idx);
 420
 421         return bp;
 422 }
 423
 424 static int ptrace_hbp_set_ctrl(unsigned int note_type,
 425                                struct task_struct *tsk,
 426                                unsigned long idx,
 427                                u32 uctrl)
 428 {
 429         int err;
 430         struct perf_event *bp;
 431         struct perf_event_attr attr;
 432         struct arch_hw_breakpoint_ctrl ctrl;
 433
 434         bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx);
 435         if (IS_ERR(bp)) {
 436                 err = PTR_ERR(bp);
 437                 return err;
 438         }
 439
 440         attr = bp->attr;
 441         decode_ctrl_reg(uctrl, &ctrl);
 442         err = ptrace_hbp_fill_attr_ctrl(note_type, ctrl, &attr);
 443         if (err)
 444                 return err;
 445
 446         return modify_user_hw_breakpoint(bp, &attr);
 447 }
 448
 449 static int ptrace_hbp_set_addr(unsigned int note_type,
 450                                struct task_struct *tsk,
 451                                unsigned long idx,
 452                                u64 addr)
 453 {
 454         int err;
 455         struct perf_event *bp;
 456         struct perf_event_attr attr;
 457
 458         bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx);
 459         if (IS_ERR(bp)) {
 460                 err = PTR_ERR(bp);
 461                 return err;
 462         }
 463
 464         attr = bp->attr;
 465         attr.bp_addr = addr;
 466         err = modify_user_hw_breakpoint(bp, &attr);
 467         return err;
 468 }
 469
 470 #define PTRACE_HBP_ADDR_SZ      sizeof(u64)
 471 #define PTRACE_HBP_CTRL_SZ      sizeof(u32)
 472 #define PTRACE_HBP_PAD_SZ       sizeof(u32)
 473
 474 static int hw_break_get(struct task_struct *target,
 475                         const struct user_regset *regset,
 476                         struct membuf to)
 477 {
 478         unsigned int note_type = regset->core_note_type;
 479         int ret, idx = 0;
 480         u32 info, ctrl;
 481         u64 addr;
 482
 483         /* Resource info */
 484         ret = ptrace_hbp_get_resource_info(note_type, &info);
 485         if (ret)
 486                 return ret;
 487
 488         membuf_write(&to, &info, sizeof(info));
 489         membuf_zero(&to, sizeof(u32));
 490         /* (address, ctrl) registers */
 491         while (to.left) {
 492                 ret = ptrace_hbp_get_addr(note_type, target, idx, &addr);
 493                 if (ret)
 494                         return ret;
 495                 ret = ptrace_hbp_get_ctrl(note_type, target, idx, &ctrl);
 496                 if (ret)
 497                         return ret;
 498                 membuf_store(&to, addr);
 499                 membuf_store(&to, ctrl);
 500                 membuf_zero(&to, sizeof(u32));
 501                 idx++;
 502         }
 503         return 0;
 504 }
 505
 506 static int hw_break_set(struct task_struct *target,
 507                         const struct user_regset *regset,
 508                         unsigned int pos, unsigned int count,
 509                         const void *kbuf, const void __user *ubuf)
 510 {
 511         unsigned int note_type = regset->core_note_type;
 512         int ret, idx = 0, offset, limit;
 513         u32 ctrl;
 514         u64 addr;
 515
 516         /* Resource info and pad */
 517         offset = offsetof(struct user_hwdebug_state, dbg_regs);
 518         ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 0, offset);
 519         if (ret)
 520                 return ret;
 521
 522         /* (address, ctrl) registers */
 523         limit = regset->n * regset->size;
 524         while (count && offset < limit) {
 525                 if (count < PTRACE_HBP_ADDR_SZ)
 526                         return -EINVAL;
 527                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &addr,
 528                                          offset, offset + PTRACE_HBP_ADDR_SZ);
 529                 if (ret)
 530                         return ret;
 531                 ret = ptrace_hbp_set_addr(note_type, target, idx, addr);
 532                 if (ret)
 533                         return ret;
 534                 offset += PTRACE_HBP_ADDR_SZ;
 535
 536                 if (!count)
 537                         break;
 538                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl,
 539                                          offset, offset + PTRACE_HBP_CTRL_SZ);
 540                 if (ret)
 541                         return ret;
 542                 ret = ptrace_hbp_set_ctrl(note_type, target, idx, ctrl);
 543                 if (ret)
 544                         return ret;
 545                 offset += PTRACE_HBP_CTRL_SZ;
 546
 547                 ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
 548                                                 offset,
 549                                                 offset + PTRACE_HBP_PAD_SZ);
 550                 if (ret)
 551                         return ret;
 552                 offset += PTRACE_HBP_PAD_SZ;
 553                 idx++;
 554         }
 555
 556         return 0;
 557 }
 558 #endif  /* CONFIG_HAVE_HW_BREAKPOINT */
 559
 560 static int gpr_get(struct task_struct *target,
 561                    const struct user_regset *regset,
 562                    struct membuf to)
 563 {
 564         struct user_pt_regs *uregs = &task_pt_regs(target)->user_regs;
 565         return membuf_write(&to, uregs, sizeof(*uregs));
 566 }
 567
 568 static int gpr_set(struct task_struct *target, const struct user_regset *regset,
 569                    unsigned int pos, unsigned int count,
 570                    const void *kbuf, const void __user *ubuf)
 571 {
 572         int ret;
 573         struct user_pt_regs newregs = task_pt_regs(target)->user_regs;
 574
 575         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, 0, -1);
 576         if (ret)
 577                 return ret;
 578
 579         if (!valid_user_regs(&newregs, target))
 580                 return -EINVAL;
 581
 582         task_pt_regs(target)->user_regs = newregs;
 583         return 0;
 584 }
 585
 586 static int fpr_active(struct task_struct *target, const struct user_regset *regset)
 587 {
 588         if (!system_supports_fpsimd())
 589                 return -ENODEV;
 590         return regset->n;
 591 }
 592
 593 /*
 594  * TODO: update fp accessors for lazy context switching (sync/flush hwstate)
 595  */
 596 static int __fpr_get(struct task_struct *target,
 597                      const struct user_regset *regset,
 598                      struct membuf to)
 599 {
 600         struct user_fpsimd_state *uregs;
 601
 602         sve_sync_to_fpsimd(target);
 603
 604         uregs = &target->thread.uw.fpsimd_state;
 605
 606         return membuf_write(&to, uregs, sizeof(*uregs));
 607 }
 608
 609 static int fpr_get(struct task_struct *target, const struct user_regset *regset,
 610                    struct membuf to)
 611 {
 612         if (!system_supports_fpsimd())
 613                 return -EINVAL;
 614
 615         if (target == current)
 616                 fpsimd_preserve_current_state();
 617
 618         return __fpr_get(target, regset, to);
 619 }
 620
 621 static int __fpr_set(struct task_struct *target,
 622                      const struct user_regset *regset,
 623                      unsigned int pos, unsigned int count,
 624                      const void *kbuf, const void __user *ubuf,
 625                      unsigned int start_pos)
 626 {
 627         int ret;
 628         struct user_fpsimd_state newstate;
 629
 630         /*
 631          * Ensure target->thread.uw.fpsimd_state is up to date, so that a
 632          * short copyin can't resurrect stale data.
 633          */
 634         sve_sync_to_fpsimd(target);
 635
 636         newstate = target->thread.uw.fpsimd_state;
 637
 638         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate,
 639                                  start_pos, start_pos + sizeof(newstate));
 640         if (ret)
 641                 return ret;
 642
 643         target->thread.uw.fpsimd_state = newstate;
 644
 645         return ret;
 646 }
 647
 648 static int fpr_set(struct task_struct *target, const struct user_regset *regset,
 649                    unsigned int pos, unsigned int count,
 650                    const void *kbuf, const void __user *ubuf)
 651 {
 652         int ret;
 653
 654         if (!system_supports_fpsimd())
 655                 return -EINVAL;
 656
 657         ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, 0);
 658         if (ret)
 659                 return ret;
 660
 661         sve_sync_from_fpsimd_zeropad(target);
 662         fpsimd_flush_task_state(target);
 663
 664         return ret;
 665 }
 666
 667 static int tls_get(struct task_struct *target, const struct user_regset *regset,
 668                    struct membuf to)
 669 {
 670         if (target == current)
 671                 tls_preserve_current_state();
 672
 673         return membuf_store(&to, target->thread.uw.tp_value);
 674 }
 675
 676 static int tls_set(struct task_struct *target, const struct user_regset *regset,
 677                    unsigned int pos, unsigned int count,
 678                    const void *kbuf, const void __user *ubuf)
 679 {
 680         int ret;
 681         unsigned long tls = target->thread.uw.tp_value;
 682
 683         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
 684         if (ret)
 685                 return ret;
 686
 687         target->thread.uw.tp_value = tls;
 688         return ret;
 689 }
 690
 691 static int system_call_get(struct task_struct *target,
 692                            const struct user_regset *regset,
 693                            struct membuf to)
 694 {
 695         return membuf_store(&to, task_pt_regs(target)->syscallno);
 696 }
 697
 698 static int system_call_set(struct task_struct *target,
 699                            const struct user_regset *regset,
 700                            unsigned int pos, unsigned int count,
 701                            const void *kbuf, const void __user *ubuf)
 702 {
 703         int syscallno = task_pt_regs(target)->syscallno;
 704         int ret;
 705
 706         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &syscallno, 0, -1);
 707         if (ret)
 708                 return ret;
 709
 710         task_pt_regs(target)->syscallno = syscallno;
 711         return ret;
 712 }
 713
 714 #ifdef CONFIG_ARM64_SVE
 715
 716 static void sve_init_header_from_task(struct user_sve_header *header,
 717                                       struct task_struct *target)
 718 {
 719         unsigned int vq;
 720
 721         memset(header, 0, sizeof(*header));
 722
 723         header->flags = test_tsk_thread_flag(target, TIF_SVE) ?
 724                 SVE_PT_REGS_SVE : SVE_PT_REGS_FPSIMD;
 725         if (test_tsk_thread_flag(target, TIF_SVE_VL_INHERIT))
 726                 header->flags |= SVE_PT_VL_INHERIT;
 727
 728         header->vl = target->thread.sve_vl;
 729         vq = sve_vq_from_vl(header->vl);
 730
 731         header->max_vl = sve_max_vl;
 732         header->size = SVE_PT_SIZE(vq, header->flags);
 733         header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl),
 734                                       SVE_PT_REGS_SVE);
 735 }
 736
 737 static unsigned int sve_size_from_header(struct user_sve_header const *header)
 738 {
 739         return ALIGN(header->size, SVE_VQ_BYTES);
 740 }
 741
 742 static int sve_get(struct task_struct *target,
 743                    const struct user_regset *regset,
 744                    struct membuf to)
 745 {
 746         struct user_sve_header header;
 747         unsigned int vq;
 748         unsigned long start, end;
 749
 750         if (!system_supports_sve())
 751                 return -EINVAL;
 752
 753         /* Header */
 754         sve_init_header_from_task(&header, target);
 755         vq = sve_vq_from_vl(header.vl);
 756
 757         membuf_write(&to, &header, sizeof(header));
 758
 759         if (target == current)
 760                 fpsimd_preserve_current_state();
 761
 762         /* Registers: FPSIMD-only case */
 763
 764         BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
 765         if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD)
 766                 return __fpr_get(target, regset, to);
 767
 768         /* Otherwise: full SVE case */
 769
 770         BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
 771         start = SVE_PT_SVE_OFFSET;
 772         end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
 773         membuf_write(&to, target->thread.sve_state, end - start);
 774
 775         start = end;
 776         end = SVE_PT_SVE_FPSR_OFFSET(vq);
 777         membuf_zero(&to, end - start);
 778
 779         /*
 780          * Copy fpsr, and fpcr which must follow contiguously in
 781          * struct fpsimd_state:
 782          */
 783         start = end;
 784         end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
 785         membuf_write(&to, &target->thread.uw.fpsimd_state.fpsr, end - start);
 786
 787         start = end;
 788         end = sve_size_from_header(&header);
 789         return membuf_zero(&to, end - start);
 790 }
 791
 792 static int sve_set(struct task_struct *target,
 793                    const struct user_regset *regset,
 794                    unsigned int pos, unsigned int count,
 795                    const void *kbuf, const void __user *ubuf)
 796 {
 797         int ret;
 798         struct user_sve_header header;
 799         unsigned int vq;
 800         unsigned long start, end;
 801
 802         if (!system_supports_sve())
 803                 return -EINVAL;
 804
 805         /* Header */
 806         if (count < sizeof(header))
 807                 return -EINVAL;
 808         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header,
 809                                  0, sizeof(header));
 810         if (ret)
 811                 goto out;
 812
 813         /*
 814          * Apart from SVE_PT_REGS_MASK, all SVE_PT_* flags are consumed by
 815          * sve_set_vector_length(), which will also validate them for us:
 816          */
 817         ret = sve_set_vector_length(target, header.vl,
 818                 ((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16);
 819         if (ret)
 820                 goto out;
 821
 822         /* Actual VL set may be less than the user asked for: */
 823         vq = sve_vq_from_vl(target->thread.sve_vl);
 824
 825         /* Registers: FPSIMD-only case */
 826
 827         BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
 828         if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD) {
 829                 ret = __fpr_set(target, regset, pos, count, kbuf, ubuf,
 830                                 SVE_PT_FPSIMD_OFFSET);
 831                 clear_tsk_thread_flag(target, TIF_SVE);
 832                 goto out;
 833         }
 834
 835         /* Otherwise: full SVE case */
 836
 837         /*
 838          * If setting a different VL from the requested VL and there is
 839          * register data, the data layout will be wrong: don't even
 840          * try to set the registers in this case.
 841          */
 842         if (count && vq != sve_vq_from_vl(header.vl)) {
 843                 ret = -EIO;
 844                 goto out;
 845         }
 846
 847         sve_alloc(target);
 848         if (!target->thread.sve_state) {
 849                 ret = -ENOMEM;
 850                 clear_tsk_thread_flag(target, TIF_SVE);
 851                 goto out;
 852         }
 853
 854         /*
 855          * Ensure target->thread.sve_state is up to date with target's
 856          * FPSIMD regs, so that a short copyin leaves trailing registers
 857          * unmodified.
 858          */
 859         fpsimd_sync_to_sve(target);
 860         set_tsk_thread_flag(target, TIF_SVE);
 861
 862         BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
 863         start = SVE_PT_SVE_OFFSET;
 864         end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
 865         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 866                                  target->thread.sve_state,
 867                                  start, end);
 868         if (ret)
 869                 goto out;
 870
 871         start = end;
 872         end = SVE_PT_SVE_FPSR_OFFSET(vq);
 873         ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
 874                                         start, end);
 875         if (ret)
 876                 goto out;
 877
 878         /*
 879          * Copy fpsr, and fpcr which must follow contiguously in
 880          * struct fpsimd_state:
 881          */
 882         start = end;
 883         end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
 884         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 885                                  &target->thread.uw.fpsimd_state.fpsr,
 886                                  start, end);
 887
 888 out:
 889         fpsimd_flush_task_state(target);
 890         return ret;
 891 }
 892
 893 #endif /* CONFIG_ARM64_SVE */
 894
 895 #ifdef CONFIG_ARM64_PTR_AUTH
 896 static int pac_mask_get(struct task_struct *target,
 897                         const struct user_regset *regset,
 898                         struct membuf to)
 899 {
 900         /*
 901          * The PAC bits can differ across data and instruction pointers
 902          * depending on TCR_EL1.TBID*, which we may make use of in future, so
 903          * we expose separate masks.
 904          */
 905         unsigned long mask = ptrauth_user_pac_mask();
 906         struct user_pac_mask uregs = {
 907                 .data_mask = mask,
 908                 .insn_mask = mask,
 909         };
 910
 911         if (!system_supports_address_auth())
 912                 return -EINVAL;
 913
 914         return membuf_write(&to, &uregs, sizeof(uregs));
 915 }
 916
 917 static int pac_enabled_keys_get(struct task_struct *target,
 918                                 const struct user_regset *regset,
 919                                 struct membuf to)
 920 {
 921         long enabled_keys = ptrauth_get_enabled_keys(target);
 922
 923         if (IS_ERR_VALUE(enabled_keys))
 924                 return enabled_keys;
 925
 926         return membuf_write(&to, &enabled_keys, sizeof(enabled_keys));
 927 }
 928
 929 static int pac_enabled_keys_set(struct task_struct *target,
 930                                 const struct user_regset *regset,
 931                                 unsigned int pos, unsigned int count,
 932                                 const void *kbuf, const void __user *ubuf)
 933 {
 934         int ret;
 935         long enabled_keys = ptrauth_get_enabled_keys(target);
 936
 937         if (IS_ERR_VALUE(enabled_keys))
 938                 return enabled_keys;
 939
 940         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &enabled_keys, 0,
 941                                  sizeof(long));
 942         if (ret)
 943                 return ret;
 944
 945         return ptrauth_set_enabled_keys(target, PR_PAC_ENABLED_KEYS_MASK,
 946                                         enabled_keys);
 947 }
 948
 949 #ifdef CONFIG_CHECKPOINT_RESTORE
 950 static __uint128_t pac_key_to_user(const struct ptrauth_key *key)
 951 {
 952         return (__uint128_t)key->hi << 64 | key->lo;
 953 }
 954
 955 static struct ptrauth_key pac_key_from_user(__uint128_t ukey)
 956 {
 957         struct ptrauth_key key = {
 958                 .lo = (unsigned long)ukey,
 959                 .hi = (unsigned long)(ukey >> 64),
 960         };
 961
 962         return key;
 963 }
 964
 965 static void pac_address_keys_to_user(struct user_pac_address_keys *ukeys,
 966                                      const struct ptrauth_keys_user *keys)
 967 {
 968         ukeys->apiakey = pac_key_to_user(&keys->apia);
 969         ukeys->apibkey = pac_key_to_user(&keys->apib);
 970         ukeys->apdakey = pac_key_to_user(&keys->apda);
 971         ukeys->apdbkey = pac_key_to_user(&keys->apdb);
 972 }
 973
 974 static void pac_address_keys_from_user(struct ptrauth_keys_user *keys,
 975                                        const struct user_pac_address_keys *ukeys)
 976 {
 977         keys->apia = pac_key_from_user(ukeys->apiakey);
 978         keys->apib = pac_key_from_user(ukeys->apibkey);
 979         keys->apda = pac_key_from_user(ukeys->apdakey);
 980         keys->apdb = pac_key_from_user(ukeys->apdbkey);
 981 }
 982
 983 static int pac_address_keys_get(struct task_struct *target,
 984                                 const struct user_regset *regset,
 985                                 struct membuf to)
 986 {
 987         struct ptrauth_keys_user *keys = &target->thread.keys_user;
 988         struct user_pac_address_keys user_keys;
 989
 990         if (!system_supports_address_auth())
 991                 return -EINVAL;
 992
 993         pac_address_keys_to_user(&user_keys, keys);
 994
 995         return membuf_write(&to, &user_keys, sizeof(user_keys));
 996 }
 997
 998 static int pac_address_keys_set(struct task_struct *target,
 999                                 const struct user_regset *regset,
1000                                 unsigned int pos, unsigned int count,
1001                                 const void *kbuf, const void __user *ubuf)
1002 {
1003         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1004         struct user_pac_address_keys user_keys;
1005         int ret;
1006
1007         if (!system_supports_address_auth())
1008                 return -EINVAL;
1009
1010         pac_address_keys_to_user(&user_keys, keys);
1011         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1012                                  &user_keys, 0, -1);
1013         if (ret)
1014                 return ret;
1015         pac_address_keys_from_user(keys, &user_keys);
1016
1017         return 0;
1018 }
1019
1020 static void pac_generic_keys_to_user(struct user_pac_generic_keys *ukeys,
1021                                      const struct ptrauth_keys_user *keys)
1022 {
1023         ukeys->apgakey = pac_key_to_user(&keys->apga);
1024 }
1025
1026 static void pac_generic_keys_from_user(struct ptrauth_keys_user *keys,
1027                                        const struct user_pac_generic_keys *ukeys)
1028 {
1029         keys->apga = pac_key_from_user(ukeys->apgakey);
1030 }
1031
1032 static int pac_generic_keys_get(struct task_struct *target,
1033                                 const struct user_regset *regset,
1034                                 struct membuf to)
1035 {
1036         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1037         struct user_pac_generic_keys user_keys;
1038
1039         if (!system_supports_generic_auth())
1040                 return -EINVAL;
1041
1042         pac_generic_keys_to_user(&user_keys, keys);
1043
1044         return membuf_write(&to, &user_keys, sizeof(user_keys));
1045 }
1046
1047 static int pac_generic_keys_set(struct task_struct *target,
1048                                 const struct user_regset *regset,
1049                                 unsigned int pos, unsigned int count,
1050                                 const void *kbuf, const void __user *ubuf)
1051 {
1052         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1053         struct user_pac_generic_keys user_keys;
1054         int ret;
1055
1056         if (!system_supports_generic_auth())
1057                 return -EINVAL;
1058
1059         pac_generic_keys_to_user(&user_keys, keys);
1060         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1061                                  &user_keys, 0, -1);
1062         if (ret)
1063                 return ret;
1064         pac_generic_keys_from_user(keys, &user_keys);
1065
1066         return 0;
1067 }
1068 #endif /* CONFIG_CHECKPOINT_RESTORE */
1069 #endif /* CONFIG_ARM64_PTR_AUTH */
1070
1071 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1072 static int tagged_addr_ctrl_get(struct task_struct *target,
1073                                 const struct user_regset *regset,
1074                                 struct membuf to)
1075 {
1076         long ctrl = get_tagged_addr_ctrl(target);
1077
1078         if (IS_ERR_VALUE(ctrl))
1079                 return ctrl;
1080
1081         return membuf_write(&to, &ctrl, sizeof(ctrl));
1082 }
1083
1084 static int tagged_addr_ctrl_set(struct task_struct *target, const struct
1085                                 user_regset *regset, unsigned int pos,
1086                                 unsigned int count, const void *kbuf, const
1087                                 void __user *ubuf)
1088 {
1089         int ret;
1090         long ctrl;
1091
1092         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, 0, -1);
1093         if (ret)
1094                 return ret;
1095
1096         return set_tagged_addr_ctrl(target, ctrl);
1097 }
1098 #endif
1099
1100 enum aarch64_regset {
1101         REGSET_GPR,
1102         REGSET_FPR,
1103         REGSET_TLS,
1104 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1105         REGSET_HW_BREAK,
1106         REGSET_HW_WATCH,
1107 #endif
1108         REGSET_SYSTEM_CALL,
1109 #ifdef CONFIG_ARM64_SVE
1110         REGSET_SVE,
1111 #endif
1112 #ifdef CONFIG_ARM64_PTR_AUTH
1113         REGSET_PAC_MASK,
1114         REGSET_PAC_ENABLED_KEYS,
1115 #ifdef CONFIG_CHECKPOINT_RESTORE
1116         REGSET_PACA_KEYS,
1117         REGSET_PACG_KEYS,
1118 #endif
1119 #endif
1120 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1121         REGSET_TAGGED_ADDR_CTRL,
1122 #endif
1123 };
1124
1125 static const struct user_regset aarch64_regsets[] = {
1126         [REGSET_GPR] = {
1127                 .core_note_type = NT_PRSTATUS,
1128                 .n = sizeof(struct user_pt_regs) / sizeof(u64),
1129                 .size = sizeof(u64),
1130                 .align = sizeof(u64),
1131                 .regset_get = gpr_get,
1132                 .set = gpr_set
1133         },
1134         [REGSET_FPR] = {
1135                 .core_note_type = NT_PRFPREG,
1136                 .n = sizeof(struct user_fpsimd_state) / sizeof(u32),
1137                 /*
1138                  * We pretend we have 32-bit registers because the fpsr and
1139                  * fpcr are 32-bits wide.
1140                  */
1141                 .size = sizeof(u32),
1142                 .align = sizeof(u32),
1143                 .active = fpr_active,
1144                 .regset_get = fpr_get,
1145                 .set = fpr_set
1146         },
1147         [REGSET_TLS] = {
1148                 .core_note_type = NT_ARM_TLS,
1149                 .n = 1,
1150                 .size = sizeof(void *),
1151                 .align = sizeof(void *),
1152                 .regset_get = tls_get,
1153                 .set = tls_set,
1154         },
1155 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1156         [REGSET_HW_BREAK] = {
1157                 .core_note_type = NT_ARM_HW_BREAK,
1158                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1159                 .size = sizeof(u32),
1160                 .align = sizeof(u32),
1161                 .regset_get = hw_break_get,
1162                 .set = hw_break_set,
1163         },
1164         [REGSET_HW_WATCH] = {
1165                 .core_note_type = NT_ARM_HW_WATCH,
1166                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1167                 .size = sizeof(u32),
1168                 .align = sizeof(u32),
1169                 .regset_get = hw_break_get,
1170                 .set = hw_break_set,
1171         },
1172 #endif
1173         [REGSET_SYSTEM_CALL] = {
1174                 .core_note_type = NT_ARM_SYSTEM_CALL,
1175                 .n = 1,
1176                 .size = sizeof(int),
1177                 .align = sizeof(int),
1178                 .regset_get = system_call_get,
1179                 .set = system_call_set,
1180         },
1181 #ifdef CONFIG_ARM64_SVE
1182         [REGSET_SVE] = { /* Scalable Vector Extension */
1183                 .core_note_type = NT_ARM_SVE,
1184                 .n = DIV_ROUND_UP(SVE_PT_SIZE(SVE_VQ_MAX, SVE_PT_REGS_SVE),
1185                                   SVE_VQ_BYTES),
1186                 .size = SVE_VQ_BYTES,
1187                 .align = SVE_VQ_BYTES,
1188                 .regset_get = sve_get,
1189                 .set = sve_set,
1190         },
1191 #endif
1192 #ifdef CONFIG_ARM64_PTR_AUTH
1193         [REGSET_PAC_MASK] = {
1194                 .core_note_type = NT_ARM_PAC_MASK,
1195                 .n = sizeof(struct user_pac_mask) / sizeof(u64),
1196                 .size = sizeof(u64),
1197                 .align = sizeof(u64),
1198                 .regset_get = pac_mask_get,
1199                 /* this cannot be set dynamically */
1200         },
1201         [REGSET_PAC_ENABLED_KEYS] = {
1202                 .core_note_type = NT_ARM_PAC_ENABLED_KEYS,
1203                 .n = 1,
1204                 .size = sizeof(long),
1205                 .align = sizeof(long),
1206                 .regset_get = pac_enabled_keys_get,
1207                 .set = pac_enabled_keys_set,
1208         },
1209 #ifdef CONFIG_CHECKPOINT_RESTORE
1210         [REGSET_PACA_KEYS] = {
1211                 .core_note_type = NT_ARM_PACA_KEYS,
1212                 .n = sizeof(struct user_pac_address_keys) / sizeof(__uint128_t),
1213                 .size = sizeof(__uint128_t),
1214                 .align = sizeof(__uint128_t),
1215                 .regset_get = pac_address_keys_get,
1216                 .set = pac_address_keys_set,
1217         },
1218         [REGSET_PACG_KEYS] = {
1219                 .core_note_type = NT_ARM_PACG_KEYS,
1220                 .n = sizeof(struct user_pac_generic_keys) / sizeof(__uint128_t),
1221                 .size = sizeof(__uint128_t),
1222                 .align = sizeof(__uint128_t),
1223                 .regset_get = pac_generic_keys_get,
1224                 .set = pac_generic_keys_set,
1225         },
1226 #endif
1227 #endif
1228 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1229         [REGSET_TAGGED_ADDR_CTRL] = {
1230                 .core_note_type = NT_ARM_TAGGED_ADDR_CTRL,
1231                 .n = 1,
1232                 .size = sizeof(long),
1233                 .align = sizeof(long),
1234                 .regset_get = tagged_addr_ctrl_get,
1235                 .set = tagged_addr_ctrl_set,
1236         },
1237 #endif
1238 };
1239
1240 static const struct user_regset_view user_aarch64_view = {
1241         .name = "aarch64", .e_machine = EM_AARCH64,
1242         .regsets = aarch64_regsets, .n = ARRAY_SIZE(aarch64_regsets)
1243 };
1244
1245 #ifdef CONFIG_COMPAT
1246 enum compat_regset {
1247         REGSET_COMPAT_GPR,
1248         REGSET_COMPAT_VFP,
1249 };
1250
1251 static inline compat_ulong_t compat_get_user_reg(struct task_struct *task, int idx)
1252 {
1253         struct pt_regs *regs = task_pt_regs(task);
1254
1255         switch (idx) {
1256         case 15:
1257                 return regs->pc;
1258         case 16:
1259                 return pstate_to_compat_psr(regs->pstate);
1260         case 17:
1261                 return regs->orig_x0;
1262         default:
1263                 return regs->regs[idx];
1264         }
1265 }
1266
1267 static int compat_gpr_get(struct task_struct *target,
1268                           const struct user_regset *regset,
1269                           struct membuf to)
1270 {
1271         int i = 0;
1272
1273         while (to.left)
1274                 membuf_store(&to, compat_get_user_reg(target, i++));
1275         return 0;
1276 }
1277
1278 static int compat_gpr_set(struct task_struct *target,
1279                           const struct user_regset *regset,
1280                           unsigned int pos, unsigned int count,
1281                           const void *kbuf, const void __user *ubuf)
1282 {
1283         struct pt_regs newregs;
1284         int ret = 0;
1285         unsigned int i, start, num_regs;
1286
1287         /* Calculate the number of AArch32 registers contained in count */
1288         num_regs = count / regset->size;
1289
1290         /* Convert pos into an register number */
1291         start = pos / regset->size;
1292
1293         if (start + num_regs > regset->n)
1294                 return -EIO;
1295
1296         newregs = *task_pt_regs(target);
1297
1298         for (i = 0; i < num_regs; ++i) {
1299                 unsigned int idx = start + i;
1300                 compat_ulong_t reg;
1301
1302                 if (kbuf) {
1303                         memcpy(&reg, kbuf, sizeof(reg));
1304                         kbuf += sizeof(reg);
1305                 } else {
1306                         ret = copy_from_user(&reg, ubuf, sizeof(reg));
1307                         if (ret) {
1308                                 ret = -EFAULT;
1309                                 break;
1310                         }
1311
1312                         ubuf += sizeof(reg);
1313                 }
1314
1315                 switch (idx) {
1316                 case 15:
1317                         newregs.pc = reg;
1318                         break;
1319                 case 16:
1320                         reg = compat_psr_to_pstate(reg);
1321                         newregs.pstate = reg;
1322                         break;
1323                 case 17:
1324                         newregs.orig_x0 = reg;
1325                         break;
1326                 default:
1327                         newregs.regs[idx] = reg;
1328                 }
1329
1330         }
1331
1332         if (valid_user_regs(&newregs.user_regs, target))
1333                 *task_pt_regs(target) = newregs;
1334         else
1335                 ret = -EINVAL;
1336
1337         return ret;
1338 }
1339
1340 static int compat_vfp_get(struct task_struct *target,
1341                           const struct user_regset *regset,
1342                           struct membuf to)
1343 {
1344         struct user_fpsimd_state *uregs;
1345         compat_ulong_t fpscr;
1346
1347         if (!system_supports_fpsimd())
1348                 return -EINVAL;
1349
1350         uregs = &target->thread.uw.fpsimd_state;
1351
1352         if (target == current)
1353                 fpsimd_preserve_current_state();
1354
1355         /*
1356          * The VFP registers are packed into the fpsimd_state, so they all sit
1357          * nicely together for us. We just need to create the fpscr separately.
1358          */
1359         membuf_write(&to, uregs, VFP_STATE_SIZE - sizeof(compat_ulong_t));
1360         fpscr = (uregs->fpsr & VFP_FPSCR_STAT_MASK) |
1361                 (uregs->fpcr & VFP_FPSCR_CTRL_MASK);
1362         return membuf_store(&to, fpscr);
1363 }
1364
1365 static int compat_vfp_set(struct task_struct *target,
1366                           const struct user_regset *regset,
1367                           unsigned int pos, unsigned int count,
1368                           const void *kbuf, const void __user *ubuf)
1369 {
1370         struct user_fpsimd_state *uregs;
1371         compat_ulong_t fpscr;
1372         int ret, vregs_end_pos;
1373
1374         if (!system_supports_fpsimd())
1375                 return -EINVAL;
1376
1377         uregs = &target->thread.uw.fpsimd_state;
1378
1379         vregs_end_pos = VFP_STATE_SIZE - sizeof(compat_ulong_t);
1380         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, uregs, 0,
1381                                  vregs_end_pos);
1382
1383         if (count && !ret) {
1384                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fpscr,
1385                                          vregs_end_pos, VFP_STATE_SIZE);
1386                 if (!ret) {
1387                         uregs->fpsr = fpscr & VFP_FPSCR_STAT_MASK;
1388                         uregs->fpcr = fpscr & VFP_FPSCR_CTRL_MASK;
1389                 }
1390         }
1391
1392         fpsimd_flush_task_state(target);
1393         return ret;
1394 }
1395
1396 static int compat_tls_get(struct task_struct *target,
1397                           const struct user_regset *regset,
1398                           struct membuf to)
1399 {
1400         return membuf_store(&to, (compat_ulong_t)target->thread.uw.tp_value);
1401 }
1402
1403 static int compat_tls_set(struct task_struct *target,
1404                           const struct user_regset *regset, unsigned int pos,
1405                           unsigned int count, const void *kbuf,
1406                           const void __user *ubuf)
1407 {
1408         int ret;
1409         compat_ulong_t tls = target->thread.uw.tp_value;
1410
1411         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
1412         if (ret)
1413                 return ret;
1414
1415         target->thread.uw.tp_value = tls;
1416         return ret;
1417 }
1418
1419 static const struct user_regset aarch32_regsets[] = {
1420         [REGSET_COMPAT_GPR] = {
1421                 .core_note_type = NT_PRSTATUS,
1422                 .n = COMPAT_ELF_NGREG,
1423                 .size = sizeof(compat_elf_greg_t),
1424                 .align = sizeof(compat_elf_greg_t),
1425                 .regset_get = compat_gpr_get,
1426                 .set = compat_gpr_set
1427         },
1428         [REGSET_COMPAT_VFP] = {
1429                 .core_note_type = NT_ARM_VFP,
1430                 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
1431                 .size = sizeof(compat_ulong_t),
1432                 .align = sizeof(compat_ulong_t),
1433                 .active = fpr_active,
1434                 .regset_get = compat_vfp_get,
1435                 .set = compat_vfp_set
1436         },
1437 };
1438
1439 static const struct user_regset_view user_aarch32_view = {
1440         .name = "aarch32", .e_machine = EM_ARM,
1441         .regsets = aarch32_regsets, .n = ARRAY_SIZE(aarch32_regsets)
1442 };
1443
1444 static const struct user_regset aarch32_ptrace_regsets[] = {
1445         [REGSET_GPR] = {
1446                 .core_note_type = NT_PRSTATUS,
1447                 .n = COMPAT_ELF_NGREG,
1448                 .size = sizeof(compat_elf_greg_t),
1449                 .align = sizeof(compat_elf_greg_t),
1450                 .regset_get = compat_gpr_get,
1451                 .set = compat_gpr_set
1452         },
1453         [REGSET_FPR] = {
1454                 .core_note_type = NT_ARM_VFP,
1455                 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
1456                 .size = sizeof(compat_ulong_t),
1457                 .align = sizeof(compat_ulong_t),
1458                 .regset_get = compat_vfp_get,
1459                 .set = compat_vfp_set
1460         },
1461         [REGSET_TLS] = {
1462                 .core_note_type = NT_ARM_TLS,
1463                 .n = 1,
1464                 .size = sizeof(compat_ulong_t),
1465                 .align = sizeof(compat_ulong_t),
1466                 .regset_get = compat_tls_get,
1467                 .set = compat_tls_set,
1468         },
1469 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1470         [REGSET_HW_BREAK] = {
1471                 .core_note_type = NT_ARM_HW_BREAK,
1472                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1473                 .size = sizeof(u32),
1474                 .align = sizeof(u32),
1475                 .regset_get = hw_break_get,
1476                 .set = hw_break_set,
1477         },
1478         [REGSET_HW_WATCH] = {
1479                 .core_note_type = NT_ARM_HW_WATCH,
1480                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1481                 .size = sizeof(u32),
1482                 .align = sizeof(u32),
1483                 .regset_get = hw_break_get,
1484                 .set = hw_break_set,
1485         },
1486 #endif
1487         [REGSET_SYSTEM_CALL] = {
1488                 .core_note_type = NT_ARM_SYSTEM_CALL,
1489                 .n = 1,
1490                 .size = sizeof(int),
1491                 .align = sizeof(int),
1492                 .regset_get = system_call_get,
1493                 .set = system_call_set,
1494         },
1495 };
1496
1497 static const struct user_regset_view user_aarch32_ptrace_view = {
1498         .name = "aarch32", .e_machine = EM_ARM,
1499         .regsets = aarch32_ptrace_regsets, .n = ARRAY_SIZE(aarch32_ptrace_regsets)
1500 };
1501
1502 static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off,
1503                                    compat_ulong_t __user *ret)
1504 {
1505         compat_ulong_t tmp;
1506
1507         if (off & 3)
1508                 return -EIO;
1509
1510         if (off == COMPAT_PT_TEXT_ADDR)
1511                 tmp = tsk->mm->start_code;
1512         else if (off == COMPAT_PT_DATA_ADDR)
1513                 tmp = tsk->mm->start_data;
1514         else if (off == COMPAT_PT_TEXT_END_ADDR)
1515                 tmp = tsk->mm->end_code;
1516         else if (off < sizeof(compat_elf_gregset_t))
1517                 tmp = compat_get_user_reg(tsk, off >> 2);
1518         else if (off >= COMPAT_USER_SZ)
1519                 return -EIO;
1520         else
1521                 tmp = 0;
1522
1523         return put_user(tmp, ret);
1524 }
1525
1526 static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off,
1527                                     compat_ulong_t val)
1528 {
1529         struct pt_regs newregs = *task_pt_regs(tsk);
1530         unsigned int idx = off / 4;
1531
1532         if (off & 3 || off >= COMPAT_USER_SZ)
1533                 return -EIO;
1534
1535         if (off >= sizeof(compat_elf_gregset_t))
1536                 return 0;
1537
1538         switch (idx) {
1539         case 15:
1540                 newregs.pc = val;
1541                 break;
1542         case 16:
1543                 newregs.pstate = compat_psr_to_pstate(val);
1544                 break;
1545         case 17:
1546                 newregs.orig_x0 = val;
1547                 break;
1548         default:
1549                 newregs.regs[idx] = val;
1550         }
1551
1552         if (!valid_user_regs(&newregs.user_regs, tsk))
1553                 return -EINVAL;
1554
1555         *task_pt_regs(tsk) = newregs;
1556         return 0;
1557 }
1558
1559 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1560
1561 /*
1562  * Convert a virtual register number into an index for a thread_info
1563  * breakpoint array. Breakpoints are identified using positive numbers
1564  * whilst watchpoints are negative. The registers are laid out as pairs
1565  * of (address, control), each pair mapping to a unique hw_breakpoint struct.
1566  * Register 0 is reserved for describing resource information.
1567  */
1568 static int compat_ptrace_hbp_num_to_idx(compat_long_t num)
1569 {
1570         return (abs(num) - 1) >> 1;
1571 }
1572
1573 static int compat_ptrace_hbp_get_resource_info(u32 *kdata)
1574 {
1575         u8 num_brps, num_wrps, debug_arch, wp_len;
1576         u32 reg = 0;
1577
1578         num_brps        = hw_breakpoint_slots(TYPE_INST);
1579         num_wrps        = hw_breakpoint_slots(TYPE_DATA);
1580
1581         debug_arch      = debug_monitors_arch();
1582         wp_len          = 8;
1583         reg             |= debug_arch;
1584         reg             <<= 8;
1585         reg             |= wp_len;
1586         reg             <<= 8;
1587         reg             |= num_wrps;
1588         reg             <<= 8;
1589         reg             |= num_brps;
1590
1591         *kdata = reg;
1592         return 0;
1593 }
1594
1595 static int compat_ptrace_hbp_get(unsigned int note_type,
1596                                  struct task_struct *tsk,
1597                                  compat_long_t num,
1598                                  u32 *kdata)
1599 {
1600         u64 addr = 0;
1601         u32 ctrl = 0;
1602
1603         int err, idx = compat_ptrace_hbp_num_to_idx(num);
1604
1605         if (num & 1) {
1606                 err = ptrace_hbp_get_addr(note_type, tsk, idx, &addr);
1607                 *kdata = (u32)addr;
1608         } else {
1609                 err = ptrace_hbp_get_ctrl(note_type, tsk, idx, &ctrl);
1610                 *kdata = ctrl;
1611         }
1612
1613         return err;
1614 }
1615
1616 static int compat_ptrace_hbp_set(unsigned int note_type,
1617                                  struct task_struct *tsk,
1618                                  compat_long_t num,
1619                                  u32 *kdata)
1620 {
1621         u64 addr;
1622         u32 ctrl;
1623
1624         int err, idx = compat_ptrace_hbp_num_to_idx(num);
1625
1626         if (num & 1) {
1627                 addr = *kdata;
1628                 err = ptrace_hbp_set_addr(note_type, tsk, idx, addr);
1629         } else {
1630                 ctrl = *kdata;
1631                 err = ptrace_hbp_set_ctrl(note_type, tsk, idx, ctrl);
1632         }
1633
1634         return err;
1635 }
1636
1637 static int compat_ptrace_gethbpregs(struct task_struct *tsk, compat_long_t num,
1638                                     compat_ulong_t __user *data)
1639 {
1640         int ret;
1641         u32 kdata;
1642
1643         /* Watchpoint */
1644         if (num < 0) {
1645                 ret = compat_ptrace_hbp_get(NT_ARM_HW_WATCH, tsk, num, &kdata);
1646         /* Resource info */
1647         } else if (num == 0) {
1648                 ret = compat_ptrace_hbp_get_resource_info(&kdata);
1649         /* Breakpoint */
1650         } else {
1651                 ret = compat_ptrace_hbp_get(NT_ARM_HW_BREAK, tsk, num, &kdata);
1652         }
1653
1654         if (!ret)
1655                 ret = put_user(kdata, data);
1656
1657         return ret;
1658 }
1659
1660 static int compat_ptrace_sethbpregs(struct task_struct *tsk, compat_long_t num,
1661                                     compat_ulong_t __user *data)
1662 {
1663         int ret;
1664         u32 kdata = 0;
1665
1666         if (num == 0)
1667                 return 0;
1668
1669         ret = get_user(kdata, data);
1670         if (ret)
1671                 return ret;
1672
1673         if (num < 0)
1674                 ret = compat_ptrace_hbp_set(NT_ARM_HW_WATCH, tsk, num, &kdata);
1675         else
1676                 ret = compat_ptrace_hbp_set(NT_ARM_HW_BREAK, tsk, num, &kdata);
1677
1678         return ret;
1679 }
1680 #endif  /* CONFIG_HAVE_HW_BREAKPOINT */
1681
1682 long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
1683                         compat_ulong_t caddr, compat_ulong_t cdata)
1684 {
1685         unsigned long addr = caddr;
1686         unsigned long data = cdata;
1687         void __user *datap = compat_ptr(data);
1688         int ret;
1689
1690         switch (request) {
1691                 case PTRACE_PEEKUSR:
1692                         ret = compat_ptrace_read_user(child, addr, datap);
1693                         break;
1694
1695                 case PTRACE_POKEUSR:
1696                         ret = compat_ptrace_write_user(child, addr, data);
1697                         break;
1698
1699                 case COMPAT_PTRACE_GETREGS:
1700                         ret = copy_regset_to_user(child,
1701                                                   &user_aarch32_view,
1702                                                   REGSET_COMPAT_GPR,
1703                                                   0, sizeof(compat_elf_gregset_t),
1704                                                   datap);
1705                         break;
1706
1707                 case COMPAT_PTRACE_SETREGS:
1708                         ret = copy_regset_from_user(child,
1709                                                     &user_aarch32_view,
1710                                                     REGSET_COMPAT_GPR,
1711                                                     0, sizeof(compat_elf_gregset_t),
1712                                                     datap);
1713                         break;
1714
1715                 case COMPAT_PTRACE_GET_THREAD_AREA:
1716                         ret = put_user((compat_ulong_t)child->thread.uw.tp_value,
1717                                        (compat_ulong_t __user *)datap);
1718                         break;
1719
1720                 case COMPAT_PTRACE_SET_SYSCALL:
1721                         task_pt_regs(child)->syscallno = data;
1722                         ret = 0;
1723                         break;
1724
1725                 case COMPAT_PTRACE_GETVFPREGS:
1726                         ret = copy_regset_to_user(child,
1727                                                   &user_aarch32_view,
1728                                                   REGSET_COMPAT_VFP,
1729                                                   0, VFP_STATE_SIZE,
1730                                                   datap);
1731                         break;
1732
1733                 case COMPAT_PTRACE_SETVFPREGS:
1734                         ret = copy_regset_from_user(child,
1735                                                     &user_aarch32_view,
1736                                                     REGSET_COMPAT_VFP,
1737                                                     0, VFP_STATE_SIZE,
1738                                                     datap);
1739                         break;
1740
1741 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1742                 case COMPAT_PTRACE_GETHBPREGS:
1743                         ret = compat_ptrace_gethbpregs(child, addr, datap);
1744                         break;
1745
1746                 case COMPAT_PTRACE_SETHBPREGS:
1747                         ret = compat_ptrace_sethbpregs(child, addr, datap);
1748                         break;
1749 #endif
1750
1751                 default:
1752                         ret = compat_ptrace_request(child, request, addr,
1753                                                     data);
1754                         break;
1755         }
1756
1757         return ret;
1758 }
1759 #endif /* CONFIG_COMPAT */
1760
1761 const struct user_regset_view *task_user_regset_view(struct task_struct *task)
1762 {
1763 #ifdef CONFIG_COMPAT
1764         /*
1765          * Core dumping of 32-bit tasks or compat ptrace requests must use the
1766          * user_aarch32_view compatible with arm32. Native ptrace requests on
1767          * 32-bit children use an extended user_aarch32_ptrace_view to allow
1768          * access to the TLS register.
1769          */
1770         if (is_compat_task())
1771                 return &user_aarch32_view;
1772         else if (is_compat_thread(task_thread_info(task)))
1773                 return &user_aarch32_ptrace_view;
1774 #endif
1775         return &user_aarch64_view;
1776 }
1777
1778 long arch_ptrace(struct task_struct *child, long request,
1779                  unsigned long addr, unsigned long data)
1780 {
1781         switch (request) {
1782         case PTRACE_PEEKMTETAGS:
1783         case PTRACE_POKEMTETAGS:
1784                 return mte_ptrace_copy_tags(child, request, addr, data);
1785         }
1786
1787         return ptrace_request(child, request, addr, data);
1788 }
1789
1790 enum ptrace_syscall_dir {
1791         PTRACE_SYSCALL_ENTER = 0,
1792         PTRACE_SYSCALL_EXIT,
1793 };
1794
1795 static void tracehook_report_syscall(struct pt_regs *regs,
1796                                      enum ptrace_syscall_dir dir)
1797 {
1798         int regno;
1799         unsigned long saved_reg;
1800
1801         /*
1802          * We have some ABI weirdness here in the way that we handle syscall
1803          * exit stops because we indicate whether or not the stop has been
1804          * signalled from syscall entry or syscall exit by clobbering a general
1805          * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
1806          * and restoring its old value after the stop. This means that:
1807          *
1808          * - Any writes by the tracer to this register during the stop are
1809          *   ignored/discarded.
1810          *
1811          * - The actual value of the register is not available during the stop,
1812          *   so the tracer cannot save it and restore it later.
1813          *
1814          * - Syscall stops behave differently to seccomp and pseudo-step traps
1815          *   (the latter do not nobble any registers).
1816          */
1817         regno = (is_compat_task() ? 12 : 7);
1818         saved_reg = regs->regs[regno];
1819         regs->regs[regno] = dir;
1820
1821         if (dir == PTRACE_SYSCALL_ENTER) {
1822                 if (tracehook_report_syscall_entry(regs))
1823                         forget_syscall(regs);
1824                 regs->regs[regno] = saved_reg;
1825         } else if (!test_thread_flag(TIF_SINGLESTEP)) {
1826                 tracehook_report_syscall_exit(regs, 0);
1827                 regs->regs[regno] = saved_reg;
1828         } else {
1829                 regs->regs[regno] = saved_reg;
1830
1831                 /*
1832                  * Signal a pseudo-step exception since we are stepping but
1833                  * tracer modifications to the registers may have rewound the
1834                  * state machine.
1835                  */
1836                 tracehook_report_syscall_exit(regs, 1);
1837         }
1838 }
1839
1840 int syscall_trace_enter(struct pt_regs *regs)
1841 {
1842         unsigned long flags = READ_ONCE(current_thread_info()->flags);
1843
1844         if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) {
1845                 tracehook_report_syscall(regs, PTRACE_SYSCALL_ENTER);
1846                 if (flags & _TIF_SYSCALL_EMU)
1847                         return NO_SYSCALL;
1848         }
1849
1850         /* Do the secure computing after ptrace; failures should be fast. */
1851         if (secure_computing() == -1)
1852                 return NO_SYSCALL;
1853
1854         if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
1855                 trace_sys_enter(regs, regs->syscallno);
1856
1857         audit_syscall_entry(regs->syscallno, regs->orig_x0, regs->regs[1],
1858                             regs->regs[2], regs->regs[3]);
1859
1860         return regs->syscallno;
1861 }
1862
1863 void syscall_trace_exit(struct pt_regs *regs)
1864 {
1865         unsigned long flags = READ_ONCE(current_thread_info()->flags);
1866
1867         audit_syscall_exit(regs);
1868
1869         if (flags & _TIF_SYSCALL_TRACEPOINT)
1870                 trace_sys_exit(regs, syscall_get_return_value(current, regs));
1871
1872         if (flags & (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP))
1873                 tracehook_report_syscall(regs, PTRACE_SYSCALL_EXIT);
1874
1875         rseq_syscall(regs);
1876 }
1877
1878 /*
1879  * SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487D.a.
1880  * We permit userspace to set SSBS (AArch64 bit 12, AArch32 bit 23) which is
1881  * not described in ARM DDI 0487D.a.
1882  * We treat PAN and UAO as RES0 bits, as they are meaningless at EL0, and may
1883  * be allocated an EL0 meaning in future.
1884  * Userspace cannot use these until they have an architectural meaning.
1885  * Note that this follows the SPSR_ELx format, not the AArch32 PSR format.
1886  * We also reserve IL for the kernel; SS is handled dynamically.
1887  */
1888 #define SPSR_EL1_AARCH64_RES0_BITS \
1889         (GENMASK_ULL(63, 32) | GENMASK_ULL(27, 26) | GENMASK_ULL(23, 22) | \
1890          GENMASK_ULL(20, 13) | GENMASK_ULL(5, 5))
1891 #define SPSR_EL1_AARCH32_RES0_BITS \
1892         (GENMASK_ULL(63, 32) | GENMASK_ULL(22, 22) | GENMASK_ULL(20, 20))
1893
1894 static int valid_compat_regs(struct user_pt_regs *regs)
1895 {
1896         regs->pstate &= ~SPSR_EL1_AARCH32_RES0_BITS;
1897
1898         if (!system_supports_mixed_endian_el0()) {
1899                 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
1900                         regs->pstate |= PSR_AA32_E_BIT;
1901                 else
1902                         regs->pstate &= ~PSR_AA32_E_BIT;
1903         }
1904
1905         if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) &&
1906             (regs->pstate & PSR_AA32_A_BIT) == 0 &&
1907             (regs->pstate & PSR_AA32_I_BIT) == 0 &&
1908             (regs->pstate & PSR_AA32_F_BIT) == 0) {
1909                 return 1;
1910         }
1911
1912         /*
1913          * Force PSR to a valid 32-bit EL0t, preserving the same bits as
1914          * arch/arm.
1915          */
1916         regs->pstate &= PSR_AA32_N_BIT | PSR_AA32_Z_BIT |
1917                         PSR_AA32_C_BIT | PSR_AA32_V_BIT |
1918                         PSR_AA32_Q_BIT | PSR_AA32_IT_MASK |
1919                         PSR_AA32_GE_MASK | PSR_AA32_E_BIT |
1920                         PSR_AA32_T_BIT;
1921         regs->pstate |= PSR_MODE32_BIT;
1922
1923         return 0;
1924 }
1925
1926 static int valid_native_regs(struct user_pt_regs *regs)
1927 {
1928         regs->pstate &= ~SPSR_EL1_AARCH64_RES0_BITS;
1929
1930         if (user_mode(regs) && !(regs->pstate & PSR_MODE32_BIT) &&
1931             (regs->pstate & PSR_D_BIT) == 0 &&
1932             (regs->pstate & PSR_A_BIT) == 0 &&
1933             (regs->pstate & PSR_I_BIT) == 0 &&
1934             (regs->pstate & PSR_F_BIT) == 0) {
1935                 return 1;
1936         }
1937
1938         /* Force PSR to a valid 64-bit EL0t */
1939         regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT;
1940
1941         return 0;
1942 }
1943
1944 /*
1945  * Are the current registers suitable for user mode? (used to maintain
1946  * security in signal handlers)
1947  */
1948 int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task)
1949 {
1950         /* https://lore.kernel.org/lkml/20191118131525.GA4180@willie-the-truck */
1951         user_regs_reset_single_step(regs, task);
1952
1953         if (is_compat_thread(task_thread_info(task)))
1954                 return valid_compat_regs(regs);
1955         else
1956                 return valid_native_regs(regs);
1957 }