arch/arm64/kvm/sys_regs.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2012,2013 - ARM Ltd
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  *
   6  * Derived from arch/arm/kvm/coproc.c:
   7  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
   8  * Authors: Rusty Russell <rusty@rustcorp.com.au>
   9  *          Christoffer Dall <c.dall@virtualopensystems.com>
  10  */
  11
  12 #include <linux/bitfield.h>
  13 #include <linux/bsearch.h>
  14 #include <linux/kvm_host.h>
  15 #include <linux/mm.h>
  16 #include <linux/printk.h>
  17 #include <linux/uaccess.h>
  18
  19 #include <asm/cacheflush.h>
  20 #include <asm/cputype.h>
  21 #include <asm/debug-monitors.h>
  22 #include <asm/esr.h>
  23 #include <asm/kvm_arm.h>
  24 #include <asm/kvm_emulate.h>
  25 #include <asm/kvm_hyp.h>
  26 #include <asm/kvm_mmu.h>
  27 #include <asm/perf_event.h>
  28 #include <asm/sysreg.h>
  29
  30 #include <trace/events/kvm.h>
  31
  32 #include "sys_regs.h"
  33
  34 #include "trace.h"
  35
  36 /*
  37  * All of this file is extremely similar to the ARM coproc.c, but the
  38  * types are different. My gut feeling is that it should be pretty
  39  * easy to merge, but that would be an ABI breakage -- again. VFP
  40  * would also need to be abstracted.
  41  *
  42  * For AArch32, we only take care of what is being trapped. Anything
  43  * that has to do with init and userspace access has to go via the
  44  * 64bit interface.
  45  */
  46
  47 #define reg_to_encoding(x)                                              \
  48         sys_reg((u32)(x)->Op0, (u32)(x)->Op1,                           \
  49                 (u32)(x)->CRn, (u32)(x)->CRm, (u32)(x)->Op2)
  50
  51 static bool read_from_write_only(struct kvm_vcpu *vcpu,
  52                                  struct sys_reg_params *params,
  53                                  const struct sys_reg_desc *r)
  54 {
  55         WARN_ONCE(1, "Unexpected sys_reg read to write-only register\n");
  56         print_sys_reg_instr(params);
  57         kvm_inject_undefined(vcpu);
  58         return false;
  59 }
  60
  61 static bool write_to_read_only(struct kvm_vcpu *vcpu,
  62                                struct sys_reg_params *params,
  63                                const struct sys_reg_desc *r)
  64 {
  65         WARN_ONCE(1, "Unexpected sys_reg write to read-only register\n");
  66         print_sys_reg_instr(params);
  67         kvm_inject_undefined(vcpu);
  68         return false;
  69 }
  70
  71 u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
  72 {
  73         u64 val = 0x8badf00d8badf00d;
  74
  75         if (vcpu->arch.sysregs_loaded_on_cpu &&
  76             __vcpu_read_sys_reg_from_cpu(reg, &val))
  77                 return val;
  78
  79         return __vcpu_sys_reg(vcpu, reg);
  80 }
  81
  82 void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
  83 {
  84         if (vcpu->arch.sysregs_loaded_on_cpu &&
  85             __vcpu_write_sys_reg_to_cpu(val, reg))
  86                 return;
  87
  88          __vcpu_sys_reg(vcpu, reg) = val;
  89 }
  90
  91 /* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
  92 static u32 cache_levels;
  93
  94 /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
  95 #define CSSELR_MAX 14
  96
  97 /* Which cache CCSIDR represents depends on CSSELR value. */
  98 static u32 get_ccsidr(u32 csselr)
  99 {
 100         u32 ccsidr;
 101
 102         /* Make sure noone else changes CSSELR during this! */
 103         local_irq_disable();
 104         write_sysreg(csselr, csselr_el1);
 105         isb();
 106         ccsidr = read_sysreg(ccsidr_el1);
 107         local_irq_enable();
 108
 109         return ccsidr;
 110 }
 111
 112 /*
 113  * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
 114  */
 115 static bool access_dcsw(struct kvm_vcpu *vcpu,
 116                         struct sys_reg_params *p,
 117                         const struct sys_reg_desc *r)
 118 {
 119         if (!p->is_write)
 120                 return read_from_write_only(vcpu, p, r);
 121
 122         /*
 123          * Only track S/W ops if we don't have FWB. It still indicates
 124          * that the guest is a bit broken (S/W operations should only
 125          * be done by firmware, knowing that there is only a single
 126          * CPU left in the system, and certainly not from non-secure
 127          * software).
 128          */
 129         if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
 130                 kvm_set_way_flush(vcpu);
 131
 132         return true;
 133 }
 134
 135 static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift)
 136 {
 137         switch (r->aarch32_map) {
 138         case AA32_LO:
 139                 *mask = GENMASK_ULL(31, 0);
 140                 *shift = 0;
 141                 break;
 142         case AA32_HI:
 143                 *mask = GENMASK_ULL(63, 32);
 144                 *shift = 32;
 145                 break;
 146         default:
 147                 *mask = GENMASK_ULL(63, 0);
 148                 *shift = 0;
 149                 break;
 150         }
 151 }
 152
 153 /*
 154  * Generic accessor for VM registers. Only called as long as HCR_TVM
 155  * is set. If the guest enables the MMU, we stop trapping the VM
 156  * sys_regs and leave it in complete control of the caches.
 157  */
 158 static bool access_vm_reg(struct kvm_vcpu *vcpu,
 159                           struct sys_reg_params *p,
 160                           const struct sys_reg_desc *r)
 161 {
 162         bool was_enabled = vcpu_has_cache_enabled(vcpu);
 163         u64 val, mask, shift;
 164
 165         BUG_ON(!p->is_write);
 166
 167         get_access_mask(r, &mask, &shift);
 168
 169         if (~mask) {
 170                 val = vcpu_read_sys_reg(vcpu, r->reg);
 171                 val &= ~mask;
 172         } else {
 173                 val = 0;
 174         }
 175
 176         val |= (p->regval & (mask >> shift)) << shift;
 177         vcpu_write_sys_reg(vcpu, val, r->reg);
 178
 179         kvm_toggle_cache(vcpu, was_enabled);
 180         return true;
 181 }
 182
 183 static bool access_actlr(struct kvm_vcpu *vcpu,
 184                          struct sys_reg_params *p,
 185                          const struct sys_reg_desc *r)
 186 {
 187         u64 mask, shift;
 188
 189         if (p->is_write)
 190                 return ignore_write(vcpu, p);
 191
 192         get_access_mask(r, &mask, &shift);
 193         p->regval = (vcpu_read_sys_reg(vcpu, r->reg) & mask) >> shift;
 194
 195         return true;
 196 }
 197
 198 /*
 199  * Trap handler for the GICv3 SGI generation system register.
 200  * Forward the request to the VGIC emulation.
 201  * The cp15_64 code makes sure this automatically works
 202  * for both AArch64 and AArch32 accesses.
 203  */
 204 static bool access_gic_sgi(struct kvm_vcpu *vcpu,
 205                            struct sys_reg_params *p,
 206                            const struct sys_reg_desc *r)
 207 {
 208         bool g1;
 209
 210         if (!p->is_write)
 211                 return read_from_write_only(vcpu, p, r);
 212
 213         /*
 214          * In a system where GICD_CTLR.DS=1, a ICC_SGI0R_EL1 access generates
 215          * Group0 SGIs only, while ICC_SGI1R_EL1 can generate either group,
 216          * depending on the SGI configuration. ICC_ASGI1R_EL1 is effectively
 217          * equivalent to ICC_SGI0R_EL1, as there is no "alternative" secure
 218          * group.
 219          */
 220         if (p->Op0 == 0) {              /* AArch32 */
 221                 switch (p->Op1) {
 222                 default:                /* Keep GCC quiet */
 223                 case 0:                 /* ICC_SGI1R */
 224                         g1 = true;
 225                         break;
 226                 case 1:                 /* ICC_ASGI1R */
 227                 case 2:                 /* ICC_SGI0R */
 228                         g1 = false;
 229                         break;
 230                 }
 231         } else {                        /* AArch64 */
 232                 switch (p->Op2) {
 233                 default:                /* Keep GCC quiet */
 234                 case 5:                 /* ICC_SGI1R_EL1 */
 235                         g1 = true;
 236                         break;
 237                 case 6:                 /* ICC_ASGI1R_EL1 */
 238                 case 7:                 /* ICC_SGI0R_EL1 */
 239                         g1 = false;
 240                         break;
 241                 }
 242         }
 243
 244         vgic_v3_dispatch_sgi(vcpu, p->regval, g1);
 245
 246         return true;
 247 }
 248
 249 static bool access_gic_sre(struct kvm_vcpu *vcpu,
 250                            struct sys_reg_params *p,
 251                            const struct sys_reg_desc *r)
 252 {
 253         if (p->is_write)
 254                 return ignore_write(vcpu, p);
 255
 256         p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
 257         return true;
 258 }
 259
 260 static bool trap_raz_wi(struct kvm_vcpu *vcpu,
 261                         struct sys_reg_params *p,
 262                         const struct sys_reg_desc *r)
 263 {
 264         if (p->is_write)
 265                 return ignore_write(vcpu, p);
 266         else
 267                 return read_zero(vcpu, p);
 268 }
 269
 270 /*
 271  * ARMv8.1 mandates at least a trivial LORegion implementation, where all the
 272  * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0
 273  * system, these registers should UNDEF. LORID_EL1 being a RO register, we
 274  * treat it separately.
 275  */
 276 static bool trap_loregion(struct kvm_vcpu *vcpu,
 277                           struct sys_reg_params *p,
 278                           const struct sys_reg_desc *r)
 279 {
 280         u64 val = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
 281         u32 sr = reg_to_encoding(r);
 282
 283         if (!(val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))) {
 284                 kvm_inject_undefined(vcpu);
 285                 return false;
 286         }
 287
 288         if (p->is_write && sr == SYS_LORID_EL1)
 289                 return write_to_read_only(vcpu, p, r);
 290
 291         return trap_raz_wi(vcpu, p, r);
 292 }
 293
 294 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
 295                            struct sys_reg_params *p,
 296                            const struct sys_reg_desc *r)
 297 {
 298         if (p->is_write) {
 299                 return ignore_write(vcpu, p);
 300         } else {
 301                 p->regval = (1 << 3);
 302                 return true;
 303         }
 304 }
 305
 306 static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
 307                                    struct sys_reg_params *p,
 308                                    const struct sys_reg_desc *r)
 309 {
 310         if (p->is_write) {
 311                 return ignore_write(vcpu, p);
 312         } else {
 313                 p->regval = read_sysreg(dbgauthstatus_el1);
 314                 return true;
 315         }
 316 }
 317
 318 /*
 319  * We want to avoid world-switching all the DBG registers all the
 320  * time:
 321  *
 322  * - If we've touched any debug register, it is likely that we're
 323  *   going to touch more of them. It then makes sense to disable the
 324  *   traps and start doing the save/restore dance
 325  * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
 326  *   then mandatory to save/restore the registers, as the guest
 327  *   depends on them.
 328  *
 329  * For this, we use a DIRTY bit, indicating the guest has modified the
 330  * debug registers, used as follow:
 331  *
 332  * On guest entry:
 333  * - If the dirty bit is set (because we're coming back from trapping),
 334  *   disable the traps, save host registers, restore guest registers.
 335  * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
 336  *   set the dirty bit, disable the traps, save host registers,
 337  *   restore guest registers.
 338  * - Otherwise, enable the traps
 339  *
 340  * On guest exit:
 341  * - If the dirty bit is set, save guest registers, restore host
 342  *   registers and clear the dirty bit. This ensure that the host can
 343  *   now use the debug registers.
 344  */
 345 static bool trap_debug_regs(struct kvm_vcpu *vcpu,
 346                             struct sys_reg_params *p,
 347                             const struct sys_reg_desc *r)
 348 {
 349         if (p->is_write) {
 350                 vcpu_write_sys_reg(vcpu, p->regval, r->reg);
 351                 vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
 352         } else {
 353                 p->regval = vcpu_read_sys_reg(vcpu, r->reg);
 354         }
 355
 356         trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
 357
 358         return true;
 359 }
 360
 361 /*
 362  * reg_to_dbg/dbg_to_reg
 363  *
 364  * A 32 bit write to a debug register leave top bits alone
 365  * A 32 bit read from a debug register only returns the bottom bits
 366  *
 367  * All writes will set the KVM_ARM64_DEBUG_DIRTY flag to ensure the
 368  * hyp.S code switches between host and guest values in future.
 369  */
 370 static void reg_to_dbg(struct kvm_vcpu *vcpu,
 371                        struct sys_reg_params *p,
 372                        const struct sys_reg_desc *rd,
 373                        u64 *dbg_reg)
 374 {
 375         u64 mask, shift, val;
 376
 377         get_access_mask(rd, &mask, &shift);
 378
 379         val = *dbg_reg;
 380         val &= ~mask;
 381         val |= (p->regval & (mask >> shift)) << shift;
 382         *dbg_reg = val;
 383
 384         vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
 385 }
 386
 387 static void dbg_to_reg(struct kvm_vcpu *vcpu,
 388                        struct sys_reg_params *p,
 389                        const struct sys_reg_desc *rd,
 390                        u64 *dbg_reg)
 391 {
 392         u64 mask, shift;
 393
 394         get_access_mask(rd, &mask, &shift);
 395         p->regval = (*dbg_reg & mask) >> shift;
 396 }
 397
 398 static bool trap_bvr(struct kvm_vcpu *vcpu,
 399                      struct sys_reg_params *p,
 400                      const struct sys_reg_desc *rd)
 401 {
 402         u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
 403
 404         if (p->is_write)
 405                 reg_to_dbg(vcpu, p, rd, dbg_reg);
 406         else
 407                 dbg_to_reg(vcpu, p, rd, dbg_reg);
 408
 409         trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
 410
 411         return true;
 412 }
 413
 414 static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
 415                 const struct kvm_one_reg *reg, void __user *uaddr)
 416 {
 417         __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
 418
 419         if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
 420                 return -EFAULT;
 421         return 0;
 422 }
 423
 424 static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
 425         const struct kvm_one_reg *reg, void __user *uaddr)
 426 {
 427         __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
 428
 429         if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
 430                 return -EFAULT;
 431         return 0;
 432 }
 433
 434 static void reset_bvr(struct kvm_vcpu *vcpu,
 435                       const struct sys_reg_desc *rd)
 436 {
 437         vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = rd->val;
 438 }
 439
 440 static bool trap_bcr(struct kvm_vcpu *vcpu,
 441                      struct sys_reg_params *p,
 442                      const struct sys_reg_desc *rd)
 443 {
 444         u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
 445
 446         if (p->is_write)
 447                 reg_to_dbg(vcpu, p, rd, dbg_reg);
 448         else
 449                 dbg_to_reg(vcpu, p, rd, dbg_reg);
 450
 451         trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
 452
 453         return true;
 454 }
 455
 456 static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
 457                 const struct kvm_one_reg *reg, void __user *uaddr)
 458 {
 459         __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
 460
 461         if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
 462                 return -EFAULT;
 463
 464         return 0;
 465 }
 466
 467 static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
 468         const struct kvm_one_reg *reg, void __user *uaddr)
 469 {
 470         __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
 471
 472         if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
 473                 return -EFAULT;
 474         return 0;
 475 }
 476
 477 static void reset_bcr(struct kvm_vcpu *vcpu,
 478                       const struct sys_reg_desc *rd)
 479 {
 480         vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = rd->val;
 481 }
 482
 483 static bool trap_wvr(struct kvm_vcpu *vcpu,
 484                      struct sys_reg_params *p,
 485                      const struct sys_reg_desc *rd)
 486 {
 487         u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
 488
 489         if (p->is_write)
 490                 reg_to_dbg(vcpu, p, rd, dbg_reg);
 491         else
 492                 dbg_to_reg(vcpu, p, rd, dbg_reg);
 493
 494         trace_trap_reg(__func__, rd->CRm, p->is_write,
 495                 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm]);
 496
 497         return true;
 498 }
 499
 500 static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
 501                 const struct kvm_one_reg *reg, void __user *uaddr)
 502 {
 503         __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
 504
 505         if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
 506                 return -EFAULT;
 507         return 0;
 508 }
 509
 510 static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
 511         const struct kvm_one_reg *reg, void __user *uaddr)
 512 {
 513         __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
 514
 515         if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
 516                 return -EFAULT;
 517         return 0;
 518 }
 519
 520 static void reset_wvr(struct kvm_vcpu *vcpu,
 521                       const struct sys_reg_desc *rd)
 522 {
 523         vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = rd->val;
 524 }
 525
 526 static bool trap_wcr(struct kvm_vcpu *vcpu,
 527                      struct sys_reg_params *p,
 528                      const struct sys_reg_desc *rd)
 529 {
 530         u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
 531
 532         if (p->is_write)
 533                 reg_to_dbg(vcpu, p, rd, dbg_reg);
 534         else
 535                 dbg_to_reg(vcpu, p, rd, dbg_reg);
 536
 537         trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
 538
 539         return true;
 540 }
 541
 542 static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
 543                 const struct kvm_one_reg *reg, void __user *uaddr)
 544 {
 545         __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
 546
 547         if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
 548                 return -EFAULT;
 549         return 0;
 550 }
 551
 552 static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
 553         const struct kvm_one_reg *reg, void __user *uaddr)
 554 {
 555         __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
 556
 557         if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
 558                 return -EFAULT;
 559         return 0;
 560 }
 561
 562 static void reset_wcr(struct kvm_vcpu *vcpu,
 563                       const struct sys_reg_desc *rd)
 564 {
 565         vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = rd->val;
 566 }
 567
 568 static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 569 {
 570         u64 amair = read_sysreg(amair_el1);
 571         vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
 572 }
 573
 574 static void reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 575 {
 576         u64 actlr = read_sysreg(actlr_el1);
 577         vcpu_write_sys_reg(vcpu, actlr, ACTLR_EL1);
 578 }
 579
 580 static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 581 {
 582         u64 mpidr;
 583
 584         /*
 585          * Map the vcpu_id into the first three affinity level fields of
 586          * the MPIDR. We limit the number of VCPUs in level 0 due to a
 587          * limitation to 16 CPUs in that level in the ICC_SGIxR registers
 588          * of the GICv3 to be able to address each CPU directly when
 589          * sending IPIs.
 590          */
 591         mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
 592         mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
 593         mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
 594         vcpu_write_sys_reg(vcpu, (1ULL << 31) | mpidr, MPIDR_EL1);
 595 }
 596
 597 static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
 598                                    const struct sys_reg_desc *r)
 599 {
 600         if (kvm_vcpu_has_pmu(vcpu))
 601                 return 0;
 602
 603         return REG_HIDDEN;
 604 }
 605
 606 static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 607 {
 608         u64 pmcr, val;
 609
 610         /* No PMU available, PMCR_EL0 may UNDEF... */
 611         if (!kvm_arm_support_pmu_v3())
 612                 return;
 613
 614         pmcr = read_sysreg(pmcr_el0);
 615         /*
 616          * Writable bits of PMCR_EL0 (ARMV8_PMU_PMCR_MASK) are reset to UNKNOWN
 617          * except PMCR.E resetting to zero.
 618          */
 619         val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
 620                | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
 621         if (!system_supports_32bit_el0())
 622                 val |= ARMV8_PMU_PMCR_LC;
 623         __vcpu_sys_reg(vcpu, r->reg) = val;
 624 }
 625
 626 static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
 627 {
 628         u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
 629         bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
 630
 631         if (!enabled)
 632                 kvm_inject_undefined(vcpu);
 633
 634         return !enabled;
 635 }
 636
 637 static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
 638 {
 639         return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
 640 }
 641
 642 static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
 643 {
 644         return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
 645 }
 646
 647 static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
 648 {
 649         return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_EN);
 650 }
 651
 652 static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
 653 {
 654         return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_EN);
 655 }
 656
 657 static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 658                         const struct sys_reg_desc *r)
 659 {
 660         u64 val;
 661
 662         if (pmu_access_el0_disabled(vcpu))
 663                 return false;
 664
 665         if (p->is_write) {
 666                 /* Only update writeable bits of PMCR */
 667                 val = __vcpu_sys_reg(vcpu, PMCR_EL0);
 668                 val &= ~ARMV8_PMU_PMCR_MASK;
 669                 val |= p->regval & ARMV8_PMU_PMCR_MASK;
 670                 if (!system_supports_32bit_el0())
 671                         val |= ARMV8_PMU_PMCR_LC;
 672                 __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
 673                 kvm_pmu_handle_pmcr(vcpu, val);
 674                 kvm_vcpu_pmu_restore_guest(vcpu);
 675         } else {
 676                 /* PMCR.P & PMCR.C are RAZ */
 677                 val = __vcpu_sys_reg(vcpu, PMCR_EL0)
 678                       & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
 679                 p->regval = val;
 680         }
 681
 682         return true;
 683 }
 684
 685 static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 686                           const struct sys_reg_desc *r)
 687 {
 688         if (pmu_access_event_counter_el0_disabled(vcpu))
 689                 return false;
 690
 691         if (p->is_write)
 692                 __vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
 693         else
 694                 /* return PMSELR.SEL field */
 695                 p->regval = __vcpu_sys_reg(vcpu, PMSELR_EL0)
 696                             & ARMV8_PMU_COUNTER_MASK;
 697
 698         return true;
 699 }
 700
 701 static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 702                           const struct sys_reg_desc *r)
 703 {
 704         u64 pmceid, mask, shift;
 705
 706         BUG_ON(p->is_write);
 707
 708         if (pmu_access_el0_disabled(vcpu))
 709                 return false;
 710
 711         get_access_mask(r, &mask, &shift);
 712
 713         pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
 714         pmceid &= mask;
 715         pmceid >>= shift;
 716
 717         p->regval = pmceid;
 718
 719         return true;
 720 }
 721
 722 static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
 723 {
 724         u64 pmcr, val;
 725
 726         pmcr = __vcpu_sys_reg(vcpu, PMCR_EL0);
 727         val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
 728         if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
 729                 kvm_inject_undefined(vcpu);
 730                 return false;
 731         }
 732
 733         return true;
 734 }
 735
 736 static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
 737                               struct sys_reg_params *p,
 738                               const struct sys_reg_desc *r)
 739 {
 740         u64 idx = ~0UL;
 741
 742         if (r->CRn == 9 && r->CRm == 13) {
 743                 if (r->Op2 == 2) {
 744                         /* PMXEVCNTR_EL0 */
 745                         if (pmu_access_event_counter_el0_disabled(vcpu))
 746                                 return false;
 747
 748                         idx = __vcpu_sys_reg(vcpu, PMSELR_EL0)
 749                               & ARMV8_PMU_COUNTER_MASK;
 750                 } else if (r->Op2 == 0) {
 751                         /* PMCCNTR_EL0 */
 752                         if (pmu_access_cycle_counter_el0_disabled(vcpu))
 753                                 return false;
 754
 755                         idx = ARMV8_PMU_CYCLE_IDX;
 756                 }
 757         } else if (r->CRn == 0 && r->CRm == 9) {
 758                 /* PMCCNTR */
 759                 if (pmu_access_event_counter_el0_disabled(vcpu))
 760                         return false;
 761
 762                 idx = ARMV8_PMU_CYCLE_IDX;
 763         } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
 764                 /* PMEVCNTRn_EL0 */
 765                 if (pmu_access_event_counter_el0_disabled(vcpu))
 766                         return false;
 767
 768                 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
 769         }
 770
 771         /* Catch any decoding mistake */
 772         WARN_ON(idx == ~0UL);
 773
 774         if (!pmu_counter_idx_valid(vcpu, idx))
 775                 return false;
 776
 777         if (p->is_write) {
 778                 if (pmu_access_el0_disabled(vcpu))
 779                         return false;
 780
 781                 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
 782         } else {
 783                 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
 784         }
 785
 786         return true;
 787 }
 788
 789 static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 790                                const struct sys_reg_desc *r)
 791 {
 792         u64 idx, reg;
 793
 794         if (pmu_access_el0_disabled(vcpu))
 795                 return false;
 796
 797         if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
 798                 /* PMXEVTYPER_EL0 */
 799                 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
 800                 reg = PMEVTYPER0_EL0 + idx;
 801         } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
 802                 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
 803                 if (idx == ARMV8_PMU_CYCLE_IDX)
 804                         reg = PMCCFILTR_EL0;
 805                 else
 806                         /* PMEVTYPERn_EL0 */
 807                         reg = PMEVTYPER0_EL0 + idx;
 808         } else {
 809                 BUG();
 810         }
 811
 812         if (!pmu_counter_idx_valid(vcpu, idx))
 813                 return false;
 814
 815         if (p->is_write) {
 816                 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
 817                 __vcpu_sys_reg(vcpu, reg) = p->regval & ARMV8_PMU_EVTYPE_MASK;
 818                 kvm_vcpu_pmu_restore_guest(vcpu);
 819         } else {
 820                 p->regval = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_MASK;
 821         }
 822
 823         return true;
 824 }
 825
 826 static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 827                            const struct sys_reg_desc *r)
 828 {
 829         u64 val, mask;
 830
 831         if (pmu_access_el0_disabled(vcpu))
 832                 return false;
 833
 834         mask = kvm_pmu_valid_counter_mask(vcpu);
 835         if (p->is_write) {
 836                 val = p->regval & mask;
 837                 if (r->Op2 & 0x1) {
 838                         /* accessing PMCNTENSET_EL0 */
 839                         __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
 840                         kvm_pmu_enable_counter_mask(vcpu, val);
 841                         kvm_vcpu_pmu_restore_guest(vcpu);
 842                 } else {
 843                         /* accessing PMCNTENCLR_EL0 */
 844                         __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
 845                         kvm_pmu_disable_counter_mask(vcpu, val);
 846                 }
 847         } else {
 848                 p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask;
 849         }
 850
 851         return true;
 852 }
 853
 854 static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 855                            const struct sys_reg_desc *r)
 856 {
 857         u64 mask = kvm_pmu_valid_counter_mask(vcpu);
 858
 859         if (check_pmu_access_disabled(vcpu, 0))
 860                 return false;
 861
 862         if (p->is_write) {
 863                 u64 val = p->regval & mask;
 864
 865                 if (r->Op2 & 0x1)
 866                         /* accessing PMINTENSET_EL1 */
 867                         __vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
 868                 else
 869                         /* accessing PMINTENCLR_EL1 */
 870                         __vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
 871         } else {
 872                 p->regval = __vcpu_sys_reg(vcpu, PMINTENSET_EL1) & mask;
 873         }
 874
 875         return true;
 876 }
 877
 878 static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 879                          const struct sys_reg_desc *r)
 880 {
 881         u64 mask = kvm_pmu_valid_counter_mask(vcpu);
 882
 883         if (pmu_access_el0_disabled(vcpu))
 884                 return false;
 885
 886         if (p->is_write) {
 887                 if (r->CRm & 0x2)
 888                         /* accessing PMOVSSET_EL0 */
 889                         __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= (p->regval & mask);
 890                 else
 891                         /* accessing PMOVSCLR_EL0 */
 892                         __vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
 893         } else {
 894                 p->regval = __vcpu_sys_reg(vcpu, PMOVSSET_EL0) & mask;
 895         }
 896
 897         return true;
 898 }
 899
 900 static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 901                            const struct sys_reg_desc *r)
 902 {
 903         u64 mask;
 904
 905         if (!p->is_write)
 906                 return read_from_write_only(vcpu, p, r);
 907
 908         if (pmu_write_swinc_el0_disabled(vcpu))
 909                 return false;
 910
 911         mask = kvm_pmu_valid_counter_mask(vcpu);
 912         kvm_pmu_software_increment(vcpu, p->regval & mask);
 913         return true;
 914 }
 915
 916 static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 917                              const struct sys_reg_desc *r)
 918 {
 919         if (p->is_write) {
 920                 if (!vcpu_mode_priv(vcpu)) {
 921                         kvm_inject_undefined(vcpu);
 922                         return false;
 923                 }
 924
 925                 __vcpu_sys_reg(vcpu, PMUSERENR_EL0) =
 926                                p->regval & ARMV8_PMU_USERENR_MASK;
 927         } else {
 928                 p->regval = __vcpu_sys_reg(vcpu, PMUSERENR_EL0)
 929                             & ARMV8_PMU_USERENR_MASK;
 930         }
 931
 932         return true;
 933 }
 934
 935 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 936 #define DBG_BCR_BVR_WCR_WVR_EL1(n)                                      \
 937         { SYS_DESC(SYS_DBGBVRn_EL1(n)),                                 \
 938           trap_bvr, reset_bvr, 0, 0, get_bvr, set_bvr },                \
 939         { SYS_DESC(SYS_DBGBCRn_EL1(n)),                                 \
 940           trap_bcr, reset_bcr, 0, 0, get_bcr, set_bcr },                \
 941         { SYS_DESC(SYS_DBGWVRn_EL1(n)),                                 \
 942           trap_wvr, reset_wvr, 0, 0,  get_wvr, set_wvr },               \
 943         { SYS_DESC(SYS_DBGWCRn_EL1(n)),                                 \
 944           trap_wcr, reset_wcr, 0, 0,  get_wcr, set_wcr }
 945
 946 #define PMU_SYS_REG(r)                                          \
 947         SYS_DESC(r), .reset = reset_unknown, .visibility = pmu_visibility
 948
 949 /* Macro to expand the PMEVCNTRn_EL0 register */
 950 #define PMU_PMEVCNTR_EL0(n)                                             \
 951         { PMU_SYS_REG(SYS_PMEVCNTRn_EL0(n)),                            \
 952           .access = access_pmu_evcntr, .reg = (PMEVCNTR0_EL0 + n), }
 953
 954 /* Macro to expand the PMEVTYPERn_EL0 register */
 955 #define PMU_PMEVTYPER_EL0(n)                                            \
 956         { PMU_SYS_REG(SYS_PMEVTYPERn_EL0(n)),                           \
 957           .access = access_pmu_evtyper, .reg = (PMEVTYPER0_EL0 + n), }
 958
 959 static bool undef_access(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 960                          const struct sys_reg_desc *r)
 961 {
 962         kvm_inject_undefined(vcpu);
 963
 964         return false;
 965 }
 966
 967 /* Macro to expand the AMU counter and type registers*/
 968 #define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), undef_access }
 969 #define AMU_AMEVTYPER0_EL0(n) { SYS_DESC(SYS_AMEVTYPER0_EL0(n)), undef_access }
 970 #define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), undef_access }
 971 #define AMU_AMEVTYPER1_EL0(n) { SYS_DESC(SYS_AMEVTYPER1_EL0(n)), undef_access }
 972
 973 static unsigned int ptrauth_visibility(const struct kvm_vcpu *vcpu,
 974                         const struct sys_reg_desc *rd)
 975 {
 976         return vcpu_has_ptrauth(vcpu) ? 0 : REG_HIDDEN;
 977 }
 978
 979 /*
 980  * If we land here on a PtrAuth access, that is because we didn't
 981  * fixup the access on exit by allowing the PtrAuth sysregs. The only
 982  * way this happens is when the guest does not have PtrAuth support
 983  * enabled.
 984  */
 985 #define __PTRAUTH_KEY(k)                                                \
 986         { SYS_DESC(SYS_## k), undef_access, reset_unknown, k,           \
 987         .visibility = ptrauth_visibility}
 988
 989 #define PTRAUTH_KEY(k)                                                  \
 990         __PTRAUTH_KEY(k ## KEYLO_EL1),                                  \
 991         __PTRAUTH_KEY(k ## KEYHI_EL1)
 992
 993 static bool access_arch_timer(struct kvm_vcpu *vcpu,
 994                               struct sys_reg_params *p,
 995                               const struct sys_reg_desc *r)
 996 {
 997         enum kvm_arch_timers tmr;
 998         enum kvm_arch_timer_regs treg;
 999         u64 reg = reg_to_encoding(r);
1000
1001         switch (reg) {
1002         case SYS_CNTP_TVAL_EL0:
1003         case SYS_AARCH32_CNTP_TVAL:
1004                 tmr = TIMER_PTIMER;
1005                 treg = TIMER_REG_TVAL;
1006                 break;
1007         case SYS_CNTP_CTL_EL0:
1008         case SYS_AARCH32_CNTP_CTL:
1009                 tmr = TIMER_PTIMER;
1010                 treg = TIMER_REG_CTL;
1011                 break;
1012         case SYS_CNTP_CVAL_EL0:
1013         case SYS_AARCH32_CNTP_CVAL:
1014                 tmr = TIMER_PTIMER;
1015                 treg = TIMER_REG_CVAL;
1016                 break;
1017         default:
1018                 BUG();
1019         }
1020
1021         if (p->is_write)
1022                 kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval);
1023         else
1024                 p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg);
1025
1026         return true;
1027 }
1028
1029 #define FEATURE(x)      (GENMASK_ULL(x##_SHIFT + 3, x##_SHIFT))
1030
1031 /* Read a sanitised cpufeature ID register by sys_reg_desc */
1032 static u64 read_id_reg(const struct kvm_vcpu *vcpu,
1033                 struct sys_reg_desc const *r, bool raz)
1034 {
1035         u32 id = reg_to_encoding(r);
1036         u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
1037
1038         switch (id) {
1039         case SYS_ID_AA64PFR0_EL1:
1040                 if (!vcpu_has_sve(vcpu))
1041                         val &= ~FEATURE(ID_AA64PFR0_SVE);
1042                 val &= ~FEATURE(ID_AA64PFR0_AMU);
1043                 val &= ~FEATURE(ID_AA64PFR0_CSV2);
1044                 val |= FIELD_PREP(FEATURE(ID_AA64PFR0_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2);
1045                 val &= ~FEATURE(ID_AA64PFR0_CSV3);
1046                 val |= FIELD_PREP(FEATURE(ID_AA64PFR0_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3);
1047                 break;
1048         case SYS_ID_AA64PFR1_EL1:
1049                 val &= ~FEATURE(ID_AA64PFR1_MTE);
1050                 if (kvm_has_mte(vcpu->kvm)) {
1051                         u64 pfr, mte;
1052
1053                         pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
1054                         mte = cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR1_MTE_SHIFT);
1055                         val |= FIELD_PREP(FEATURE(ID_AA64PFR1_MTE), mte);
1056                 }
1057                 break;
1058         case SYS_ID_AA64ISAR1_EL1:
1059                 if (!vcpu_has_ptrauth(vcpu))
1060                         val &= ~(FEATURE(ID_AA64ISAR1_APA) |
1061                                  FEATURE(ID_AA64ISAR1_API) |
1062                                  FEATURE(ID_AA64ISAR1_GPA) |
1063                                  FEATURE(ID_AA64ISAR1_GPI));
1064                 break;
1065         case SYS_ID_AA64DFR0_EL1:
1066                 /* Limit debug to ARMv8.0 */
1067                 val &= ~FEATURE(ID_AA64DFR0_DEBUGVER);
1068                 val |= FIELD_PREP(FEATURE(ID_AA64DFR0_DEBUGVER), 6);
1069                 /* Limit guests to PMUv3 for ARMv8.4 */
1070                 val = cpuid_feature_cap_perfmon_field(val,
1071                                                       ID_AA64DFR0_PMUVER_SHIFT,
1072                                                       kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_PMUVER_8_4 : 0);
1073                 /* Hide SPE from guests */
1074                 val &= ~FEATURE(ID_AA64DFR0_PMSVER);
1075                 break;
1076         case SYS_ID_DFR0_EL1:
1077                 /* Limit guests to PMUv3 for ARMv8.4 */
1078                 val = cpuid_feature_cap_perfmon_field(val,
1079                                                       ID_DFR0_PERFMON_SHIFT,
1080                                                       kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_PERFMON_8_4 : 0);
1081                 break;
1082         }
1083
1084         return val;
1085 }
1086
1087 static unsigned int id_visibility(const struct kvm_vcpu *vcpu,
1088                                   const struct sys_reg_desc *r)
1089 {
1090         u32 id = reg_to_encoding(r);
1091
1092         switch (id) {
1093         case SYS_ID_AA64ZFR0_EL1:
1094                 if (!vcpu_has_sve(vcpu))
1095                         return REG_RAZ;
1096                 break;
1097         }
1098
1099         return 0;
1100 }
1101
1102 /* cpufeature ID register access trap handlers */
1103
1104 static bool __access_id_reg(struct kvm_vcpu *vcpu,
1105                             struct sys_reg_params *p,
1106                             const struct sys_reg_desc *r,
1107                             bool raz)
1108 {
1109         if (p->is_write)
1110                 return write_to_read_only(vcpu, p, r);
1111
1112         p->regval = read_id_reg(vcpu, r, raz);
1113         return true;
1114 }
1115
1116 static bool access_id_reg(struct kvm_vcpu *vcpu,
1117                           struct sys_reg_params *p,
1118                           const struct sys_reg_desc *r)
1119 {
1120         bool raz = sysreg_visible_as_raz(vcpu, r);
1121
1122         return __access_id_reg(vcpu, p, r, raz);
1123 }
1124
1125 static bool access_raz_id_reg(struct kvm_vcpu *vcpu,
1126                               struct sys_reg_params *p,
1127                               const struct sys_reg_desc *r)
1128 {
1129         return __access_id_reg(vcpu, p, r, true);
1130 }
1131
1132 static int reg_from_user(u64 *val, const void __user *uaddr, u64 id);
1133 static int reg_to_user(void __user *uaddr, const u64 *val, u64 id);
1134 static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
1135
1136 /* Visibility overrides for SVE-specific control registers */
1137 static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
1138                                    const struct sys_reg_desc *rd)
1139 {
1140         if (vcpu_has_sve(vcpu))
1141                 return 0;
1142
1143         return REG_HIDDEN;
1144 }
1145
1146 static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
1147                                const struct sys_reg_desc *rd,
1148                                const struct kvm_one_reg *reg, void __user *uaddr)
1149 {
1150         const u64 id = sys_reg_to_index(rd);
1151         u8 csv2, csv3;
1152         int err;
1153         u64 val;
1154
1155         err = reg_from_user(&val, uaddr, id);
1156         if (err)
1157                 return err;
1158
1159         /*
1160          * Allow AA64PFR0_EL1.CSV2 to be set from userspace as long as
1161          * it doesn't promise more than what is actually provided (the
1162          * guest could otherwise be covered in ectoplasmic residue).
1163          */
1164         csv2 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV2_SHIFT);
1165         if (csv2 > 1 ||
1166             (csv2 && arm64_get_spectre_v2_state() != SPECTRE_UNAFFECTED))
1167                 return -EINVAL;
1168
1169         /* Same thing for CSV3 */
1170         csv3 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV3_SHIFT);
1171         if (csv3 > 1 ||
1172             (csv3 && arm64_get_meltdown_state() != SPECTRE_UNAFFECTED))
1173                 return -EINVAL;
1174
1175         /* We can only differ with CSV[23], and anything else is an error */
1176         val ^= read_id_reg(vcpu, rd, false);
1177         val &= ~((0xFUL << ID_AA64PFR0_CSV2_SHIFT) |
1178                  (0xFUL << ID_AA64PFR0_CSV3_SHIFT));
1179         if (val)
1180                 return -EINVAL;
1181
1182         vcpu->kvm->arch.pfr0_csv2 = csv2;
1183         vcpu->kvm->arch.pfr0_csv3 = csv3 ;
1184
1185         return 0;
1186 }
1187
1188 /*
1189  * cpufeature ID register user accessors
1190  *
1191  * For now, these registers are immutable for userspace, so no values
1192  * are stored, and for set_id_reg() we don't allow the effective value
1193  * to be changed.
1194  */
1195 static int __get_id_reg(const struct kvm_vcpu *vcpu,
1196                         const struct sys_reg_desc *rd, void __user *uaddr,
1197                         bool raz)
1198 {
1199         const u64 id = sys_reg_to_index(rd);
1200         const u64 val = read_id_reg(vcpu, rd, raz);
1201
1202         return reg_to_user(uaddr, &val, id);
1203 }
1204
1205 static int __set_id_reg(const struct kvm_vcpu *vcpu,
1206                         const struct sys_reg_desc *rd, void __user *uaddr,
1207                         bool raz)
1208 {
1209         const u64 id = sys_reg_to_index(rd);
1210         int err;
1211         u64 val;
1212
1213         err = reg_from_user(&val, uaddr, id);
1214         if (err)
1215                 return err;
1216
1217         /* This is what we mean by invariant: you can't change it. */
1218         if (val != read_id_reg(vcpu, rd, raz))
1219                 return -EINVAL;
1220
1221         return 0;
1222 }
1223
1224 static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1225                       const struct kvm_one_reg *reg, void __user *uaddr)
1226 {
1227         bool raz = sysreg_visible_as_raz(vcpu, rd);
1228
1229         return __get_id_reg(vcpu, rd, uaddr, raz);
1230 }
1231
1232 static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1233                       const struct kvm_one_reg *reg, void __user *uaddr)
1234 {
1235         bool raz = sysreg_visible_as_raz(vcpu, rd);
1236
1237         return __set_id_reg(vcpu, rd, uaddr, raz);
1238 }
1239
1240 static int get_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1241                           const struct kvm_one_reg *reg, void __user *uaddr)
1242 {
1243         return __get_id_reg(vcpu, rd, uaddr, true);
1244 }
1245
1246 static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1247                           const struct kvm_one_reg *reg, void __user *uaddr)
1248 {
1249         return __set_id_reg(vcpu, rd, uaddr, true);
1250 }
1251
1252 static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1253                        const struct sys_reg_desc *r)
1254 {
1255         if (p->is_write)
1256                 return write_to_read_only(vcpu, p, r);
1257
1258         p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0);
1259         return true;
1260 }
1261
1262 static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1263                          const struct sys_reg_desc *r)
1264 {
1265         if (p->is_write)
1266                 return write_to_read_only(vcpu, p, r);
1267
1268         p->regval = read_sysreg(clidr_el1);
1269         return true;
1270 }
1271
1272 static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1273                           const struct sys_reg_desc *r)
1274 {
1275         int reg = r->reg;
1276
1277         if (p->is_write)
1278                 vcpu_write_sys_reg(vcpu, p->regval, reg);
1279         else
1280                 p->regval = vcpu_read_sys_reg(vcpu, reg);
1281         return true;
1282 }
1283
1284 static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1285                           const struct sys_reg_desc *r)
1286 {
1287         u32 csselr;
1288
1289         if (p->is_write)
1290                 return write_to_read_only(vcpu, p, r);
1291
1292         csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1);
1293         p->regval = get_ccsidr(csselr);
1294
1295         /*
1296          * Guests should not be doing cache operations by set/way at all, and
1297          * for this reason, we trap them and attempt to infer the intent, so
1298          * that we can flush the entire guest's address space at the appropriate
1299          * time.
1300          * To prevent this trapping from causing performance problems, let's
1301          * expose the geometry of all data and unified caches (which are
1302          * guaranteed to be PIPT and thus non-aliasing) as 1 set and 1 way.
1303          * [If guests should attempt to infer aliasing properties from the
1304          * geometry (which is not permitted by the architecture), they would
1305          * only do so for virtually indexed caches.]
1306          */
1307         if (!(csselr & 1)) // data or unified cache
1308                 p->regval &= ~GENMASK(27, 3);
1309         return true;
1310 }
1311
1312 static unsigned int mte_visibility(const struct kvm_vcpu *vcpu,
1313                                    const struct sys_reg_desc *rd)
1314 {
1315         if (kvm_has_mte(vcpu->kvm))
1316                 return 0;
1317
1318         return REG_HIDDEN;
1319 }
1320
1321 #define MTE_REG(name) {                         \
1322         SYS_DESC(SYS_##name),                   \
1323         .access = undef_access,                 \
1324         .reset = reset_unknown,                 \
1325         .reg = name,                            \
1326         .visibility = mte_visibility,           \
1327 }
1328
1329 /* sys_reg_desc initialiser for known cpufeature ID registers */
1330 #define ID_SANITISED(name) {                    \
1331         SYS_DESC(SYS_##name),                   \
1332         .access = access_id_reg,                \
1333         .get_user = get_id_reg,                 \
1334         .set_user = set_id_reg,                 \
1335         .visibility = id_visibility,            \
1336 }
1337
1338 /*
1339  * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
1340  * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
1341  * (1 <= crm < 8, 0 <= Op2 < 8).
1342  */
1343 #define ID_UNALLOCATED(crm, op2) {                      \
1344         Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2),     \
1345         .access = access_raz_id_reg,                    \
1346         .get_user = get_raz_id_reg,                     \
1347         .set_user = set_raz_id_reg,                     \
1348 }
1349
1350 /*
1351  * sys_reg_desc initialiser for known ID registers that we hide from guests.
1352  * For now, these are exposed just like unallocated ID regs: they appear
1353  * RAZ for the guest.
1354  */
1355 #define ID_HIDDEN(name) {                       \
1356         SYS_DESC(SYS_##name),                   \
1357         .access = access_raz_id_reg,            \
1358         .get_user = get_raz_id_reg,             \
1359         .set_user = set_raz_id_reg,             \
1360 }
1361
1362 /*
1363  * Architected system registers.
1364  * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
1365  *
1366  * Debug handling: We do trap most, if not all debug related system
1367  * registers. The implementation is good enough to ensure that a guest
1368  * can use these with minimal performance degradation. The drawback is
1369  * that we don't implement any of the external debug, none of the
1370  * OSlock protocol. This should be revisited if we ever encounter a
1371  * more demanding guest...
1372  */
1373 static const struct sys_reg_desc sys_reg_descs[] = {
1374         { SYS_DESC(SYS_DC_ISW), access_dcsw },
1375         { SYS_DESC(SYS_DC_CSW), access_dcsw },
1376         { SYS_DESC(SYS_DC_CISW), access_dcsw },
1377
1378         DBG_BCR_BVR_WCR_WVR_EL1(0),
1379         DBG_BCR_BVR_WCR_WVR_EL1(1),
1380         { SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
1381         { SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
1382         DBG_BCR_BVR_WCR_WVR_EL1(2),
1383         DBG_BCR_BVR_WCR_WVR_EL1(3),
1384         DBG_BCR_BVR_WCR_WVR_EL1(4),
1385         DBG_BCR_BVR_WCR_WVR_EL1(5),
1386         DBG_BCR_BVR_WCR_WVR_EL1(6),
1387         DBG_BCR_BVR_WCR_WVR_EL1(7),
1388         DBG_BCR_BVR_WCR_WVR_EL1(8),
1389         DBG_BCR_BVR_WCR_WVR_EL1(9),
1390         DBG_BCR_BVR_WCR_WVR_EL1(10),
1391         DBG_BCR_BVR_WCR_WVR_EL1(11),
1392         DBG_BCR_BVR_WCR_WVR_EL1(12),
1393         DBG_BCR_BVR_WCR_WVR_EL1(13),
1394         DBG_BCR_BVR_WCR_WVR_EL1(14),
1395         DBG_BCR_BVR_WCR_WVR_EL1(15),
1396
1397         { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
1398         { SYS_DESC(SYS_OSLAR_EL1), trap_raz_wi },
1399         { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1 },
1400         { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
1401         { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
1402         { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
1403         { SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
1404         { SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
1405
1406         { SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
1407         { SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
1408         // DBGDTR[TR]X_EL0 share the same encoding
1409         { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
1410
1411         { SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
1412
1413         { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
1414
1415         /*
1416          * ID regs: all ID_SANITISED() entries here must have corresponding
1417          * entries in arm64_ftr_regs[].
1418          */
1419
1420         /* AArch64 mappings of the AArch32 ID registers */
1421         /* CRm=1 */
1422         ID_SANITISED(ID_PFR0_EL1),
1423         ID_SANITISED(ID_PFR1_EL1),
1424         ID_SANITISED(ID_DFR0_EL1),
1425         ID_HIDDEN(ID_AFR0_EL1),
1426         ID_SANITISED(ID_MMFR0_EL1),
1427         ID_SANITISED(ID_MMFR1_EL1),
1428         ID_SANITISED(ID_MMFR2_EL1),
1429         ID_SANITISED(ID_MMFR3_EL1),
1430
1431         /* CRm=2 */
1432         ID_SANITISED(ID_ISAR0_EL1),
1433         ID_SANITISED(ID_ISAR1_EL1),
1434         ID_SANITISED(ID_ISAR2_EL1),
1435         ID_SANITISED(ID_ISAR3_EL1),
1436         ID_SANITISED(ID_ISAR4_EL1),
1437         ID_SANITISED(ID_ISAR5_EL1),
1438         ID_SANITISED(ID_MMFR4_EL1),
1439         ID_SANITISED(ID_ISAR6_EL1),
1440
1441         /* CRm=3 */
1442         ID_SANITISED(MVFR0_EL1),
1443         ID_SANITISED(MVFR1_EL1),
1444         ID_SANITISED(MVFR2_EL1),
1445         ID_UNALLOCATED(3,3),
1446         ID_SANITISED(ID_PFR2_EL1),
1447         ID_HIDDEN(ID_DFR1_EL1),
1448         ID_SANITISED(ID_MMFR5_EL1),
1449         ID_UNALLOCATED(3,7),
1450
1451         /* AArch64 ID registers */
1452         /* CRm=4 */
1453         { SYS_DESC(SYS_ID_AA64PFR0_EL1), .access = access_id_reg,
1454           .get_user = get_id_reg, .set_user = set_id_aa64pfr0_el1, },
1455         ID_SANITISED(ID_AA64PFR1_EL1),
1456         ID_UNALLOCATED(4,2),
1457         ID_UNALLOCATED(4,3),
1458         ID_SANITISED(ID_AA64ZFR0_EL1),
1459         ID_UNALLOCATED(4,5),
1460         ID_UNALLOCATED(4,6),
1461         ID_UNALLOCATED(4,7),
1462
1463         /* CRm=5 */
1464         ID_SANITISED(ID_AA64DFR0_EL1),
1465         ID_SANITISED(ID_AA64DFR1_EL1),
1466         ID_UNALLOCATED(5,2),
1467         ID_UNALLOCATED(5,3),
1468         ID_HIDDEN(ID_AA64AFR0_EL1),
1469         ID_HIDDEN(ID_AA64AFR1_EL1),
1470         ID_UNALLOCATED(5,6),
1471         ID_UNALLOCATED(5,7),
1472
1473         /* CRm=6 */
1474         ID_SANITISED(ID_AA64ISAR0_EL1),
1475         ID_SANITISED(ID_AA64ISAR1_EL1),
1476         ID_UNALLOCATED(6,2),
1477         ID_UNALLOCATED(6,3),
1478         ID_UNALLOCATED(6,4),
1479         ID_UNALLOCATED(6,5),
1480         ID_UNALLOCATED(6,6),
1481         ID_UNALLOCATED(6,7),
1482
1483         /* CRm=7 */
1484         ID_SANITISED(ID_AA64MMFR0_EL1),
1485         ID_SANITISED(ID_AA64MMFR1_EL1),
1486         ID_SANITISED(ID_AA64MMFR2_EL1),
1487         ID_UNALLOCATED(7,3),
1488         ID_UNALLOCATED(7,4),
1489         ID_UNALLOCATED(7,5),
1490         ID_UNALLOCATED(7,6),
1491         ID_UNALLOCATED(7,7),
1492
1493         { SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
1494         { SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
1495         { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
1496
1497         MTE_REG(RGSR_EL1),
1498         MTE_REG(GCR_EL1),
1499
1500         { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
1501         { SYS_DESC(SYS_TRFCR_EL1), undef_access },
1502         { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
1503         { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
1504         { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
1505
1506         PTRAUTH_KEY(APIA),
1507         PTRAUTH_KEY(APIB),
1508         PTRAUTH_KEY(APDA),
1509         PTRAUTH_KEY(APDB),
1510         PTRAUTH_KEY(APGA),
1511
1512         { SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
1513         { SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
1514         { SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
1515
1516         { SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
1517         { SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
1518         { SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
1519         { SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
1520         { SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
1521         { SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
1522         { SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
1523         { SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
1524
1525         MTE_REG(TFSR_EL1),
1526         MTE_REG(TFSRE0_EL1),
1527
1528         { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
1529         { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
1530
1531         { SYS_DESC(SYS_PMSCR_EL1), undef_access },
1532         { SYS_DESC(SYS_PMSNEVFR_EL1), undef_access },
1533         { SYS_DESC(SYS_PMSICR_EL1), undef_access },
1534         { SYS_DESC(SYS_PMSIRR_EL1), undef_access },
1535         { SYS_DESC(SYS_PMSFCR_EL1), undef_access },
1536         { SYS_DESC(SYS_PMSEVFR_EL1), undef_access },
1537         { SYS_DESC(SYS_PMSLATFR_EL1), undef_access },
1538         { SYS_DESC(SYS_PMSIDR_EL1), undef_access },
1539         { SYS_DESC(SYS_PMBLIMITR_EL1), undef_access },
1540         { SYS_DESC(SYS_PMBPTR_EL1), undef_access },
1541         { SYS_DESC(SYS_PMBSR_EL1), undef_access },
1542         /* PMBIDR_EL1 is not trapped */
1543
1544         { PMU_SYS_REG(SYS_PMINTENSET_EL1),
1545           .access = access_pminten, .reg = PMINTENSET_EL1 },
1546         { PMU_SYS_REG(SYS_PMINTENCLR_EL1),
1547           .access = access_pminten, .reg = PMINTENSET_EL1 },
1548         { SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi },
1549
1550         { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
1551         { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
1552
1553         { SYS_DESC(SYS_LORSA_EL1), trap_loregion },
1554         { SYS_DESC(SYS_LOREA_EL1), trap_loregion },
1555         { SYS_DESC(SYS_LORN_EL1), trap_loregion },
1556         { SYS_DESC(SYS_LORC_EL1), trap_loregion },
1557         { SYS_DESC(SYS_LORID_EL1), trap_loregion },
1558
1559         { SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 },
1560         { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
1561
1562         { SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
1563         { SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
1564         { SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
1565         { SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
1566         { SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
1567         { SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
1568         { SYS_DESC(SYS_ICC_ASGI1R_EL1), access_gic_sgi },
1569         { SYS_DESC(SYS_ICC_SGI0R_EL1), access_gic_sgi },
1570         { SYS_DESC(SYS_ICC_IAR1_EL1), write_to_read_only },
1571         { SYS_DESC(SYS_ICC_EOIR1_EL1), read_from_write_only },
1572         { SYS_DESC(SYS_ICC_HPPIR1_EL1), write_to_read_only },
1573         { SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
1574
1575         { SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
1576         { SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
1577
1578         { SYS_DESC(SYS_SCXTNUM_EL1), undef_access },
1579
1580         { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
1581
1582         { SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr },
1583         { SYS_DESC(SYS_CLIDR_EL1), access_clidr },
1584         { SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 },
1585         { SYS_DESC(SYS_CTR_EL0), access_ctr },
1586
1587         { PMU_SYS_REG(SYS_PMCR_EL0), .access = access_pmcr,
1588           .reset = reset_pmcr, .reg = PMCR_EL0 },
1589         { PMU_SYS_REG(SYS_PMCNTENSET_EL0),
1590           .access = access_pmcnten, .reg = PMCNTENSET_EL0 },
1591         { PMU_SYS_REG(SYS_PMCNTENCLR_EL0),
1592           .access = access_pmcnten, .reg = PMCNTENSET_EL0 },
1593         { PMU_SYS_REG(SYS_PMOVSCLR_EL0),
1594           .access = access_pmovs, .reg = PMOVSSET_EL0 },
1595         { PMU_SYS_REG(SYS_PMSWINC_EL0),
1596           .access = access_pmswinc, .reg = PMSWINC_EL0 },
1597         { PMU_SYS_REG(SYS_PMSELR_EL0),
1598           .access = access_pmselr, .reg = PMSELR_EL0 },
1599         { PMU_SYS_REG(SYS_PMCEID0_EL0),
1600           .access = access_pmceid, .reset = NULL },
1601         { PMU_SYS_REG(SYS_PMCEID1_EL0),
1602           .access = access_pmceid, .reset = NULL },
1603         { PMU_SYS_REG(SYS_PMCCNTR_EL0),
1604           .access = access_pmu_evcntr, .reg = PMCCNTR_EL0 },
1605         { PMU_SYS_REG(SYS_PMXEVTYPER_EL0),
1606           .access = access_pmu_evtyper, .reset = NULL },
1607         { PMU_SYS_REG(SYS_PMXEVCNTR_EL0),
1608           .access = access_pmu_evcntr, .reset = NULL },
1609         /*
1610          * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
1611          * in 32bit mode. Here we choose to reset it as zero for consistency.
1612          */
1613         { PMU_SYS_REG(SYS_PMUSERENR_EL0), .access = access_pmuserenr,
1614           .reset = reset_val, .reg = PMUSERENR_EL0, .val = 0 },
1615         { PMU_SYS_REG(SYS_PMOVSSET_EL0),
1616           .access = access_pmovs, .reg = PMOVSSET_EL0 },
1617
1618         { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
1619         { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
1620
1621         { SYS_DESC(SYS_SCXTNUM_EL0), undef_access },
1622
1623         { SYS_DESC(SYS_AMCR_EL0), undef_access },
1624         { SYS_DESC(SYS_AMCFGR_EL0), undef_access },
1625         { SYS_DESC(SYS_AMCGCR_EL0), undef_access },
1626         { SYS_DESC(SYS_AMUSERENR_EL0), undef_access },
1627         { SYS_DESC(SYS_AMCNTENCLR0_EL0), undef_access },
1628         { SYS_DESC(SYS_AMCNTENSET0_EL0), undef_access },
1629         { SYS_DESC(SYS_AMCNTENCLR1_EL0), undef_access },
1630         { SYS_DESC(SYS_AMCNTENSET1_EL0), undef_access },
1631         AMU_AMEVCNTR0_EL0(0),
1632         AMU_AMEVCNTR0_EL0(1),
1633         AMU_AMEVCNTR0_EL0(2),
1634         AMU_AMEVCNTR0_EL0(3),
1635         AMU_AMEVCNTR0_EL0(4),
1636         AMU_AMEVCNTR0_EL0(5),
1637         AMU_AMEVCNTR0_EL0(6),
1638         AMU_AMEVCNTR0_EL0(7),
1639         AMU_AMEVCNTR0_EL0(8),
1640         AMU_AMEVCNTR0_EL0(9),
1641         AMU_AMEVCNTR0_EL0(10),
1642         AMU_AMEVCNTR0_EL0(11),
1643         AMU_AMEVCNTR0_EL0(12),
1644         AMU_AMEVCNTR0_EL0(13),
1645         AMU_AMEVCNTR0_EL0(14),
1646         AMU_AMEVCNTR0_EL0(15),
1647         AMU_AMEVTYPER0_EL0(0),
1648         AMU_AMEVTYPER0_EL0(1),
1649         AMU_AMEVTYPER0_EL0(2),
1650         AMU_AMEVTYPER0_EL0(3),
1651         AMU_AMEVTYPER0_EL0(4),
1652         AMU_AMEVTYPER0_EL0(5),
1653         AMU_AMEVTYPER0_EL0(6),
1654         AMU_AMEVTYPER0_EL0(7),
1655         AMU_AMEVTYPER0_EL0(8),
1656         AMU_AMEVTYPER0_EL0(9),
1657         AMU_AMEVTYPER0_EL0(10),
1658         AMU_AMEVTYPER0_EL0(11),
1659         AMU_AMEVTYPER0_EL0(12),
1660         AMU_AMEVTYPER0_EL0(13),
1661         AMU_AMEVTYPER0_EL0(14),
1662         AMU_AMEVTYPER0_EL0(15),
1663         AMU_AMEVCNTR1_EL0(0),
1664         AMU_AMEVCNTR1_EL0(1),
1665         AMU_AMEVCNTR1_EL0(2),
1666         AMU_AMEVCNTR1_EL0(3),
1667         AMU_AMEVCNTR1_EL0(4),
1668         AMU_AMEVCNTR1_EL0(5),
1669         AMU_AMEVCNTR1_EL0(6),
1670         AMU_AMEVCNTR1_EL0(7),
1671         AMU_AMEVCNTR1_EL0(8),
1672         AMU_AMEVCNTR1_EL0(9),
1673         AMU_AMEVCNTR1_EL0(10),
1674         AMU_AMEVCNTR1_EL0(11),
1675         AMU_AMEVCNTR1_EL0(12),
1676         AMU_AMEVCNTR1_EL0(13),
1677         AMU_AMEVCNTR1_EL0(14),
1678         AMU_AMEVCNTR1_EL0(15),
1679         AMU_AMEVTYPER1_EL0(0),
1680         AMU_AMEVTYPER1_EL0(1),
1681         AMU_AMEVTYPER1_EL0(2),
1682         AMU_AMEVTYPER1_EL0(3),
1683         AMU_AMEVTYPER1_EL0(4),
1684         AMU_AMEVTYPER1_EL0(5),
1685         AMU_AMEVTYPER1_EL0(6),
1686         AMU_AMEVTYPER1_EL0(7),
1687         AMU_AMEVTYPER1_EL0(8),
1688         AMU_AMEVTYPER1_EL0(9),
1689         AMU_AMEVTYPER1_EL0(10),
1690         AMU_AMEVTYPER1_EL0(11),
1691         AMU_AMEVTYPER1_EL0(12),
1692         AMU_AMEVTYPER1_EL0(13),
1693         AMU_AMEVTYPER1_EL0(14),
1694         AMU_AMEVTYPER1_EL0(15),
1695
1696         { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
1697         { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
1698         { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
1699
1700         /* PMEVCNTRn_EL0 */
1701         PMU_PMEVCNTR_EL0(0),
1702         PMU_PMEVCNTR_EL0(1),
1703         PMU_PMEVCNTR_EL0(2),
1704         PMU_PMEVCNTR_EL0(3),
1705         PMU_PMEVCNTR_EL0(4),
1706         PMU_PMEVCNTR_EL0(5),
1707         PMU_PMEVCNTR_EL0(6),
1708         PMU_PMEVCNTR_EL0(7),
1709         PMU_PMEVCNTR_EL0(8),
1710         PMU_PMEVCNTR_EL0(9),
1711         PMU_PMEVCNTR_EL0(10),
1712         PMU_PMEVCNTR_EL0(11),
1713         PMU_PMEVCNTR_EL0(12),
1714         PMU_PMEVCNTR_EL0(13),
1715         PMU_PMEVCNTR_EL0(14),
1716         PMU_PMEVCNTR_EL0(15),
1717         PMU_PMEVCNTR_EL0(16),
1718         PMU_PMEVCNTR_EL0(17),
1719         PMU_PMEVCNTR_EL0(18),
1720         PMU_PMEVCNTR_EL0(19),
1721         PMU_PMEVCNTR_EL0(20),
1722         PMU_PMEVCNTR_EL0(21),
1723         PMU_PMEVCNTR_EL0(22),
1724         PMU_PMEVCNTR_EL0(23),
1725         PMU_PMEVCNTR_EL0(24),
1726         PMU_PMEVCNTR_EL0(25),
1727         PMU_PMEVCNTR_EL0(26),
1728         PMU_PMEVCNTR_EL0(27),
1729         PMU_PMEVCNTR_EL0(28),
1730         PMU_PMEVCNTR_EL0(29),
1731         PMU_PMEVCNTR_EL0(30),
1732         /* PMEVTYPERn_EL0 */
1733         PMU_PMEVTYPER_EL0(0),
1734         PMU_PMEVTYPER_EL0(1),
1735         PMU_PMEVTYPER_EL0(2),
1736         PMU_PMEVTYPER_EL0(3),
1737         PMU_PMEVTYPER_EL0(4),
1738         PMU_PMEVTYPER_EL0(5),
1739         PMU_PMEVTYPER_EL0(6),
1740         PMU_PMEVTYPER_EL0(7),
1741         PMU_PMEVTYPER_EL0(8),
1742         PMU_PMEVTYPER_EL0(9),
1743         PMU_PMEVTYPER_EL0(10),
1744         PMU_PMEVTYPER_EL0(11),
1745         PMU_PMEVTYPER_EL0(12),
1746         PMU_PMEVTYPER_EL0(13),
1747         PMU_PMEVTYPER_EL0(14),
1748         PMU_PMEVTYPER_EL0(15),
1749         PMU_PMEVTYPER_EL0(16),
1750         PMU_PMEVTYPER_EL0(17),
1751         PMU_PMEVTYPER_EL0(18),
1752         PMU_PMEVTYPER_EL0(19),
1753         PMU_PMEVTYPER_EL0(20),
1754         PMU_PMEVTYPER_EL0(21),
1755         PMU_PMEVTYPER_EL0(22),
1756         PMU_PMEVTYPER_EL0(23),
1757         PMU_PMEVTYPER_EL0(24),
1758         PMU_PMEVTYPER_EL0(25),
1759         PMU_PMEVTYPER_EL0(26),
1760         PMU_PMEVTYPER_EL0(27),
1761         PMU_PMEVTYPER_EL0(28),
1762         PMU_PMEVTYPER_EL0(29),
1763         PMU_PMEVTYPER_EL0(30),
1764         /*
1765          * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
1766          * in 32bit mode. Here we choose to reset it as zero for consistency.
1767          */
1768         { PMU_SYS_REG(SYS_PMCCFILTR_EL0), .access = access_pmu_evtyper,
1769           .reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 },
1770
1771         { SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 },
1772         { SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 },
1773         { SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 },
1774 };
1775
1776 static bool trap_dbgdidr(struct kvm_vcpu *vcpu,
1777                         struct sys_reg_params *p,
1778                         const struct sys_reg_desc *r)
1779 {
1780         if (p->is_write) {
1781                 return ignore_write(vcpu, p);
1782         } else {
1783                 u64 dfr = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1784                 u64 pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1785                 u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT);
1786
1787                 p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
1788                              (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
1789                              (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
1790                              | (6 << 16) | (1 << 15) | (el3 << 14) | (el3 << 12));
1791                 return true;
1792         }
1793 }
1794
1795 /*
1796  * AArch32 debug register mappings
1797  *
1798  * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
1799  * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
1800  *
1801  * None of the other registers share their location, so treat them as
1802  * if they were 64bit.
1803  */
1804 #define DBG_BCR_BVR_WCR_WVR(n)                                                \
1805         /* DBGBVRn */                                                         \
1806         { AA32(LO), Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
1807         /* DBGBCRn */                                                         \
1808         { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n },           \
1809         /* DBGWVRn */                                                         \
1810         { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n },           \
1811         /* DBGWCRn */                                                         \
1812         { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
1813
1814 #define DBGBXVR(n)                                                            \
1815         { AA32(HI), Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_bvr, NULL, n }
1816
1817 /*
1818  * Trapped cp14 registers. We generally ignore most of the external
1819  * debug, on the principle that they don't really make sense to a
1820  * guest. Revisit this one day, would this principle change.
1821  */
1822 static const struct sys_reg_desc cp14_regs[] = {
1823         /* DBGDIDR */
1824         { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgdidr },
1825         /* DBGDTRRXext */
1826         { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
1827
1828         DBG_BCR_BVR_WCR_WVR(0),
1829         /* DBGDSCRint */
1830         { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
1831         DBG_BCR_BVR_WCR_WVR(1),
1832         /* DBGDCCINT */
1833         { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug_regs, NULL, MDCCINT_EL1 },
1834         /* DBGDSCRext */
1835         { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug_regs, NULL, MDSCR_EL1 },
1836         DBG_BCR_BVR_WCR_WVR(2),
1837         /* DBGDTR[RT]Xint */
1838         { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
1839         /* DBGDTR[RT]Xext */
1840         { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
1841         DBG_BCR_BVR_WCR_WVR(3),
1842         DBG_BCR_BVR_WCR_WVR(4),
1843         DBG_BCR_BVR_WCR_WVR(5),
1844         /* DBGWFAR */
1845         { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
1846         /* DBGOSECCR */
1847         { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
1848         DBG_BCR_BVR_WCR_WVR(6),
1849         /* DBGVCR */
1850         { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug_regs, NULL, DBGVCR32_EL2 },
1851         DBG_BCR_BVR_WCR_WVR(7),
1852         DBG_BCR_BVR_WCR_WVR(8),
1853         DBG_BCR_BVR_WCR_WVR(9),
1854         DBG_BCR_BVR_WCR_WVR(10),
1855         DBG_BCR_BVR_WCR_WVR(11),
1856         DBG_BCR_BVR_WCR_WVR(12),
1857         DBG_BCR_BVR_WCR_WVR(13),
1858         DBG_BCR_BVR_WCR_WVR(14),
1859         DBG_BCR_BVR_WCR_WVR(15),
1860
1861         /* DBGDRAR (32bit) */
1862         { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
1863
1864         DBGBXVR(0),
1865         /* DBGOSLAR */
1866         { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi },
1867         DBGBXVR(1),
1868         /* DBGOSLSR */
1869         { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 },
1870         DBGBXVR(2),
1871         DBGBXVR(3),
1872         /* DBGOSDLR */
1873         { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
1874         DBGBXVR(4),
1875         /* DBGPRCR */
1876         { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
1877         DBGBXVR(5),
1878         DBGBXVR(6),
1879         DBGBXVR(7),
1880         DBGBXVR(8),
1881         DBGBXVR(9),
1882         DBGBXVR(10),
1883         DBGBXVR(11),
1884         DBGBXVR(12),
1885         DBGBXVR(13),
1886         DBGBXVR(14),
1887         DBGBXVR(15),
1888
1889         /* DBGDSAR (32bit) */
1890         { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
1891
1892         /* DBGDEVID2 */
1893         { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
1894         /* DBGDEVID1 */
1895         { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
1896         /* DBGDEVID */
1897         { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
1898         /* DBGCLAIMSET */
1899         { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
1900         /* DBGCLAIMCLR */
1901         { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
1902         /* DBGAUTHSTATUS */
1903         { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
1904 };
1905
1906 /* Trapped cp14 64bit registers */
1907 static const struct sys_reg_desc cp14_64_regs[] = {
1908         /* DBGDRAR (64bit) */
1909         { Op1( 0), CRm( 1), .access = trap_raz_wi },
1910
1911         /* DBGDSAR (64bit) */
1912         { Op1( 0), CRm( 2), .access = trap_raz_wi },
1913 };
1914
1915 /* Macro to expand the PMEVCNTRn register */
1916 #define PMU_PMEVCNTR(n)                                                 \
1917         /* PMEVCNTRn */                                                 \
1918         { Op1(0), CRn(0b1110),                                          \
1919           CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)),         \
1920           access_pmu_evcntr }
1921
1922 /* Macro to expand the PMEVTYPERn register */
1923 #define PMU_PMEVTYPER(n)                                                \
1924         /* PMEVTYPERn */                                                \
1925         { Op1(0), CRn(0b1110),                                          \
1926           CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)),         \
1927           access_pmu_evtyper }
1928
1929 /*
1930  * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
1931  * depending on the way they are accessed (as a 32bit or a 64bit
1932  * register).
1933  */
1934 static const struct sys_reg_desc cp15_regs[] = {
1935         { Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr },
1936         { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, SCTLR_EL1 },
1937         /* ACTLR */
1938         { AA32(LO), Op1( 0), CRn( 1), CRm( 0), Op2( 1), access_actlr, NULL, ACTLR_EL1 },
1939         /* ACTLR2 */
1940         { AA32(HI), Op1( 0), CRn( 1), CRm( 0), Op2( 3), access_actlr, NULL, ACTLR_EL1 },
1941         { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
1942         { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, TTBR1_EL1 },
1943         /* TTBCR */
1944         { AA32(LO), Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, TCR_EL1 },
1945         /* TTBCR2 */
1946         { AA32(HI), Op1( 0), CRn( 2), CRm( 0), Op2( 3), access_vm_reg, NULL, TCR_EL1 },
1947         { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, DACR32_EL2 },
1948         /* DFSR */
1949         { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, ESR_EL1 },
1950         { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, IFSR32_EL2 },
1951         /* ADFSR */
1952         { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, AFSR0_EL1 },
1953         /* AIFSR */
1954         { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, AFSR1_EL1 },
1955         /* DFAR */
1956         { AA32(LO), Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, FAR_EL1 },
1957         /* IFAR */
1958         { AA32(HI), Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, FAR_EL1 },
1959
1960         /*
1961          * DC{C,I,CI}SW operations:
1962          */
1963         { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
1964         { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
1965         { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
1966
1967         /* PMU */
1968         { Op1( 0), CRn( 9), CRm(12), Op2( 0), access_pmcr },
1969         { Op1( 0), CRn( 9), CRm(12), Op2( 1), access_pmcnten },
1970         { Op1( 0), CRn( 9), CRm(12), Op2( 2), access_pmcnten },
1971         { Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },
1972         { Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },
1973         { Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
1974         { AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1975         { AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
1976         { Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
1977         { Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
1978         { Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
1979         { Op1( 0), CRn( 9), CRm(14), Op2( 0), access_pmuserenr },
1980         { Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },
1981         { Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },
1982         { Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },
1983         { AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 4), access_pmceid },
1984         { AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 5), access_pmceid },
1985         /* PMMIR */
1986         { Op1( 0), CRn( 9), CRm(14), Op2( 6), trap_raz_wi },
1987
1988         /* PRRR/MAIR0 */
1989         { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },
1990         /* NMRR/MAIR1 */
1991         { AA32(HI), Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, MAIR_EL1 },
1992         /* AMAIR0 */
1993         { AA32(LO), Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, AMAIR_EL1 },
1994         /* AMAIR1 */
1995         { AA32(HI), Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, AMAIR_EL1 },
1996
1997         /* ICC_SRE */
1998         { Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
1999
2000         { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, CONTEXTIDR_EL1 },
2001
2002         /* Arch Tmers */
2003         { SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer },
2004         { SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer },
2005
2006         /* PMEVCNTRn */
2007         PMU_PMEVCNTR(0),
2008         PMU_PMEVCNTR(1),
2009         PMU_PMEVCNTR(2),
2010         PMU_PMEVCNTR(3),
2011         PMU_PMEVCNTR(4),
2012         PMU_PMEVCNTR(5),
2013         PMU_PMEVCNTR(6),
2014         PMU_PMEVCNTR(7),
2015         PMU_PMEVCNTR(8),
2016         PMU_PMEVCNTR(9),
2017         PMU_PMEVCNTR(10),
2018         PMU_PMEVCNTR(11),
2019         PMU_PMEVCNTR(12),
2020         PMU_PMEVCNTR(13),
2021         PMU_PMEVCNTR(14),
2022         PMU_PMEVCNTR(15),
2023         PMU_PMEVCNTR(16),
2024         PMU_PMEVCNTR(17),
2025         PMU_PMEVCNTR(18),
2026         PMU_PMEVCNTR(19),
2027         PMU_PMEVCNTR(20),
2028         PMU_PMEVCNTR(21),
2029         PMU_PMEVCNTR(22),
2030         PMU_PMEVCNTR(23),
2031         PMU_PMEVCNTR(24),
2032         PMU_PMEVCNTR(25),
2033         PMU_PMEVCNTR(26),
2034         PMU_PMEVCNTR(27),
2035         PMU_PMEVCNTR(28),
2036         PMU_PMEVCNTR(29),
2037         PMU_PMEVCNTR(30),
2038         /* PMEVTYPERn */
2039         PMU_PMEVTYPER(0),
2040         PMU_PMEVTYPER(1),
2041         PMU_PMEVTYPER(2),
2042         PMU_PMEVTYPER(3),
2043         PMU_PMEVTYPER(4),
2044         PMU_PMEVTYPER(5),
2045         PMU_PMEVTYPER(6),
2046         PMU_PMEVTYPER(7),
2047         PMU_PMEVTYPER(8),
2048         PMU_PMEVTYPER(9),
2049         PMU_PMEVTYPER(10),
2050         PMU_PMEVTYPER(11),
2051         PMU_PMEVTYPER(12),
2052         PMU_PMEVTYPER(13),
2053         PMU_PMEVTYPER(14),
2054         PMU_PMEVTYPER(15),
2055         PMU_PMEVTYPER(16),
2056         PMU_PMEVTYPER(17),
2057         PMU_PMEVTYPER(18),
2058         PMU_PMEVTYPER(19),
2059         PMU_PMEVTYPER(20),
2060         PMU_PMEVTYPER(21),
2061         PMU_PMEVTYPER(22),
2062         PMU_PMEVTYPER(23),
2063         PMU_PMEVTYPER(24),
2064         PMU_PMEVTYPER(25),
2065         PMU_PMEVTYPER(26),
2066         PMU_PMEVTYPER(27),
2067         PMU_PMEVTYPER(28),
2068         PMU_PMEVTYPER(29),
2069         PMU_PMEVTYPER(30),
2070         /* PMCCFILTR */
2071         { Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper },
2072
2073         { Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr },
2074         { Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr },
2075         { Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, CSSELR_EL1 },
2076 };
2077
2078 static const struct sys_reg_desc cp15_64_regs[] = {
2079         { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
2080         { Op1( 0), CRn( 0), CRm( 9), Op2( 0), access_pmu_evcntr },
2081         { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI1R */
2082         { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR1_EL1 },
2083         { Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */
2084         { Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI0R */
2085         { SYS_DESC(SYS_AARCH32_CNTP_CVAL),    access_arch_timer },
2086 };
2087
2088 static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
2089                               bool is_32)
2090 {
2091         unsigned int i;
2092
2093         for (i = 0; i < n; i++) {
2094                 if (!is_32 && table[i].reg && !table[i].reset) {
2095                         kvm_err("sys_reg table %p entry %d has lacks reset\n",
2096                                 table, i);
2097                         return 1;
2098                 }
2099
2100                 if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
2101                         kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
2102                         return 1;
2103                 }
2104         }
2105
2106         return 0;
2107 }
2108
2109 static int match_sys_reg(const void *key, const void *elt)
2110 {
2111         const unsigned long pval = (unsigned long)key;
2112         const struct sys_reg_desc *r = elt;
2113
2114         return pval - reg_to_encoding(r);
2115 }
2116
2117 static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
2118                                          const struct sys_reg_desc table[],
2119                                          unsigned int num)
2120 {
2121         unsigned long pval = reg_to_encoding(params);
2122
2123         return bsearch((void *)pval, table, num, sizeof(table[0]), match_sys_reg);
2124 }
2125
2126 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu)
2127 {
2128         kvm_inject_undefined(vcpu);
2129         return 1;
2130 }
2131
2132 static void perform_access(struct kvm_vcpu *vcpu,
2133                            struct sys_reg_params *params,
2134                            const struct sys_reg_desc *r)
2135 {
2136         trace_kvm_sys_access(*vcpu_pc(vcpu), params, r);
2137
2138         /* Check for regs disabled by runtime config */
2139         if (sysreg_hidden(vcpu, r)) {
2140                 kvm_inject_undefined(vcpu);
2141                 return;
2142         }
2143
2144         /*
2145          * Not having an accessor means that we have configured a trap
2146          * that we don't know how to handle. This certainly qualifies
2147          * as a gross bug that should be fixed right away.
2148          */
2149         BUG_ON(!r->access);
2150
2151         /* Skip instruction if instructed so */
2152         if (likely(r->access(vcpu, params, r)))
2153                 kvm_incr_pc(vcpu);
2154 }
2155
2156 /*
2157  * emulate_cp --  tries to match a sys_reg access in a handling table, and
2158  *                call the corresponding trap handler.
2159  *
2160  * @params: pointer to the descriptor of the access
2161  * @table: array of trap descriptors
2162  * @num: size of the trap descriptor array
2163  *
2164  * Return 0 if the access has been handled, and -1 if not.
2165  */
2166 static int emulate_cp(struct kvm_vcpu *vcpu,
2167                       struct sys_reg_params *params,
2168                       const struct sys_reg_desc *table,
2169                       size_t num)
2170 {
2171         const struct sys_reg_desc *r;
2172
2173         if (!table)
2174                 return -1;      /* Not handled */
2175
2176         r = find_reg(params, table, num);
2177
2178         if (r) {
2179                 perform_access(vcpu, params, r);
2180                 return 0;
2181         }
2182
2183         /* Not handled */
2184         return -1;
2185 }
2186
2187 static void unhandled_cp_access(struct kvm_vcpu *vcpu,
2188                                 struct sys_reg_params *params)
2189 {
2190         u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
2191         int cp = -1;
2192
2193         switch (esr_ec) {
2194         case ESR_ELx_EC_CP15_32:
2195         case ESR_ELx_EC_CP15_64:
2196                 cp = 15;
2197                 break;
2198         case ESR_ELx_EC_CP14_MR:
2199         case ESR_ELx_EC_CP14_64:
2200                 cp = 14;
2201                 break;
2202         default:
2203                 WARN_ON(1);
2204         }
2205
2206         print_sys_reg_msg(params,
2207                           "Unsupported guest CP%d access at: %08lx [%08lx]\n",
2208                           cp, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
2209         kvm_inject_undefined(vcpu);
2210 }
2211
2212 /**
2213  * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
2214  * @vcpu: The VCPU pointer
2215  * @run:  The kvm_run struct
2216  */
2217 static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
2218                             const struct sys_reg_desc *global,
2219                             size_t nr_global)
2220 {
2221         struct sys_reg_params params;
2222         u32 esr = kvm_vcpu_get_esr(vcpu);
2223         int Rt = kvm_vcpu_sys_get_rt(vcpu);
2224         int Rt2 = (esr >> 10) & 0x1f;
2225
2226         params.CRm = (esr >> 1) & 0xf;
2227         params.is_write = ((esr & 1) == 0);
2228
2229         params.Op0 = 0;
2230         params.Op1 = (esr >> 16) & 0xf;
2231         params.Op2 = 0;
2232         params.CRn = 0;
2233
2234         /*
2235          * Make a 64-bit value out of Rt and Rt2. As we use the same trap
2236          * backends between AArch32 and AArch64, we get away with it.
2237          */
2238         if (params.is_write) {
2239                 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
2240                 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
2241         }
2242
2243         /*
2244          * If the table contains a handler, handle the
2245          * potential register operation in the case of a read and return
2246          * with success.
2247          */
2248         if (!emulate_cp(vcpu, &params, global, nr_global)) {
2249                 /* Split up the value between registers for the read side */
2250                 if (!params.is_write) {
2251                         vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
2252                         vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
2253                 }
2254
2255                 return 1;
2256         }
2257
2258         unhandled_cp_access(vcpu, &params);
2259         return 1;
2260 }
2261
2262 /**
2263  * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
2264  * @vcpu: The VCPU pointer
2265  * @run:  The kvm_run struct
2266  */
2267 static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
2268                             const struct sys_reg_desc *global,
2269                             size_t nr_global)
2270 {
2271         struct sys_reg_params params;
2272         u32 esr = kvm_vcpu_get_esr(vcpu);
2273         int Rt  = kvm_vcpu_sys_get_rt(vcpu);
2274
2275         params.CRm = (esr >> 1) & 0xf;
2276         params.regval = vcpu_get_reg(vcpu, Rt);
2277         params.is_write = ((esr & 1) == 0);
2278         params.CRn = (esr >> 10) & 0xf;
2279         params.Op0 = 0;
2280         params.Op1 = (esr >> 14) & 0x7;
2281         params.Op2 = (esr >> 17) & 0x7;
2282
2283         if (!emulate_cp(vcpu, &params, global, nr_global)) {
2284                 if (!params.is_write)
2285                         vcpu_set_reg(vcpu, Rt, params.regval);
2286                 return 1;
2287         }
2288
2289         unhandled_cp_access(vcpu, &params);
2290         return 1;
2291 }
2292
2293 int kvm_handle_cp15_64(struct kvm_vcpu *vcpu)
2294 {
2295         return kvm_handle_cp_64(vcpu, cp15_64_regs, ARRAY_SIZE(cp15_64_regs));
2296 }
2297
2298 int kvm_handle_cp15_32(struct kvm_vcpu *vcpu)
2299 {
2300         return kvm_handle_cp_32(vcpu, cp15_regs, ARRAY_SIZE(cp15_regs));
2301 }
2302
2303 int kvm_handle_cp14_64(struct kvm_vcpu *vcpu)
2304 {
2305         return kvm_handle_cp_64(vcpu, cp14_64_regs, ARRAY_SIZE(cp14_64_regs));
2306 }
2307
2308 int kvm_handle_cp14_32(struct kvm_vcpu *vcpu)
2309 {
2310         return kvm_handle_cp_32(vcpu, cp14_regs, ARRAY_SIZE(cp14_regs));
2311 }
2312
2313 static bool is_imp_def_sys_reg(struct sys_reg_params *params)
2314 {
2315         // See ARM DDI 0487E.a, section D12.3.2
2316         return params->Op0 == 3 && (params->CRn & 0b1011) == 0b1011;
2317 }
2318
2319 static int emulate_sys_reg(struct kvm_vcpu *vcpu,
2320                            struct sys_reg_params *params)
2321 {
2322         const struct sys_reg_desc *r;
2323
2324         r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2325
2326         if (likely(r)) {
2327                 perform_access(vcpu, params, r);
2328         } else if (is_imp_def_sys_reg(params)) {
2329                 kvm_inject_undefined(vcpu);
2330         } else {
2331                 print_sys_reg_msg(params,
2332                                   "Unsupported guest sys_reg access at: %lx [%08lx]\n",
2333                                   *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
2334                 kvm_inject_undefined(vcpu);
2335         }
2336         return 1;
2337 }
2338
2339 /**
2340  * kvm_reset_sys_regs - sets system registers to reset value
2341  * @vcpu: The VCPU pointer
2342  *
2343  * This function finds the right table above and sets the registers on the
2344  * virtual CPU struct to their architecturally defined reset values.
2345  */
2346 void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
2347 {
2348         unsigned long i;
2349
2350         for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++)
2351                 if (sys_reg_descs[i].reset)
2352                         sys_reg_descs[i].reset(vcpu, &sys_reg_descs[i]);
2353 }
2354
2355 /**
2356  * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
2357  * @vcpu: The VCPU pointer
2358  */
2359 int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
2360 {
2361         struct sys_reg_params params;
2362         unsigned long esr = kvm_vcpu_get_esr(vcpu);
2363         int Rt = kvm_vcpu_sys_get_rt(vcpu);
2364         int ret;
2365
2366         trace_kvm_handle_sys_reg(esr);
2367
2368         params.Op0 = (esr >> 20) & 3;
2369         params.Op1 = (esr >> 14) & 0x7;
2370         params.CRn = (esr >> 10) & 0xf;
2371         params.CRm = (esr >> 1) & 0xf;
2372         params.Op2 = (esr >> 17) & 0x7;
2373         params.regval = vcpu_get_reg(vcpu, Rt);
2374         params.is_write = !(esr & 1);
2375
2376         ret = emulate_sys_reg(vcpu, &params);
2377
2378         if (!params.is_write)
2379                 vcpu_set_reg(vcpu, Rt, params.regval);
2380         return ret;
2381 }
2382
2383 /******************************************************************************
2384  * Userspace API
2385  *****************************************************************************/
2386
2387 static bool index_to_params(u64 id, struct sys_reg_params *params)
2388 {
2389         switch (id & KVM_REG_SIZE_MASK) {
2390         case KVM_REG_SIZE_U64:
2391                 /* Any unused index bits means it's not valid. */
2392                 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
2393                               | KVM_REG_ARM_COPROC_MASK
2394                               | KVM_REG_ARM64_SYSREG_OP0_MASK
2395                               | KVM_REG_ARM64_SYSREG_OP1_MASK
2396                               | KVM_REG_ARM64_SYSREG_CRN_MASK
2397                               | KVM_REG_ARM64_SYSREG_CRM_MASK
2398                               | KVM_REG_ARM64_SYSREG_OP2_MASK))
2399                         return false;
2400                 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
2401                                >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
2402                 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
2403                                >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
2404                 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
2405                                >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
2406                 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
2407                                >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
2408                 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
2409                                >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
2410                 return true;
2411         default:
2412                 return false;
2413         }
2414 }
2415
2416 const struct sys_reg_desc *find_reg_by_id(u64 id,
2417                                           struct sys_reg_params *params,
2418                                           const struct sys_reg_desc table[],
2419                                           unsigned int num)
2420 {
2421         if (!index_to_params(id, params))
2422                 return NULL;
2423
2424         return find_reg(params, table, num);
2425 }
2426
2427 /* Decode an index value, and find the sys_reg_desc entry. */
2428 static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
2429                                                     u64 id)
2430 {
2431         const struct sys_reg_desc *r;
2432         struct sys_reg_params params;
2433
2434         /* We only do sys_reg for now. */
2435         if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
2436                 return NULL;
2437
2438         if (!index_to_params(id, &params))
2439                 return NULL;
2440
2441         r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2442
2443         /* Not saved in the sys_reg array and not otherwise accessible? */
2444         if (r && !(r->reg || r->get_user))
2445                 r = NULL;
2446
2447         return r;
2448 }
2449
2450 /*
2451  * These are the invariant sys_reg registers: we let the guest see the
2452  * host versions of these, so they're part of the guest state.
2453  *
2454  * A future CPU may provide a mechanism to present different values to
2455  * the guest, or a future kvm may trap them.
2456  */
2457
2458 #define FUNCTION_INVARIANT(reg)                                         \
2459         static void get_##reg(struct kvm_vcpu *v,                       \
2460                               const struct sys_reg_desc *r)             \
2461         {                                                               \
2462                 ((struct sys_reg_desc *)r)->val = read_sysreg(reg);     \
2463         }
2464
2465 FUNCTION_INVARIANT(midr_el1)
2466 FUNCTION_INVARIANT(revidr_el1)
2467 FUNCTION_INVARIANT(clidr_el1)
2468 FUNCTION_INVARIANT(aidr_el1)
2469
2470 static void get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r)
2471 {
2472         ((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0);
2473 }
2474
2475 /* ->val is filled in by kvm_sys_reg_table_init() */
2476 static struct sys_reg_desc invariant_sys_regs[] = {
2477         { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
2478         { SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
2479         { SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },
2480         { SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
2481         { SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
2482 };
2483
2484 static int reg_from_user(u64 *val, const void __user *uaddr, u64 id)
2485 {
2486         if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
2487                 return -EFAULT;
2488         return 0;
2489 }
2490
2491 static int reg_to_user(void __user *uaddr, const u64 *val, u64 id)
2492 {
2493         if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
2494                 return -EFAULT;
2495         return 0;
2496 }
2497
2498 static int get_invariant_sys_reg(u64 id, void __user *uaddr)
2499 {
2500         struct sys_reg_params params;
2501         const struct sys_reg_desc *r;
2502
2503         r = find_reg_by_id(id, &params, invariant_sys_regs,
2504                            ARRAY_SIZE(invariant_sys_regs));
2505         if (!r)
2506                 return -ENOENT;
2507
2508         return reg_to_user(uaddr, &r->val, id);
2509 }
2510
2511 static int set_invariant_sys_reg(u64 id, void __user *uaddr)
2512 {
2513         struct sys_reg_params params;
2514         const struct sys_reg_desc *r;
2515         int err;
2516         u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
2517
2518         r = find_reg_by_id(id, &params, invariant_sys_regs,
2519                            ARRAY_SIZE(invariant_sys_regs));
2520         if (!r)
2521                 return -ENOENT;
2522
2523         err = reg_from_user(&val, uaddr, id);
2524         if (err)
2525                 return err;
2526
2527         /* This is what we mean by invariant: you can't change it. */
2528         if (r->val != val)
2529                 return -EINVAL;
2530
2531         return 0;
2532 }
2533
2534 static bool is_valid_cache(u32 val)
2535 {
2536         u32 level, ctype;
2537
2538         if (val >= CSSELR_MAX)
2539                 return false;
2540
2541         /* Bottom bit is Instruction or Data bit.  Next 3 bits are level. */
2542         level = (val >> 1);
2543         ctype = (cache_levels >> (level * 3)) & 7;
2544
2545         switch (ctype) {
2546         case 0: /* No cache */
2547                 return false;
2548         case 1: /* Instruction cache only */
2549                 return (val & 1);
2550         case 2: /* Data cache only */
2551         case 4: /* Unified cache */
2552                 return !(val & 1);
2553         case 3: /* Separate instruction and data caches */
2554                 return true;
2555         default: /* Reserved: we can't know instruction or data. */
2556                 return false;
2557         }
2558 }
2559
2560 static int demux_c15_get(u64 id, void __user *uaddr)
2561 {
2562         u32 val;
2563         u32 __user *uval = uaddr;
2564
2565         /* Fail if we have unknown bits set. */
2566         if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2567                    | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2568                 return -ENOENT;
2569
2570         switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2571         case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2572                 if (KVM_REG_SIZE(id) != 4)
2573                         return -ENOENT;
2574                 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2575                         >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2576                 if (!is_valid_cache(val))
2577                         return -ENOENT;
2578
2579                 return put_user(get_ccsidr(val), uval);
2580         default:
2581                 return -ENOENT;
2582         }
2583 }
2584
2585 static int demux_c15_set(u64 id, void __user *uaddr)
2586 {
2587         u32 val, newval;
2588         u32 __user *uval = uaddr;
2589
2590         /* Fail if we have unknown bits set. */
2591         if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2592                    | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2593                 return -ENOENT;
2594
2595         switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2596         case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2597                 if (KVM_REG_SIZE(id) != 4)
2598                         return -ENOENT;
2599                 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2600                         >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2601                 if (!is_valid_cache(val))
2602                         return -ENOENT;
2603
2604                 if (get_user(newval, uval))
2605                         return -EFAULT;
2606
2607                 /* This is also invariant: you can't change it. */
2608                 if (newval != get_ccsidr(val))
2609                         return -EINVAL;
2610                 return 0;
2611         default:
2612                 return -ENOENT;
2613         }
2614 }
2615
2616 int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2617 {
2618         const struct sys_reg_desc *r;
2619         void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2620
2621         if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2622                 return demux_c15_get(reg->id, uaddr);
2623
2624         if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2625                 return -ENOENT;
2626
2627         r = index_to_sys_reg_desc(vcpu, reg->id);
2628         if (!r)
2629                 return get_invariant_sys_reg(reg->id, uaddr);
2630
2631         /* Check for regs disabled by runtime config */
2632         if (sysreg_hidden(vcpu, r))
2633                 return -ENOENT;
2634
2635         if (r->get_user)
2636                 return (r->get_user)(vcpu, r, reg, uaddr);
2637
2638         return reg_to_user(uaddr, &__vcpu_sys_reg(vcpu, r->reg), reg->id);
2639 }
2640
2641 int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2642 {
2643         const struct sys_reg_desc *r;
2644         void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2645
2646         if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2647                 return demux_c15_set(reg->id, uaddr);
2648
2649         if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2650                 return -ENOENT;
2651
2652         r = index_to_sys_reg_desc(vcpu, reg->id);
2653         if (!r)
2654                 return set_invariant_sys_reg(reg->id, uaddr);
2655
2656         /* Check for regs disabled by runtime config */
2657         if (sysreg_hidden(vcpu, r))
2658                 return -ENOENT;
2659
2660         if (r->set_user)
2661                 return (r->set_user)(vcpu, r, reg, uaddr);
2662
2663         return reg_from_user(&__vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);
2664 }
2665
2666 static unsigned int num_demux_regs(void)
2667 {
2668         unsigned int i, count = 0;
2669
2670         for (i = 0; i < CSSELR_MAX; i++)
2671                 if (is_valid_cache(i))
2672                         count++;
2673
2674         return count;
2675 }
2676
2677 static int write_demux_regids(u64 __user *uindices)
2678 {
2679         u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
2680         unsigned int i;
2681
2682         val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
2683         for (i = 0; i < CSSELR_MAX; i++) {
2684                 if (!is_valid_cache(i))
2685                         continue;
2686                 if (put_user(val | i, uindices))
2687                         return -EFAULT;
2688                 uindices++;
2689         }
2690         return 0;
2691 }
2692
2693 static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
2694 {
2695         return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
2696                 KVM_REG_ARM64_SYSREG |
2697                 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
2698                 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
2699                 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
2700                 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
2701                 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
2702 }
2703
2704 static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
2705 {
2706         if (!*uind)
2707                 return true;
2708
2709         if (put_user(sys_reg_to_index(reg), *uind))
2710                 return false;
2711
2712         (*uind)++;
2713         return true;
2714 }
2715
2716 static int walk_one_sys_reg(const struct kvm_vcpu *vcpu,
2717                             const struct sys_reg_desc *rd,
2718                             u64 __user **uind,
2719                             unsigned int *total)
2720 {
2721         /*
2722          * Ignore registers we trap but don't save,
2723          * and for which no custom user accessor is provided.
2724          */
2725         if (!(rd->reg || rd->get_user))
2726                 return 0;
2727
2728         if (sysreg_hidden(vcpu, rd))
2729                 return 0;
2730
2731         if (!copy_reg_to_user(rd, uind))
2732                 return -EFAULT;
2733
2734         (*total)++;
2735         return 0;
2736 }
2737
2738 /* Assumed ordered tables, see kvm_sys_reg_table_init. */
2739 static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
2740 {
2741         const struct sys_reg_desc *i2, *end2;
2742         unsigned int total = 0;
2743         int err;
2744
2745         i2 = sys_reg_descs;
2746         end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
2747
2748         while (i2 != end2) {
2749                 err = walk_one_sys_reg(vcpu, i2++, &uind, &total);
2750                 if (err)
2751                         return err;
2752         }
2753         return total;
2754 }
2755
2756 unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
2757 {
2758         return ARRAY_SIZE(invariant_sys_regs)
2759                 + num_demux_regs()
2760                 + walk_sys_regs(vcpu, (u64 __user *)NULL);
2761 }
2762
2763 int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
2764 {
2765         unsigned int i;
2766         int err;
2767
2768         /* Then give them all the invariant registers' indices. */
2769         for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
2770                 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
2771                         return -EFAULT;
2772                 uindices++;
2773         }
2774
2775         err = walk_sys_regs(vcpu, uindices);
2776         if (err < 0)
2777                 return err;
2778         uindices += err;
2779
2780         return write_demux_regids(uindices);
2781 }
2782
2783 void kvm_sys_reg_table_init(void)
2784 {
2785         unsigned int i;
2786         struct sys_reg_desc clidr;
2787
2788         /* Make sure tables are unique and in order. */
2789         BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false));
2790         BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true));
2791         BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true));
2792         BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true));
2793         BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true));
2794         BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false));
2795
2796         /* We abuse the reset function to overwrite the table itself. */
2797         for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
2798                 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
2799
2800         /*
2801          * CLIDR format is awkward, so clean it up.  See ARM B4.1.20:
2802          *
2803          *   If software reads the Cache Type fields from Ctype1
2804          *   upwards, once it has seen a value of 0b000, no caches
2805          *   exist at further-out levels of the hierarchy. So, for
2806          *   example, if Ctype3 is the first Cache Type field with a
2807          *   value of 0b000, the values of Ctype4 to Ctype7 must be
2808          *   ignored.
2809          */
2810         get_clidr_el1(NULL, &clidr); /* Ugly... */
2811         cache_levels = clidr.val;
2812         for (i = 0; i < 7; i++)
2813                 if (((cache_levels >> (i*3)) & 7) == 0)
2814                         break;
2815         /* Clear all higher bits. */
2816         cache_levels &= (1 << (i*3))-1;
2817 }