arch/mips/kvm/vz.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS: Support for hardware virtualization extensions
   7  *
   8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
   9  * Authors: Yann Le Du <ledu@kymasys.com>
  10  */
  11
  12 #include <linux/errno.h>
  13 #include <linux/err.h>
  14 #include <linux/module.h>
  15 #include <linux/preempt.h>
  16 #include <linux/vmalloc.h>
  17 #include <asm/cacheflush.h>
  18 #include <asm/cacheops.h>
  19 #include <asm/cmpxchg.h>
  20 #include <asm/fpu.h>
  21 #include <asm/hazards.h>
  22 #include <asm/inst.h>
  23 #include <asm/mmu_context.h>
  24 #include <asm/r4kcache.h>
  25 #include <asm/time.h>
  26 #include <asm/tlb.h>
  27 #include <asm/tlbex.h>
  28
  29 #include <linux/kvm_host.h>
  30
  31 #include "interrupt.h"
  32 #ifdef CONFIG_CPU_LOONGSON64
  33 #include "loongson_regs.h"
  34 #endif
  35
  36 #include "trace.h"
  37
  38 /* Pointers to last VCPU loaded on each physical CPU */
  39 static struct kvm_vcpu *last_vcpu[NR_CPUS];
  40 /* Pointers to last VCPU executed on each physical CPU */
  41 static struct kvm_vcpu *last_exec_vcpu[NR_CPUS];
  42
  43 /*
  44  * Number of guest VTLB entries to use, so we can catch inconsistency between
  45  * CPUs.
  46  */
  47 static unsigned int kvm_vz_guest_vtlb_size;
  48
  49 static inline long kvm_vz_read_gc0_ebase(void)
  50 {
  51         if (sizeof(long) == 8 && cpu_has_ebase_wg)
  52                 return read_gc0_ebase_64();
  53         else
  54                 return read_gc0_ebase();
  55 }
  56
  57 static inline void kvm_vz_write_gc0_ebase(long v)
  58 {
  59         /*
  60          * First write with WG=1 to write upper bits, then write again in case
  61          * WG should be left at 0.
  62          * write_gc0_ebase_64() is no longer UNDEFINED since R6.
  63          */
  64         if (sizeof(long) == 8 &&
  65             (cpu_has_mips64r6 || cpu_has_ebase_wg)) {
  66                 write_gc0_ebase_64(v | MIPS_EBASE_WG);
  67                 write_gc0_ebase_64(v);
  68         } else {
  69                 write_gc0_ebase(v | MIPS_EBASE_WG);
  70                 write_gc0_ebase(v);
  71         }
  72 }
  73
  74 /*
  75  * These Config bits may be writable by the guest:
  76  * Config:      [K23, KU] (!TLB), K0
  77  * Config1:     (none)
  78  * Config2:     [TU, SU] (impl)
  79  * Config3:     ISAOnExc
  80  * Config4:     FTLBPageSize
  81  * Config5:     K, CV, MSAEn, UFE, FRE, SBRI, UFR
  82  */
  83
  84 static inline unsigned int kvm_vz_config_guest_wrmask(struct kvm_vcpu *vcpu)
  85 {
  86         return CONF_CM_CMASK;
  87 }
  88
  89 static inline unsigned int kvm_vz_config1_guest_wrmask(struct kvm_vcpu *vcpu)
  90 {
  91         return 0;
  92 }
  93
  94 static inline unsigned int kvm_vz_config2_guest_wrmask(struct kvm_vcpu *vcpu)
  95 {
  96         return 0;
  97 }
  98
  99 static inline unsigned int kvm_vz_config3_guest_wrmask(struct kvm_vcpu *vcpu)
 100 {
 101         return MIPS_CONF3_ISA_OE;
 102 }
 103
 104 static inline unsigned int kvm_vz_config4_guest_wrmask(struct kvm_vcpu *vcpu)
 105 {
 106         /* no need to be exact */
 107         return MIPS_CONF4_VFTLBPAGESIZE;
 108 }
 109
 110 static inline unsigned int kvm_vz_config5_guest_wrmask(struct kvm_vcpu *vcpu)
 111 {
 112         unsigned int mask = MIPS_CONF5_K | MIPS_CONF5_CV | MIPS_CONF5_SBRI;
 113
 114         /* Permit MSAEn changes if MSA supported and enabled */
 115         if (kvm_mips_guest_has_msa(&vcpu->arch))
 116                 mask |= MIPS_CONF5_MSAEN;
 117
 118         /*
 119          * Permit guest FPU mode changes if FPU is enabled and the relevant
 120          * feature exists according to FIR register.
 121          */
 122         if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
 123                 if (cpu_has_ufr)
 124                         mask |= MIPS_CONF5_UFR;
 125                 if (cpu_has_fre)
 126                         mask |= MIPS_CONF5_FRE | MIPS_CONF5_UFE;
 127         }
 128
 129         return mask;
 130 }
 131
 132 static inline unsigned int kvm_vz_config6_guest_wrmask(struct kvm_vcpu *vcpu)
 133 {
 134         return LOONGSON_CONF6_INTIMER | LOONGSON_CONF6_EXTIMER;
 135 }
 136
 137 /*
 138  * VZ optionally allows these additional Config bits to be written by root:
 139  * Config:      M, [MT]
 140  * Config1:     M, [MMUSize-1, C2, MD, PC, WR, CA], FP
 141  * Config2:     M
 142  * Config3:     M, MSAP, [BPG], ULRI, [DSP2P, DSPP], CTXTC, [ITL, LPA, VEIC,
 143  *              VInt, SP, CDMM, MT, SM, TL]
 144  * Config4:     M, [VTLBSizeExt, MMUSizeExt]
 145  * Config5:     MRP
 146  */
 147
 148 static inline unsigned int kvm_vz_config_user_wrmask(struct kvm_vcpu *vcpu)
 149 {
 150         return kvm_vz_config_guest_wrmask(vcpu) | MIPS_CONF_M;
 151 }
 152
 153 static inline unsigned int kvm_vz_config1_user_wrmask(struct kvm_vcpu *vcpu)
 154 {
 155         unsigned int mask = kvm_vz_config1_guest_wrmask(vcpu) | MIPS_CONF_M;
 156
 157         /* Permit FPU to be present if FPU is supported */
 158         if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
 159                 mask |= MIPS_CONF1_FP;
 160
 161         return mask;
 162 }
 163
 164 static inline unsigned int kvm_vz_config2_user_wrmask(struct kvm_vcpu *vcpu)
 165 {
 166         return kvm_vz_config2_guest_wrmask(vcpu) | MIPS_CONF_M;
 167 }
 168
 169 static inline unsigned int kvm_vz_config3_user_wrmask(struct kvm_vcpu *vcpu)
 170 {
 171         unsigned int mask = kvm_vz_config3_guest_wrmask(vcpu) | MIPS_CONF_M |
 172                 MIPS_CONF3_ULRI | MIPS_CONF3_CTXTC;
 173
 174         /* Permit MSA to be present if MSA is supported */
 175         if (kvm_mips_guest_can_have_msa(&vcpu->arch))
 176                 mask |= MIPS_CONF3_MSA;
 177
 178         return mask;
 179 }
 180
 181 static inline unsigned int kvm_vz_config4_user_wrmask(struct kvm_vcpu *vcpu)
 182 {
 183         return kvm_vz_config4_guest_wrmask(vcpu) | MIPS_CONF_M;
 184 }
 185
 186 static inline unsigned int kvm_vz_config5_user_wrmask(struct kvm_vcpu *vcpu)
 187 {
 188         return kvm_vz_config5_guest_wrmask(vcpu) | MIPS_CONF5_MRP;
 189 }
 190
 191 static inline unsigned int kvm_vz_config6_user_wrmask(struct kvm_vcpu *vcpu)
 192 {
 193         return kvm_vz_config6_guest_wrmask(vcpu) |
 194                 LOONGSON_CONF6_SFBEN | LOONGSON_CONF6_FTLBDIS;
 195 }
 196
 197 static gpa_t kvm_vz_gva_to_gpa_cb(gva_t gva)
 198 {
 199         /* VZ guest has already converted gva to gpa */
 200         return gva;
 201 }
 202
 203 static void kvm_vz_queue_irq(struct kvm_vcpu *vcpu, unsigned int priority)
 204 {
 205         set_bit(priority, &vcpu->arch.pending_exceptions);
 206         clear_bit(priority, &vcpu->arch.pending_exceptions_clr);
 207 }
 208
 209 static void kvm_vz_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int priority)
 210 {
 211         clear_bit(priority, &vcpu->arch.pending_exceptions);
 212         set_bit(priority, &vcpu->arch.pending_exceptions_clr);
 213 }
 214
 215 static void kvm_vz_queue_timer_int_cb(struct kvm_vcpu *vcpu)
 216 {
 217         /*
 218          * timer expiry is asynchronous to vcpu execution therefore defer guest
 219          * cp0 accesses
 220          */
 221         kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER);
 222 }
 223
 224 static void kvm_vz_dequeue_timer_int_cb(struct kvm_vcpu *vcpu)
 225 {
 226         /*
 227          * timer expiry is asynchronous to vcpu execution therefore defer guest
 228          * cp0 accesses
 229          */
 230         kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_TIMER);
 231 }
 232
 233 static void kvm_vz_queue_io_int_cb(struct kvm_vcpu *vcpu,
 234                                    struct kvm_mips_interrupt *irq)
 235 {
 236         int intr = (int)irq->irq;
 237
 238         /*
 239          * interrupts are asynchronous to vcpu execution therefore defer guest
 240          * cp0 accesses
 241          */
 242         kvm_vz_queue_irq(vcpu, kvm_irq_to_priority(intr));
 243 }
 244
 245 static void kvm_vz_dequeue_io_int_cb(struct kvm_vcpu *vcpu,
 246                                      struct kvm_mips_interrupt *irq)
 247 {
 248         int intr = (int)irq->irq;
 249
 250         /*
 251          * interrupts are asynchronous to vcpu execution therefore defer guest
 252          * cp0 accesses
 253          */
 254         kvm_vz_dequeue_irq(vcpu, kvm_irq_to_priority(-intr));
 255 }
 256
 257 static int kvm_vz_irq_deliver_cb(struct kvm_vcpu *vcpu, unsigned int priority,
 258                                  u32 cause)
 259 {
 260         u32 irq = (priority < MIPS_EXC_MAX) ?
 261                 kvm_priority_to_irq[priority] : 0;
 262
 263         switch (priority) {
 264         case MIPS_EXC_INT_TIMER:
 265                 set_gc0_cause(C_TI);
 266                 break;
 267
 268         case MIPS_EXC_INT_IO_1:
 269         case MIPS_EXC_INT_IO_2:
 270         case MIPS_EXC_INT_IPI_1:
 271         case MIPS_EXC_INT_IPI_2:
 272                 if (cpu_has_guestctl2)
 273                         set_c0_guestctl2(irq);
 274                 else
 275                         set_gc0_cause(irq);
 276                 break;
 277
 278         default:
 279                 break;
 280         }
 281
 282         clear_bit(priority, &vcpu->arch.pending_exceptions);
 283         return 1;
 284 }
 285
 286 static int kvm_vz_irq_clear_cb(struct kvm_vcpu *vcpu, unsigned int priority,
 287                                u32 cause)
 288 {
 289         u32 irq = (priority < MIPS_EXC_MAX) ?
 290                 kvm_priority_to_irq[priority] : 0;
 291
 292         switch (priority) {
 293         case MIPS_EXC_INT_TIMER:
 294                 /*
 295                  * Call to kvm_write_c0_guest_compare() clears Cause.TI in
 296                  * kvm_mips_emulate_CP0(). Explicitly clear irq associated with
 297                  * Cause.IP[IPTI] if GuestCtl2 virtual interrupt register not
 298                  * supported or if not using GuestCtl2 Hardware Clear.
 299                  */
 300                 if (cpu_has_guestctl2) {
 301                         if (!(read_c0_guestctl2() & (irq << 14)))
 302                                 clear_c0_guestctl2(irq);
 303                 } else {
 304                         clear_gc0_cause(irq);
 305                 }
 306                 break;
 307
 308         case MIPS_EXC_INT_IO_1:
 309         case MIPS_EXC_INT_IO_2:
 310         case MIPS_EXC_INT_IPI_1:
 311         case MIPS_EXC_INT_IPI_2:
 312                 /* Clear GuestCtl2.VIP irq if not using Hardware Clear */
 313                 if (cpu_has_guestctl2) {
 314                         if (!(read_c0_guestctl2() & (irq << 14)))
 315                                 clear_c0_guestctl2(irq);
 316                 } else {
 317                         clear_gc0_cause(irq);
 318                 }
 319                 break;
 320
 321         default:
 322                 break;
 323         }
 324
 325         clear_bit(priority, &vcpu->arch.pending_exceptions_clr);
 326         return 1;
 327 }
 328
 329 /*
 330  * VZ guest timer handling.
 331  */
 332
 333 /**
 334  * kvm_vz_should_use_htimer() - Find whether to use the VZ hard guest timer.
 335  * @vcpu:       Virtual CPU.
 336  *
 337  * Returns:     true if the VZ GTOffset & real guest CP0_Count should be used
 338  *              instead of software emulation of guest timer.
 339  *              false otherwise.
 340  */
 341 static bool kvm_vz_should_use_htimer(struct kvm_vcpu *vcpu)
 342 {
 343         if (kvm_mips_count_disabled(vcpu))
 344                 return false;
 345
 346         /* Chosen frequency must match real frequency */
 347         if (mips_hpt_frequency != vcpu->arch.count_hz)
 348                 return false;
 349
 350         /* We don't support a CP0_GTOffset with fewer bits than CP0_Count */
 351         if (current_cpu_data.gtoffset_mask != 0xffffffff)
 352                 return false;
 353
 354         return true;
 355 }
 356
 357 /**
 358  * _kvm_vz_restore_stimer() - Restore soft timer state.
 359  * @vcpu:       Virtual CPU.
 360  * @compare:    CP0_Compare register value, restored by caller.
 361  * @cause:      CP0_Cause register to restore.
 362  *
 363  * Restore VZ state relating to the soft timer. The hard timer can be enabled
 364  * later.
 365  */
 366 static void _kvm_vz_restore_stimer(struct kvm_vcpu *vcpu, u32 compare,
 367                                    u32 cause)
 368 {
 369         /*
 370          * Avoid spurious counter interrupts by setting Guest CP0_Count to just
 371          * after Guest CP0_Compare.
 372          */
 373         write_c0_gtoffset(compare - read_c0_count());
 374
 375         back_to_back_c0_hazard();
 376         write_gc0_cause(cause);
 377 }
 378
 379 /**
 380  * _kvm_vz_restore_htimer() - Restore hard timer state.
 381  * @vcpu:       Virtual CPU.
 382  * @compare:    CP0_Compare register value, restored by caller.
 383  * @cause:      CP0_Cause register to restore.
 384  *
 385  * Restore hard timer Guest.Count & Guest.Cause taking care to preserve the
 386  * value of Guest.CP0_Cause.TI while restoring Guest.CP0_Cause.
 387  */
 388 static void _kvm_vz_restore_htimer(struct kvm_vcpu *vcpu,
 389                                    u32 compare, u32 cause)
 390 {
 391         u32 start_count, after_count;
 392         ktime_t freeze_time;
 393         unsigned long flags;
 394
 395         /*
 396          * Freeze the soft-timer and sync the guest CP0_Count with it. We do
 397          * this with interrupts disabled to avoid latency.
 398          */
 399         local_irq_save(flags);
 400         freeze_time = kvm_mips_freeze_hrtimer(vcpu, &start_count);
 401         write_c0_gtoffset(start_count - read_c0_count());
 402         local_irq_restore(flags);
 403
 404         /* restore guest CP0_Cause, as TI may already be set */
 405         back_to_back_c0_hazard();
 406         write_gc0_cause(cause);
 407
 408         /*
 409          * The above sequence isn't atomic and would result in lost timer
 410          * interrupts if we're not careful. Detect if a timer interrupt is due
 411          * and assert it.
 412          */
 413         back_to_back_c0_hazard();
 414         after_count = read_gc0_count();
 415         if (after_count - start_count > compare - start_count - 1)
 416                 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER);
 417 }
 418
 419 /**
 420  * kvm_vz_restore_timer() - Restore timer state.
 421  * @vcpu:       Virtual CPU.
 422  *
 423  * Restore soft timer state from saved context.
 424  */
 425 static void kvm_vz_restore_timer(struct kvm_vcpu *vcpu)
 426 {
 427         struct mips_coproc *cop0 = vcpu->arch.cop0;
 428         u32 cause, compare;
 429
 430         compare = kvm_read_sw_gc0_compare(cop0);
 431         cause = kvm_read_sw_gc0_cause(cop0);
 432
 433         write_gc0_compare(compare);
 434         _kvm_vz_restore_stimer(vcpu, compare, cause);
 435 }
 436
 437 /**
 438  * kvm_vz_acquire_htimer() - Switch to hard timer state.
 439  * @vcpu:       Virtual CPU.
 440  *
 441  * Restore hard timer state on top of existing soft timer state if possible.
 442  *
 443  * Since hard timer won't remain active over preemption, preemption should be
 444  * disabled by the caller.
 445  */
 446 void kvm_vz_acquire_htimer(struct kvm_vcpu *vcpu)
 447 {
 448         u32 gctl0;
 449
 450         gctl0 = read_c0_guestctl0();
 451         if (!(gctl0 & MIPS_GCTL0_GT) && kvm_vz_should_use_htimer(vcpu)) {
 452                 /* enable guest access to hard timer */
 453                 write_c0_guestctl0(gctl0 | MIPS_GCTL0_GT);
 454
 455                 _kvm_vz_restore_htimer(vcpu, read_gc0_compare(),
 456                                        read_gc0_cause());
 457         }
 458 }
 459
 460 /**
 461  * _kvm_vz_save_htimer() - Switch to software emulation of guest timer.
 462  * @vcpu:       Virtual CPU.
 463  * @compare:    Pointer to write compare value to.
 464  * @cause:      Pointer to write cause value to.
 465  *
 466  * Save VZ guest timer state and switch to software emulation of guest CP0
 467  * timer. The hard timer must already be in use, so preemption should be
 468  * disabled.
 469  */
 470 static void _kvm_vz_save_htimer(struct kvm_vcpu *vcpu,
 471                                 u32 *out_compare, u32 *out_cause)
 472 {
 473         u32 cause, compare, before_count, end_count;
 474         ktime_t before_time;
 475
 476         compare = read_gc0_compare();
 477         *out_compare = compare;
 478
 479         before_time = ktime_get();
 480
 481         /*
 482          * Record the CP0_Count *prior* to saving CP0_Cause, so we have a time
 483          * at which no pending timer interrupt is missing.
 484          */
 485         before_count = read_gc0_count();
 486         back_to_back_c0_hazard();
 487         cause = read_gc0_cause();
 488         *out_cause = cause;
 489
 490         /*
 491          * Record a final CP0_Count which we will transfer to the soft-timer.
 492          * This is recorded *after* saving CP0_Cause, so we don't get any timer
 493          * interrupts from just after the final CP0_Count point.
 494          */
 495         back_to_back_c0_hazard();
 496         end_count = read_gc0_count();
 497
 498         /*
 499          * The above sequence isn't atomic, so we could miss a timer interrupt
 500          * between reading CP0_Cause and end_count. Detect and record any timer
 501          * interrupt due between before_count and end_count.
 502          */
 503         if (end_count - before_count > compare - before_count - 1)
 504                 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER);
 505
 506         /*
 507          * Restore soft-timer, ignoring a small amount of negative drift due to
 508          * delay between freeze_hrtimer and setting CP0_GTOffset.
 509          */
 510         kvm_mips_restore_hrtimer(vcpu, before_time, end_count, -0x10000);
 511 }
 512
 513 /**
 514  * kvm_vz_save_timer() - Save guest timer state.
 515  * @vcpu:       Virtual CPU.
 516  *
 517  * Save VZ guest timer state and switch to soft guest timer if hard timer was in
 518  * use.
 519  */
 520 static void kvm_vz_save_timer(struct kvm_vcpu *vcpu)
 521 {
 522         struct mips_coproc *cop0 = vcpu->arch.cop0;
 523         u32 gctl0, compare, cause;
 524
 525         gctl0 = read_c0_guestctl0();
 526         if (gctl0 & MIPS_GCTL0_GT) {
 527                 /* disable guest use of hard timer */
 528                 write_c0_guestctl0(gctl0 & ~MIPS_GCTL0_GT);
 529
 530                 /* save hard timer state */
 531                 _kvm_vz_save_htimer(vcpu, &compare, &cause);
 532         } else {
 533                 compare = read_gc0_compare();
 534                 cause = read_gc0_cause();
 535         }
 536
 537         /* save timer-related state to VCPU context */
 538         kvm_write_sw_gc0_cause(cop0, cause);
 539         kvm_write_sw_gc0_compare(cop0, compare);
 540 }
 541
 542 /**
 543  * kvm_vz_lose_htimer() - Ensure hard guest timer is not in use.
 544  * @vcpu:       Virtual CPU.
 545  *
 546  * Transfers the state of the hard guest timer to the soft guest timer, leaving
 547  * guest state intact so it can continue to be used with the soft timer.
 548  */
 549 void kvm_vz_lose_htimer(struct kvm_vcpu *vcpu)
 550 {
 551         u32 gctl0, compare, cause;
 552
 553         preempt_disable();
 554         gctl0 = read_c0_guestctl0();
 555         if (gctl0 & MIPS_GCTL0_GT) {
 556                 /* disable guest use of timer */
 557                 write_c0_guestctl0(gctl0 & ~MIPS_GCTL0_GT);
 558
 559                 /* switch to soft timer */
 560                 _kvm_vz_save_htimer(vcpu, &compare, &cause);
 561
 562                 /* leave soft timer in usable state */
 563                 _kvm_vz_restore_stimer(vcpu, compare, cause);
 564         }
 565         preempt_enable();
 566 }
 567
 568 /**
 569  * is_eva_access() - Find whether an instruction is an EVA memory accessor.
 570  * @inst:       32-bit instruction encoding.
 571  *
 572  * Finds whether @inst encodes an EVA memory access instruction, which would
 573  * indicate that emulation of it should access the user mode address space
 574  * instead of the kernel mode address space. This matters for MUSUK segments
 575  * which are TLB mapped for user mode but unmapped for kernel mode.
 576  *
 577  * Returns:     Whether @inst encodes an EVA accessor instruction.
 578  */
 579 static bool is_eva_access(union mips_instruction inst)
 580 {
 581         if (inst.spec3_format.opcode != spec3_op)
 582                 return false;
 583
 584         switch (inst.spec3_format.func) {
 585         case lwle_op:
 586         case lwre_op:
 587         case cachee_op:
 588         case sbe_op:
 589         case she_op:
 590         case sce_op:
 591         case swe_op:
 592         case swle_op:
 593         case swre_op:
 594         case prefe_op:
 595         case lbue_op:
 596         case lhue_op:
 597         case lbe_op:
 598         case lhe_op:
 599         case lle_op:
 600         case lwe_op:
 601                 return true;
 602         default:
 603                 return false;
 604         }
 605 }
 606
 607 /**
 608  * is_eva_am_mapped() - Find whether an access mode is mapped.
 609  * @vcpu:       KVM VCPU state.
 610  * @am:         3-bit encoded access mode.
 611  * @eu:         Segment becomes unmapped and uncached when Status.ERL=1.
 612  *
 613  * Decode @am to find whether it encodes a mapped segment for the current VCPU
 614  * state. Where necessary @eu and the actual instruction causing the fault are
 615  * taken into account to make the decision.
 616  *
 617  * Returns:     Whether the VCPU faulted on a TLB mapped address.
 618  */
 619 static bool is_eva_am_mapped(struct kvm_vcpu *vcpu, unsigned int am, bool eu)
 620 {
 621         u32 am_lookup;
 622         int err;
 623
 624         /*
 625          * Interpret access control mode. We assume address errors will already
 626          * have been caught by the guest, leaving us with:
 627          *      AM      UM  SM  KM  31..24 23..16
 628          * UK    0 000          Unm   0      0
 629          * MK    1 001          TLB   1
 630          * MSK   2 010      TLB TLB   1
 631          * MUSK  3 011  TLB TLB TLB   1
 632          * MUSUK 4 100  TLB TLB Unm   0      1
 633          * USK   5 101      Unm Unm   0      0
 634          * -     6 110                0      0
 635          * UUSK  7 111  Unm Unm Unm   0      0
 636          *
 637          * We shift a magic value by AM across the sign bit to find if always
 638          * TLB mapped, and if not shift by 8 again to find if it depends on KM.
 639          */
 640         am_lookup = 0x70080000 << am;
 641         if ((s32)am_lookup < 0) {
 642                 /*
 643                  * MK, MSK, MUSK
 644                  * Always TLB mapped, unless SegCtl.EU && ERL
 645                  */
 646                 if (!eu || !(read_gc0_status() & ST0_ERL))
 647                         return true;
 648         } else {
 649                 am_lookup <<= 8;
 650                 if ((s32)am_lookup < 0) {
 651                         union mips_instruction inst;
 652                         unsigned int status;
 653                         u32 *opc;
 654
 655                         /*
 656                          * MUSUK
 657                          * TLB mapped if not in kernel mode
 658                          */
 659                         status = read_gc0_status();
 660                         if (!(status & (ST0_EXL | ST0_ERL)) &&
 661                             (status & ST0_KSU))
 662                                 return true;
 663                         /*
 664                          * EVA access instructions in kernel
 665                          * mode access user address space.
 666                          */
 667                         opc = (u32 *)vcpu->arch.pc;
 668                         if (vcpu->arch.host_cp0_cause & CAUSEF_BD)
 669                                 opc += 1;
 670                         err = kvm_get_badinstr(opc, vcpu, &inst.word);
 671                         if (!err && is_eva_access(inst))
 672                                 return true;
 673                 }
 674         }
 675
 676         return false;
 677 }
 678
 679 /**
 680  * kvm_vz_gva_to_gpa() - Convert valid GVA to GPA.
 681  * @vcpu:       KVM VCPU state.
 682  * @gva:        Guest virtual address to convert.
 683  * @gpa:        Output guest physical address.
 684  *
 685  * Convert a guest virtual address (GVA) which is valid according to the guest
 686  * context, to a guest physical address (GPA).
 687  *
 688  * Returns:     0 on success.
 689  *              -errno on failure.
 690  */
 691 static int kvm_vz_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
 692                              unsigned long *gpa)
 693 {
 694         u32 gva32 = gva;
 695         unsigned long segctl;
 696
 697         if ((long)gva == (s32)gva32) {
 698                 /* Handle canonical 32-bit virtual address */
 699                 if (cpu_guest_has_segments) {
 700                         unsigned long mask, pa;
 701
 702                         switch (gva32 >> 29) {
 703                         case 0:
 704                         case 1: /* CFG5 (1GB) */
 705                                 segctl = read_gc0_segctl2() >> 16;
 706                                 mask = (unsigned long)0xfc0000000ull;
 707                                 break;
 708                         case 2:
 709                         case 3: /* CFG4 (1GB) */
 710                                 segctl = read_gc0_segctl2();
 711                                 mask = (unsigned long)0xfc0000000ull;
 712                                 break;
 713                         case 4: /* CFG3 (512MB) */
 714                                 segctl = read_gc0_segctl1() >> 16;
 715                                 mask = (unsigned long)0xfe0000000ull;
 716                                 break;
 717                         case 5: /* CFG2 (512MB) */
 718                                 segctl = read_gc0_segctl1();
 719                                 mask = (unsigned long)0xfe0000000ull;
 720                                 break;
 721                         case 6: /* CFG1 (512MB) */
 722                                 segctl = read_gc0_segctl0() >> 16;
 723                                 mask = (unsigned long)0xfe0000000ull;
 724                                 break;
 725                         case 7: /* CFG0 (512MB) */
 726                                 segctl = read_gc0_segctl0();
 727                                 mask = (unsigned long)0xfe0000000ull;
 728                                 break;
 729                         default:
 730                                 /*
 731                                  * GCC 4.9 isn't smart enough to figure out that
 732                                  * segctl and mask are always initialised.
 733                                  */
 734                                 unreachable();
 735                         }
 736
 737                         if (is_eva_am_mapped(vcpu, (segctl >> 4) & 0x7,
 738                                              segctl & 0x0008))
 739                                 goto tlb_mapped;
 740
 741                         /* Unmapped, find guest physical address */
 742                         pa = (segctl << 20) & mask;
 743                         pa |= gva32 & ~mask;
 744                         *gpa = pa;
 745                         return 0;
 746                 } else if ((s32)gva32 < (s32)0xc0000000) {
 747                         /* legacy unmapped KSeg0 or KSeg1 */
 748                         *gpa = gva32 & 0x1fffffff;
 749                         return 0;
 750                 }
 751 #ifdef CONFIG_64BIT
 752         } else if ((gva & 0xc000000000000000) == 0x8000000000000000) {
 753                 /* XKPHYS */
 754                 if (cpu_guest_has_segments) {
 755                         /*
 756                          * Each of the 8 regions can be overridden by SegCtl2.XR
 757                          * to use SegCtl1.XAM.
 758                          */
 759                         segctl = read_gc0_segctl2();
 760                         if (segctl & (1ull << (56 + ((gva >> 59) & 0x7)))) {
 761                                 segctl = read_gc0_segctl1();
 762                                 if (is_eva_am_mapped(vcpu, (segctl >> 59) & 0x7,
 763                                                      0))
 764                                         goto tlb_mapped;
 765                         }
 766
 767                 }
 768                 /*
 769                  * Traditionally fully unmapped.
 770                  * Bits 61:59 specify the CCA, which we can just mask off here.
 771                  * Bits 58:PABITS should be zero, but we shouldn't have got here
 772                  * if it wasn't.
 773                  */
 774                 *gpa = gva & 0x07ffffffffffffff;
 775                 return 0;
 776 #endif
 777         }
 778
 779 tlb_mapped:
 780         return kvm_vz_guest_tlb_lookup(vcpu, gva, gpa);
 781 }
 782
 783 /**
 784  * kvm_vz_badvaddr_to_gpa() - Convert GVA BadVAddr from root exception to GPA.
 785  * @vcpu:       KVM VCPU state.
 786  * @badvaddr:   Root BadVAddr.
 787  * @gpa:        Output guest physical address.
 788  *
 789  * VZ implementations are permitted to report guest virtual addresses (GVA) in
 790  * BadVAddr on a root exception during guest execution, instead of the more
 791  * convenient guest physical addresses (GPA). When we get a GVA, this function
 792  * converts it to a GPA, taking into account guest segmentation and guest TLB
 793  * state.
 794  *
 795  * Returns:     0 on success.
 796  *              -errno on failure.
 797  */
 798 static int kvm_vz_badvaddr_to_gpa(struct kvm_vcpu *vcpu, unsigned long badvaddr,
 799                                   unsigned long *gpa)
 800 {
 801         unsigned int gexccode = (vcpu->arch.host_cp0_guestctl0 &
 802                                  MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT;
 803
 804         /* If BadVAddr is GPA, then all is well in the world */
 805         if (likely(gexccode == MIPS_GCTL0_GEXC_GPA)) {
 806                 *gpa = badvaddr;
 807                 return 0;
 808         }
 809
 810         /* Otherwise we'd expect it to be GVA ... */
 811         if (WARN(gexccode != MIPS_GCTL0_GEXC_GVA,
 812                  "Unexpected gexccode %#x\n", gexccode))
 813                 return -EINVAL;
 814
 815         /* ... and we need to perform the GVA->GPA translation in software */
 816         return kvm_vz_gva_to_gpa(vcpu, badvaddr, gpa);
 817 }
 818
 819 static int kvm_trap_vz_no_handler(struct kvm_vcpu *vcpu)
 820 {
 821         u32 *opc = (u32 *) vcpu->arch.pc;
 822         u32 cause = vcpu->arch.host_cp0_cause;
 823         u32 exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE;
 824         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
 825         u32 inst = 0;
 826
 827         /*
 828          *  Fetch the instruction.
 829          */
 830         if (cause & CAUSEF_BD)
 831                 opc += 1;
 832         kvm_get_badinstr(opc, vcpu, &inst);
 833
 834         kvm_err("Exception Code: %d not handled @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#x\n",
 835                 exccode, opc, inst, badvaddr,
 836                 read_gc0_status());
 837         kvm_arch_vcpu_dump_regs(vcpu);
 838         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 839         return RESUME_HOST;
 840 }
 841
 842 static unsigned long mips_process_maar(unsigned int op, unsigned long val)
 843 {
 844         /* Mask off unused bits */
 845         unsigned long mask = 0xfffff000 | MIPS_MAAR_S | MIPS_MAAR_VL;
 846
 847         if (read_gc0_pagegrain() & PG_ELPA)
 848                 mask |= 0x00ffffff00000000ull;
 849         if (cpu_guest_has_mvh)
 850                 mask |= MIPS_MAAR_VH;
 851
 852         /* Set or clear VH */
 853         if (op == mtc_op) {
 854                 /* clear VH */
 855                 val &= ~MIPS_MAAR_VH;
 856         } else if (op == dmtc_op) {
 857                 /* set VH to match VL */
 858                 val &= ~MIPS_MAAR_VH;
 859                 if (val & MIPS_MAAR_VL)
 860                         val |= MIPS_MAAR_VH;
 861         }
 862
 863         return val & mask;
 864 }
 865
 866 static void kvm_write_maari(struct kvm_vcpu *vcpu, unsigned long val)
 867 {
 868         struct mips_coproc *cop0 = vcpu->arch.cop0;
 869
 870         val &= MIPS_MAARI_INDEX;
 871         if (val == MIPS_MAARI_INDEX)
 872                 kvm_write_sw_gc0_maari(cop0, ARRAY_SIZE(vcpu->arch.maar) - 1);
 873         else if (val < ARRAY_SIZE(vcpu->arch.maar))
 874                 kvm_write_sw_gc0_maari(cop0, val);
 875 }
 876
 877 static enum emulation_result kvm_vz_gpsi_cop0(union mips_instruction inst,
 878                                               u32 *opc, u32 cause,
 879                                               struct kvm_vcpu *vcpu)
 880 {
 881         struct mips_coproc *cop0 = vcpu->arch.cop0;
 882         enum emulation_result er = EMULATE_DONE;
 883         u32 rt, rd, sel;
 884         unsigned long curr_pc;
 885         unsigned long val;
 886
 887         /*
 888          * Update PC and hold onto current PC in case there is
 889          * an error and we want to rollback the PC
 890          */
 891         curr_pc = vcpu->arch.pc;
 892         er = update_pc(vcpu, cause);
 893         if (er == EMULATE_FAIL)
 894                 return er;
 895
 896         if (inst.co_format.co) {
 897                 switch (inst.co_format.func) {
 898                 case wait_op:
 899                         er = kvm_mips_emul_wait(vcpu);
 900                         break;
 901                 default:
 902                         er = EMULATE_FAIL;
 903                 }
 904         } else {
 905                 rt = inst.c0r_format.rt;
 906                 rd = inst.c0r_format.rd;
 907                 sel = inst.c0r_format.sel;
 908
 909                 switch (inst.c0r_format.rs) {
 910                 case dmfc_op:
 911                 case mfc_op:
 912 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
 913                         cop0->stat[rd][sel]++;
 914 #endif
 915                         if (rd == MIPS_CP0_COUNT &&
 916                             sel == 0) {                 /* Count */
 917                                 val = kvm_mips_read_count(vcpu);
 918                         } else if (rd == MIPS_CP0_COMPARE &&
 919                                    sel == 0) {          /* Compare */
 920                                 val = read_gc0_compare();
 921                         } else if (rd == MIPS_CP0_LLADDR &&
 922                                    sel == 0) {          /* LLAddr */
 923                                 if (cpu_guest_has_rw_llb)
 924                                         val = read_gc0_lladdr() &
 925                                                 MIPS_LLADDR_LLB;
 926                                 else
 927                                         val = 0;
 928                         } else if (rd == MIPS_CP0_LLADDR &&
 929                                    sel == 1 &&          /* MAAR */
 930                                    cpu_guest_has_maar &&
 931                                    !cpu_guest_has_dyn_maar) {
 932                                 /* MAARI must be in range */
 933                                 BUG_ON(kvm_read_sw_gc0_maari(cop0) >=
 934                                                 ARRAY_SIZE(vcpu->arch.maar));
 935                                 val = vcpu->arch.maar[
 936                                         kvm_read_sw_gc0_maari(cop0)];
 937                         } else if ((rd == MIPS_CP0_PRID &&
 938                                     (sel == 0 ||        /* PRid */
 939                                      sel == 2 ||        /* CDMMBase */
 940                                      sel == 3)) ||      /* CMGCRBase */
 941                                    (rd == MIPS_CP0_STATUS &&
 942                                     (sel == 2 ||        /* SRSCtl */
 943                                      sel == 3)) ||      /* SRSMap */
 944                                    (rd == MIPS_CP0_CONFIG &&
 945                                     (sel == 6 ||        /* Config6 */
 946                                      sel == 7)) ||      /* Config7 */
 947                                    (rd == MIPS_CP0_LLADDR &&
 948                                     (sel == 2) &&       /* MAARI */
 949                                     cpu_guest_has_maar &&
 950                                     !cpu_guest_has_dyn_maar) ||
 951                                    (rd == MIPS_CP0_ERRCTL &&
 952                                     (sel == 0))) {      /* ErrCtl */
 953                                 val = cop0->reg[rd][sel];
 954 #ifdef CONFIG_CPU_LOONGSON64
 955                         } else if (rd == MIPS_CP0_DIAG &&
 956                                    (sel == 0)) {        /* Diag */
 957                                 val = cop0->reg[rd][sel];
 958 #endif
 959                         } else {
 960                                 val = 0;
 961                                 er = EMULATE_FAIL;
 962                         }
 963
 964                         if (er != EMULATE_FAIL) {
 965                                 /* Sign extend */
 966                                 if (inst.c0r_format.rs == mfc_op)
 967                                         val = (int)val;
 968                                 vcpu->arch.gprs[rt] = val;
 969                         }
 970
 971                         trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mfc_op) ?
 972                                         KVM_TRACE_MFC0 : KVM_TRACE_DMFC0,
 973                                       KVM_TRACE_COP0(rd, sel), val);
 974                         break;
 975
 976                 case dmtc_op:
 977                 case mtc_op:
 978 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
 979                         cop0->stat[rd][sel]++;
 980 #endif
 981                         val = vcpu->arch.gprs[rt];
 982                         trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mtc_op) ?
 983                                         KVM_TRACE_MTC0 : KVM_TRACE_DMTC0,
 984                                       KVM_TRACE_COP0(rd, sel), val);
 985
 986                         if (rd == MIPS_CP0_COUNT &&
 987                             sel == 0) {                 /* Count */
 988                                 kvm_vz_lose_htimer(vcpu);
 989                                 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]);
 990                         } else if (rd == MIPS_CP0_COMPARE &&
 991                                    sel == 0) {          /* Compare */
 992                                 kvm_mips_write_compare(vcpu,
 993                                                        vcpu->arch.gprs[rt],
 994                                                        true);
 995                         } else if (rd == MIPS_CP0_LLADDR &&
 996                                    sel == 0) {          /* LLAddr */
 997                                 /*
 998                                  * P5600 generates GPSI on guest MTC0 LLAddr.
 999                                  * Only allow the guest to clear LLB.
1000                                  */
1001                                 if (cpu_guest_has_rw_llb &&
1002                                     !(val & MIPS_LLADDR_LLB))
1003                                         write_gc0_lladdr(0);
1004                         } else if (rd == MIPS_CP0_LLADDR &&
1005                                    sel == 1 &&          /* MAAR */
1006                                    cpu_guest_has_maar &&
1007                                    !cpu_guest_has_dyn_maar) {
1008                                 val = mips_process_maar(inst.c0r_format.rs,
1009                                                         val);
1010
1011                                 /* MAARI must be in range */
1012                                 BUG_ON(kvm_read_sw_gc0_maari(cop0) >=
1013                                                 ARRAY_SIZE(vcpu->arch.maar));
1014                                 vcpu->arch.maar[kvm_read_sw_gc0_maari(cop0)] =
1015                                                                         val;
1016                         } else if (rd == MIPS_CP0_LLADDR &&
1017                                    (sel == 2) &&        /* MAARI */
1018                                    cpu_guest_has_maar &&
1019                                    !cpu_guest_has_dyn_maar) {
1020                                 kvm_write_maari(vcpu, val);
1021                         } else if (rd == MIPS_CP0_CONFIG &&
1022                                    (sel == 6)) {
1023                                 cop0->reg[rd][sel] = (int)val;
1024                         } else if (rd == MIPS_CP0_ERRCTL &&
1025                                    (sel == 0)) {        /* ErrCtl */
1026                                 /* ignore the written value */
1027 #ifdef CONFIG_CPU_LOONGSON64
1028                         } else if (rd == MIPS_CP0_DIAG &&
1029                                    (sel == 0)) {        /* Diag */
1030                                 unsigned long flags;
1031
1032                                 local_irq_save(flags);
1033                                 if (val & LOONGSON_DIAG_BTB) {
1034                                         /* Flush BTB */
1035                                         set_c0_diag(LOONGSON_DIAG_BTB);
1036                                 }
1037                                 if (val & LOONGSON_DIAG_ITLB) {
1038                                         /* Flush ITLB */
1039                                         set_c0_diag(LOONGSON_DIAG_ITLB);
1040                                 }
1041                                 if (val & LOONGSON_DIAG_DTLB) {
1042                                         /* Flush DTLB */
1043                                         set_c0_diag(LOONGSON_DIAG_DTLB);
1044                                 }
1045                                 if (val & LOONGSON_DIAG_VTLB) {
1046                                         /* Flush VTLB */
1047                                         kvm_loongson_clear_guest_vtlb();
1048                                 }
1049                                 if (val & LOONGSON_DIAG_FTLB) {
1050                                         /* Flush FTLB */
1051                                         kvm_loongson_clear_guest_ftlb();
1052                                 }
1053                                 local_irq_restore(flags);
1054 #endif
1055                         } else {
1056                                 er = EMULATE_FAIL;
1057                         }
1058                         break;
1059
1060                 default:
1061                         er = EMULATE_FAIL;
1062                         break;
1063                 }
1064         }
1065         /* Rollback PC only if emulation was unsuccessful */
1066         if (er == EMULATE_FAIL) {
1067                 kvm_err("[%#lx]%s: unsupported cop0 instruction 0x%08x\n",
1068                         curr_pc, __func__, inst.word);
1069
1070                 vcpu->arch.pc = curr_pc;
1071         }
1072
1073         return er;
1074 }
1075
1076 static enum emulation_result kvm_vz_gpsi_cache(union mips_instruction inst,
1077                                                u32 *opc, u32 cause,
1078                                                struct kvm_vcpu *vcpu)
1079 {
1080         enum emulation_result er = EMULATE_DONE;
1081         u32 cache, op_inst, op, base;
1082         s16 offset;
1083         struct kvm_vcpu_arch *arch = &vcpu->arch;
1084         unsigned long va, curr_pc;
1085
1086         /*
1087          * Update PC and hold onto current PC in case there is
1088          * an error and we want to rollback the PC
1089          */
1090         curr_pc = vcpu->arch.pc;
1091         er = update_pc(vcpu, cause);
1092         if (er == EMULATE_FAIL)
1093                 return er;
1094
1095         base = inst.i_format.rs;
1096         op_inst = inst.i_format.rt;
1097         if (cpu_has_mips_r6)
1098                 offset = inst.spec3_format.simmediate;
1099         else
1100                 offset = inst.i_format.simmediate;
1101         cache = op_inst & CacheOp_Cache;
1102         op = op_inst & CacheOp_Op;
1103
1104         va = arch->gprs[base] + offset;
1105
1106         kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1107                   cache, op, base, arch->gprs[base], offset);
1108
1109         /* Secondary or tirtiary cache ops ignored */
1110         if (cache != Cache_I && cache != Cache_D)
1111                 return EMULATE_DONE;
1112
1113         switch (op_inst) {
1114         case Index_Invalidate_I:
1115                 flush_icache_line_indexed(va);
1116                 return EMULATE_DONE;
1117         case Index_Writeback_Inv_D:
1118                 flush_dcache_line_indexed(va);
1119                 return EMULATE_DONE;
1120         case Hit_Invalidate_I:
1121         case Hit_Invalidate_D:
1122         case Hit_Writeback_Inv_D:
1123                 if (boot_cpu_type() == CPU_CAVIUM_OCTEON3) {
1124                         /* We can just flush entire icache */
1125                         local_flush_icache_range(0, 0);
1126                         return EMULATE_DONE;
1127                 }
1128
1129                 /* So far, other platforms support guest hit cache ops */
1130                 break;
1131         default:
1132                 break;
1133         }
1134
1135         kvm_err("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1136                 curr_pc, vcpu->arch.gprs[31], cache, op, base, arch->gprs[base],
1137                 offset);
1138         /* Rollback PC */
1139         vcpu->arch.pc = curr_pc;
1140
1141         return EMULATE_FAIL;
1142 }
1143
1144 #ifdef CONFIG_CPU_LOONGSON64
1145 static enum emulation_result kvm_vz_gpsi_lwc2(union mips_instruction inst,
1146                                               u32 *opc, u32 cause,
1147                                               struct kvm_vcpu *vcpu)
1148 {
1149         unsigned int rs, rd;
1150         unsigned int hostcfg;
1151         unsigned long curr_pc;
1152         enum emulation_result er = EMULATE_DONE;
1153
1154         /*
1155          * Update PC and hold onto current PC in case there is
1156          * an error and we want to rollback the PC
1157          */
1158         curr_pc = vcpu->arch.pc;
1159         er = update_pc(vcpu, cause);
1160         if (er == EMULATE_FAIL)
1161                 return er;
1162
1163         rs = inst.loongson3_lscsr_format.rs;
1164         rd = inst.loongson3_lscsr_format.rd;
1165         switch (inst.loongson3_lscsr_format.fr) {
1166         case 0x8:  /* Read CPUCFG */
1167                 ++vcpu->stat.vz_cpucfg_exits;
1168                 hostcfg = read_cpucfg(vcpu->arch.gprs[rs]);
1169
1170                 switch (vcpu->arch.gprs[rs]) {
1171                 case LOONGSON_CFG0:
1172                         vcpu->arch.gprs[rd] = 0x14c000;
1173                         break;
1174                 case LOONGSON_CFG1:
1175                         hostcfg &= (LOONGSON_CFG1_FP | LOONGSON_CFG1_MMI |
1176                                     LOONGSON_CFG1_MSA1 | LOONGSON_CFG1_MSA2 |
1177                                     LOONGSON_CFG1_SFBP);
1178                         vcpu->arch.gprs[rd] = hostcfg;
1179                         break;
1180                 case LOONGSON_CFG2:
1181                         hostcfg &= (LOONGSON_CFG2_LEXT1 | LOONGSON_CFG2_LEXT2 |
1182                                     LOONGSON_CFG2_LEXT3 | LOONGSON_CFG2_LSPW);
1183                         vcpu->arch.gprs[rd] = hostcfg;
1184                         break;
1185                 case LOONGSON_CFG3:
1186                         vcpu->arch.gprs[rd] = hostcfg;
1187                         break;
1188                 default:
1189                         /* Don't export any other advanced features to guest */
1190                         vcpu->arch.gprs[rd] = 0;
1191                         break;
1192                 }
1193                 break;
1194
1195         default:
1196                 kvm_err("lwc2 emulate not impl %d rs %lx @%lx\n",
1197                         inst.loongson3_lscsr_format.fr, vcpu->arch.gprs[rs], curr_pc);
1198                 er = EMULATE_FAIL;
1199                 break;
1200         }
1201
1202         /* Rollback PC only if emulation was unsuccessful */
1203         if (er == EMULATE_FAIL) {
1204                 kvm_err("[%#lx]%s: unsupported lwc2 instruction 0x%08x 0x%08x\n",
1205                         curr_pc, __func__, inst.word, inst.loongson3_lscsr_format.fr);
1206
1207                 vcpu->arch.pc = curr_pc;
1208         }
1209
1210         return er;
1211 }
1212 #endif
1213
1214 static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc,
1215                                                      struct kvm_vcpu *vcpu)
1216 {
1217         enum emulation_result er = EMULATE_DONE;
1218         struct kvm_vcpu_arch *arch = &vcpu->arch;
1219         union mips_instruction inst;
1220         int rd, rt, sel;
1221         int err;
1222
1223         /*
1224          *  Fetch the instruction.
1225          */
1226         if (cause & CAUSEF_BD)
1227                 opc += 1;
1228         err = kvm_get_badinstr(opc, vcpu, &inst.word);
1229         if (err)
1230                 return EMULATE_FAIL;
1231
1232         switch (inst.r_format.opcode) {
1233         case cop0_op:
1234                 er = kvm_vz_gpsi_cop0(inst, opc, cause, vcpu);
1235                 break;
1236 #ifndef CONFIG_CPU_MIPSR6
1237         case cache_op:
1238                 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1239                 er = kvm_vz_gpsi_cache(inst, opc, cause, vcpu);
1240                 break;
1241 #endif
1242 #ifdef CONFIG_CPU_LOONGSON64
1243         case lwc2_op:
1244                 er = kvm_vz_gpsi_lwc2(inst, opc, cause, vcpu);
1245                 break;
1246 #endif
1247         case spec3_op:
1248                 switch (inst.spec3_format.func) {
1249 #ifdef CONFIG_CPU_MIPSR6
1250                 case cache6_op:
1251                         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1252                         er = kvm_vz_gpsi_cache(inst, opc, cause, vcpu);
1253                         break;
1254 #endif
1255                 case rdhwr_op:
1256                         if (inst.r_format.rs || (inst.r_format.re >> 3))
1257                                 goto unknown;
1258
1259                         rd = inst.r_format.rd;
1260                         rt = inst.r_format.rt;
1261                         sel = inst.r_format.re & 0x7;
1262
1263                         switch (rd) {
1264                         case MIPS_HWR_CC:       /* Read count register */
1265                                 arch->gprs[rt] =
1266                                         (long)(int)kvm_mips_read_count(vcpu);
1267                                 break;
1268                         default:
1269                                 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR,
1270                                               KVM_TRACE_HWR(rd, sel), 0);
1271                                 goto unknown;
1272                         }
1273
1274                         trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR,
1275                                       KVM_TRACE_HWR(rd, sel), arch->gprs[rt]);
1276
1277                         er = update_pc(vcpu, cause);
1278                         break;
1279                 default:
1280                         goto unknown;
1281                 }
1282                 break;
1283 unknown:
1284
1285         default:
1286                 kvm_err("GPSI exception not supported (%p/%#x)\n",
1287                                 opc, inst.word);
1288                 kvm_arch_vcpu_dump_regs(vcpu);
1289                 er = EMULATE_FAIL;
1290                 break;
1291         }
1292
1293         return er;
1294 }
1295
1296 static enum emulation_result kvm_trap_vz_handle_gsfc(u32 cause, u32 *opc,
1297                                                      struct kvm_vcpu *vcpu)
1298 {
1299         enum emulation_result er = EMULATE_DONE;
1300         struct kvm_vcpu_arch *arch = &vcpu->arch;
1301         union mips_instruction inst;
1302         int err;
1303
1304         /*
1305          *  Fetch the instruction.
1306          */
1307         if (cause & CAUSEF_BD)
1308                 opc += 1;
1309         err = kvm_get_badinstr(opc, vcpu, &inst.word);
1310         if (err)
1311                 return EMULATE_FAIL;
1312
1313         /* complete MTC0 on behalf of guest and advance EPC */
1314         if (inst.c0r_format.opcode == cop0_op &&
1315             inst.c0r_format.rs == mtc_op &&
1316             inst.c0r_format.z == 0) {
1317                 int rt = inst.c0r_format.rt;
1318                 int rd = inst.c0r_format.rd;
1319                 int sel = inst.c0r_format.sel;
1320                 unsigned int val = arch->gprs[rt];
1321                 unsigned int old_val, change;
1322
1323                 trace_kvm_hwr(vcpu, KVM_TRACE_MTC0, KVM_TRACE_COP0(rd, sel),
1324                               val);
1325
1326                 if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1327                         /* FR bit should read as zero if no FPU */
1328                         if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1329                                 val &= ~(ST0_CU1 | ST0_FR);
1330
1331                         /*
1332                          * Also don't allow FR to be set if host doesn't support
1333                          * it.
1334                          */
1335                         if (!(boot_cpu_data.fpu_id & MIPS_FPIR_F64))
1336                                 val &= ~ST0_FR;
1337
1338                         old_val = read_gc0_status();
1339                         change = val ^ old_val;
1340
1341                         if (change & ST0_FR) {
1342                                 /*
1343                                  * FPU and Vector register state is made
1344                                  * UNPREDICTABLE by a change of FR, so don't
1345                                  * even bother saving it.
1346                                  */
1347                                 kvm_drop_fpu(vcpu);
1348                         }
1349
1350                         /*
1351                          * If MSA state is already live, it is undefined how it
1352                          * interacts with FR=0 FPU state, and we don't want to
1353                          * hit reserved instruction exceptions trying to save
1354                          * the MSA state later when CU=1 && FR=1, so play it
1355                          * safe and save it first.
1356                          */
1357                         if (change & ST0_CU1 && !(val & ST0_FR) &&
1358                             vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1359                                 kvm_lose_fpu(vcpu);
1360
1361                         write_gc0_status(val);
1362                 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1363                         u32 old_cause = read_gc0_cause();
1364                         u32 change = old_cause ^ val;
1365
1366                         /* DC bit enabling/disabling timer? */
1367                         if (change & CAUSEF_DC) {
1368                                 if (val & CAUSEF_DC) {
1369                                         kvm_vz_lose_htimer(vcpu);
1370                                         kvm_mips_count_disable_cause(vcpu);
1371                                 } else {
1372                                         kvm_mips_count_enable_cause(vcpu);
1373                                 }
1374                         }
1375
1376                         /* Only certain bits are RW to the guest */
1377                         change &= (CAUSEF_DC | CAUSEF_IV | CAUSEF_WP |
1378                                    CAUSEF_IP0 | CAUSEF_IP1);
1379
1380                         /* WP can only be cleared */
1381                         change &= ~CAUSEF_WP | old_cause;
1382
1383                         write_gc0_cause(old_cause ^ change);
1384                 } else if ((rd == MIPS_CP0_STATUS) && (sel == 1)) { /* IntCtl */
1385                         write_gc0_intctl(val);
1386                 } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1387                         old_val = read_gc0_config5();
1388                         change = val ^ old_val;
1389                         /* Handle changes in FPU/MSA modes */
1390                         preempt_disable();
1391
1392                         /*
1393                          * Propagate FRE changes immediately if the FPU
1394                          * context is already loaded.
1395                          */
1396                         if (change & MIPS_CONF5_FRE &&
1397                             vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1398                                 change_c0_config5(MIPS_CONF5_FRE, val);
1399
1400                         preempt_enable();
1401
1402                         val = old_val ^
1403                                 (change & kvm_vz_config5_guest_wrmask(vcpu));
1404                         write_gc0_config5(val);
1405                 } else {
1406                         kvm_err("Handle GSFC, unsupported field change @ %p: %#x\n",
1407                             opc, inst.word);
1408                         er = EMULATE_FAIL;
1409                 }
1410
1411                 if (er != EMULATE_FAIL)
1412                         er = update_pc(vcpu, cause);
1413         } else {
1414                 kvm_err("Handle GSFC, unrecognized instruction @ %p: %#x\n",
1415                         opc, inst.word);
1416                 er = EMULATE_FAIL;
1417         }
1418
1419         return er;
1420 }
1421
1422 static enum emulation_result kvm_trap_vz_handle_ghfc(u32 cause, u32 *opc,
1423                                                      struct kvm_vcpu *vcpu)
1424 {
1425         /*
1426          * Presumably this is due to MC (guest mode change), so lets trace some
1427          * relevant info.
1428          */
1429         trace_kvm_guest_mode_change(vcpu);
1430
1431         return EMULATE_DONE;
1432 }
1433
1434 static enum emulation_result kvm_trap_vz_handle_hc(u32 cause, u32 *opc,
1435                                                    struct kvm_vcpu *vcpu)
1436 {
1437         enum emulation_result er;
1438         union mips_instruction inst;
1439         unsigned long curr_pc;
1440         int err;
1441
1442         if (cause & CAUSEF_BD)
1443                 opc += 1;
1444         err = kvm_get_badinstr(opc, vcpu, &inst.word);
1445         if (err)
1446                 return EMULATE_FAIL;
1447
1448         /*
1449          * Update PC and hold onto current PC in case there is
1450          * an error and we want to rollback the PC
1451          */
1452         curr_pc = vcpu->arch.pc;
1453         er = update_pc(vcpu, cause);
1454         if (er == EMULATE_FAIL)
1455                 return er;
1456
1457         er = kvm_mips_emul_hypcall(vcpu, inst);
1458         if (er == EMULATE_FAIL)
1459                 vcpu->arch.pc = curr_pc;
1460
1461         return er;
1462 }
1463
1464 static enum emulation_result kvm_trap_vz_no_handler_guest_exit(u32 gexccode,
1465                                                         u32 cause,
1466                                                         u32 *opc,
1467                                                         struct kvm_vcpu *vcpu)
1468 {
1469         u32 inst;
1470
1471         /*
1472          *  Fetch the instruction.
1473          */
1474         if (cause & CAUSEF_BD)
1475                 opc += 1;
1476         kvm_get_badinstr(opc, vcpu, &inst);
1477
1478         kvm_err("Guest Exception Code: %d not yet handled @ PC: %p, inst: 0x%08x  Status: %#x\n",
1479                 gexccode, opc, inst, read_gc0_status());
1480
1481         return EMULATE_FAIL;
1482 }
1483
1484 static int kvm_trap_vz_handle_guest_exit(struct kvm_vcpu *vcpu)
1485 {
1486         u32 *opc = (u32 *) vcpu->arch.pc;
1487         u32 cause = vcpu->arch.host_cp0_cause;
1488         enum emulation_result er = EMULATE_DONE;
1489         u32 gexccode = (vcpu->arch.host_cp0_guestctl0 &
1490                         MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT;
1491         int ret = RESUME_GUEST;
1492
1493         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_GEXCCODE_BASE + gexccode);
1494         switch (gexccode) {
1495         case MIPS_GCTL0_GEXC_GPSI:
1496                 ++vcpu->stat.vz_gpsi_exits;
1497                 er = kvm_trap_vz_handle_gpsi(cause, opc, vcpu);
1498                 break;
1499         case MIPS_GCTL0_GEXC_GSFC:
1500                 ++vcpu->stat.vz_gsfc_exits;
1501                 er = kvm_trap_vz_handle_gsfc(cause, opc, vcpu);
1502                 break;
1503         case MIPS_GCTL0_GEXC_HC:
1504                 ++vcpu->stat.vz_hc_exits;
1505                 er = kvm_trap_vz_handle_hc(cause, opc, vcpu);
1506                 break;
1507         case MIPS_GCTL0_GEXC_GRR:
1508                 ++vcpu->stat.vz_grr_exits;
1509                 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
1510                                                        vcpu);
1511                 break;
1512         case MIPS_GCTL0_GEXC_GVA:
1513                 ++vcpu->stat.vz_gva_exits;
1514                 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
1515                                                        vcpu);
1516                 break;
1517         case MIPS_GCTL0_GEXC_GHFC:
1518                 ++vcpu->stat.vz_ghfc_exits;
1519                 er = kvm_trap_vz_handle_ghfc(cause, opc, vcpu);
1520                 break;
1521         case MIPS_GCTL0_GEXC_GPA:
1522                 ++vcpu->stat.vz_gpa_exits;
1523                 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
1524                                                        vcpu);
1525                 break;
1526         default:
1527                 ++vcpu->stat.vz_resvd_exits;
1528                 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
1529                                                        vcpu);
1530                 break;
1531
1532         }
1533
1534         if (er == EMULATE_DONE) {
1535                 ret = RESUME_GUEST;
1536         } else if (er == EMULATE_HYPERCALL) {
1537                 ret = kvm_mips_handle_hypcall(vcpu);
1538         } else {
1539                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1540                 ret = RESUME_HOST;
1541         }
1542         return ret;
1543 }
1544
1545 /**
1546  * kvm_trap_vz_handle_cop_unusuable() - Guest used unusable coprocessor.
1547  * @vcpu:       Virtual CPU context.
1548  *
1549  * Handle when the guest attempts to use a coprocessor which hasn't been allowed
1550  * by the root context.
1551  */
1552 static int kvm_trap_vz_handle_cop_unusable(struct kvm_vcpu *vcpu)
1553 {
1554         u32 cause = vcpu->arch.host_cp0_cause;
1555         enum emulation_result er = EMULATE_FAIL;
1556         int ret = RESUME_GUEST;
1557
1558         if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) {
1559                 /*
1560                  * If guest FPU not present, the FPU operation should have been
1561                  * treated as a reserved instruction!
1562                  * If FPU already in use, we shouldn't get this at all.
1563                  */
1564                 if (WARN_ON(!kvm_mips_guest_has_fpu(&vcpu->arch) ||
1565                             vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)) {
1566                         preempt_enable();
1567                         return EMULATE_FAIL;
1568                 }
1569
1570                 kvm_own_fpu(vcpu);
1571                 er = EMULATE_DONE;
1572         }
1573         /* other coprocessors not handled */
1574
1575         switch (er) {
1576         case EMULATE_DONE:
1577                 ret = RESUME_GUEST;
1578                 break;
1579
1580         case EMULATE_FAIL:
1581                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1582                 ret = RESUME_HOST;
1583                 break;
1584
1585         default:
1586                 BUG();
1587         }
1588         return ret;
1589 }
1590
1591 /**
1592  * kvm_trap_vz_handle_msa_disabled() - Guest used MSA while disabled in root.
1593  * @vcpu:       Virtual CPU context.
1594  *
1595  * Handle when the guest attempts to use MSA when it is disabled in the root
1596  * context.
1597  */
1598 static int kvm_trap_vz_handle_msa_disabled(struct kvm_vcpu *vcpu)
1599 {
1600         /*
1601          * If MSA not present or not exposed to guest or FR=0, the MSA operation
1602          * should have been treated as a reserved instruction!
1603          * Same if CU1=1, FR=0.
1604          * If MSA already in use, we shouldn't get this at all.
1605          */
1606         if (!kvm_mips_guest_has_msa(&vcpu->arch) ||
1607             (read_gc0_status() & (ST0_CU1 | ST0_FR)) == ST0_CU1 ||
1608             !(read_gc0_config5() & MIPS_CONF5_MSAEN) ||
1609             vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) {
1610                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1611                 return RESUME_HOST;
1612         }
1613
1614         kvm_own_msa(vcpu);
1615
1616         return RESUME_GUEST;
1617 }
1618
1619 static int kvm_trap_vz_handle_tlb_ld_miss(struct kvm_vcpu *vcpu)
1620 {
1621         struct kvm_run *run = vcpu->run;
1622         u32 *opc = (u32 *) vcpu->arch.pc;
1623         u32 cause = vcpu->arch.host_cp0_cause;
1624         ulong badvaddr = vcpu->arch.host_cp0_badvaddr;
1625         union mips_instruction inst;
1626         enum emulation_result er = EMULATE_DONE;
1627         int err, ret = RESUME_GUEST;
1628
1629         if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, false)) {
1630                 /* A code fetch fault doesn't count as an MMIO */
1631                 if (kvm_is_ifetch_fault(&vcpu->arch)) {
1632                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1633                         return RESUME_HOST;
1634                 }
1635
1636                 /* Fetch the instruction */
1637                 if (cause & CAUSEF_BD)
1638                         opc += 1;
1639                 err = kvm_get_badinstr(opc, vcpu, &inst.word);
1640                 if (err) {
1641                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1642                         return RESUME_HOST;
1643                 }
1644
1645                 /* Treat as MMIO */
1646                 er = kvm_mips_emulate_load(inst, cause, vcpu);
1647                 if (er == EMULATE_FAIL) {
1648                         kvm_err("Guest Emulate Load from MMIO space failed: PC: %p, BadVaddr: %#lx\n",
1649                                 opc, badvaddr);
1650                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1651                 }
1652         }
1653
1654         if (er == EMULATE_DONE) {
1655                 ret = RESUME_GUEST;
1656         } else if (er == EMULATE_DO_MMIO) {
1657                 run->exit_reason = KVM_EXIT_MMIO;
1658                 ret = RESUME_HOST;
1659         } else {
1660                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1661                 ret = RESUME_HOST;
1662         }
1663         return ret;
1664 }
1665
1666 static int kvm_trap_vz_handle_tlb_st_miss(struct kvm_vcpu *vcpu)
1667 {
1668         struct kvm_run *run = vcpu->run;
1669         u32 *opc = (u32 *) vcpu->arch.pc;
1670         u32 cause = vcpu->arch.host_cp0_cause;
1671         ulong badvaddr = vcpu->arch.host_cp0_badvaddr;
1672         union mips_instruction inst;
1673         enum emulation_result er = EMULATE_DONE;
1674         int err;
1675         int ret = RESUME_GUEST;
1676
1677         /* Just try the access again if we couldn't do the translation */
1678         if (kvm_vz_badvaddr_to_gpa(vcpu, badvaddr, &badvaddr))
1679                 return RESUME_GUEST;
1680         vcpu->arch.host_cp0_badvaddr = badvaddr;
1681
1682         if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, true)) {
1683                 /* Fetch the instruction */
1684                 if (cause & CAUSEF_BD)
1685                         opc += 1;
1686                 err = kvm_get_badinstr(opc, vcpu, &inst.word);
1687                 if (err) {
1688                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1689                         return RESUME_HOST;
1690                 }
1691
1692                 /* Treat as MMIO */
1693                 er = kvm_mips_emulate_store(inst, cause, vcpu);
1694                 if (er == EMULATE_FAIL) {
1695                         kvm_err("Guest Emulate Store to MMIO space failed: PC: %p, BadVaddr: %#lx\n",
1696                                 opc, badvaddr);
1697                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1698                 }
1699         }
1700
1701         if (er == EMULATE_DONE) {
1702                 ret = RESUME_GUEST;
1703         } else if (er == EMULATE_DO_MMIO) {
1704                 run->exit_reason = KVM_EXIT_MMIO;
1705                 ret = RESUME_HOST;
1706         } else {
1707                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1708                 ret = RESUME_HOST;
1709         }
1710         return ret;
1711 }
1712
1713 static u64 kvm_vz_get_one_regs[] = {
1714         KVM_REG_MIPS_CP0_INDEX,
1715         KVM_REG_MIPS_CP0_ENTRYLO0,
1716         KVM_REG_MIPS_CP0_ENTRYLO1,
1717         KVM_REG_MIPS_CP0_CONTEXT,
1718         KVM_REG_MIPS_CP0_PAGEMASK,
1719         KVM_REG_MIPS_CP0_PAGEGRAIN,
1720         KVM_REG_MIPS_CP0_WIRED,
1721         KVM_REG_MIPS_CP0_HWRENA,
1722         KVM_REG_MIPS_CP0_BADVADDR,
1723         KVM_REG_MIPS_CP0_COUNT,
1724         KVM_REG_MIPS_CP0_ENTRYHI,
1725         KVM_REG_MIPS_CP0_COMPARE,
1726         KVM_REG_MIPS_CP0_STATUS,
1727         KVM_REG_MIPS_CP0_INTCTL,
1728         KVM_REG_MIPS_CP0_CAUSE,
1729         KVM_REG_MIPS_CP0_EPC,
1730         KVM_REG_MIPS_CP0_PRID,
1731         KVM_REG_MIPS_CP0_EBASE,
1732         KVM_REG_MIPS_CP0_CONFIG,
1733         KVM_REG_MIPS_CP0_CONFIG1,
1734         KVM_REG_MIPS_CP0_CONFIG2,
1735         KVM_REG_MIPS_CP0_CONFIG3,
1736         KVM_REG_MIPS_CP0_CONFIG4,
1737         KVM_REG_MIPS_CP0_CONFIG5,
1738         KVM_REG_MIPS_CP0_CONFIG6,
1739 #ifdef CONFIG_64BIT
1740         KVM_REG_MIPS_CP0_XCONTEXT,
1741 #endif
1742         KVM_REG_MIPS_CP0_ERROREPC,
1743
1744         KVM_REG_MIPS_COUNT_CTL,
1745         KVM_REG_MIPS_COUNT_RESUME,
1746         KVM_REG_MIPS_COUNT_HZ,
1747 };
1748
1749 static u64 kvm_vz_get_one_regs_contextconfig[] = {
1750         KVM_REG_MIPS_CP0_CONTEXTCONFIG,
1751 #ifdef CONFIG_64BIT
1752         KVM_REG_MIPS_CP0_XCONTEXTCONFIG,
1753 #endif
1754 };
1755
1756 static u64 kvm_vz_get_one_regs_segments[] = {
1757         KVM_REG_MIPS_CP0_SEGCTL0,
1758         KVM_REG_MIPS_CP0_SEGCTL1,
1759         KVM_REG_MIPS_CP0_SEGCTL2,
1760 };
1761
1762 static u64 kvm_vz_get_one_regs_htw[] = {
1763         KVM_REG_MIPS_CP0_PWBASE,
1764         KVM_REG_MIPS_CP0_PWFIELD,
1765         KVM_REG_MIPS_CP0_PWSIZE,
1766         KVM_REG_MIPS_CP0_PWCTL,
1767 };
1768
1769 static u64 kvm_vz_get_one_regs_kscratch[] = {
1770         KVM_REG_MIPS_CP0_KSCRATCH1,
1771         KVM_REG_MIPS_CP0_KSCRATCH2,
1772         KVM_REG_MIPS_CP0_KSCRATCH3,
1773         KVM_REG_MIPS_CP0_KSCRATCH4,
1774         KVM_REG_MIPS_CP0_KSCRATCH5,
1775         KVM_REG_MIPS_CP0_KSCRATCH6,
1776 };
1777
1778 static unsigned long kvm_vz_num_regs(struct kvm_vcpu *vcpu)
1779 {
1780         unsigned long ret;
1781
1782         ret = ARRAY_SIZE(kvm_vz_get_one_regs);
1783         if (cpu_guest_has_userlocal)
1784                 ++ret;
1785         if (cpu_guest_has_badinstr)
1786                 ++ret;
1787         if (cpu_guest_has_badinstrp)
1788                 ++ret;
1789         if (cpu_guest_has_contextconfig)
1790                 ret += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig);
1791         if (cpu_guest_has_segments)
1792                 ret += ARRAY_SIZE(kvm_vz_get_one_regs_segments);
1793         if (cpu_guest_has_htw || cpu_guest_has_ldpte)
1794                 ret += ARRAY_SIZE(kvm_vz_get_one_regs_htw);
1795         if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar)
1796                 ret += 1 + ARRAY_SIZE(vcpu->arch.maar);
1797         ret += __arch_hweight8(cpu_data[0].guest.kscratch_mask);
1798
1799         return ret;
1800 }
1801
1802 static int kvm_vz_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices)
1803 {
1804         u64 index;
1805         unsigned int i;
1806
1807         if (copy_to_user(indices, kvm_vz_get_one_regs,
1808                          sizeof(kvm_vz_get_one_regs)))
1809                 return -EFAULT;
1810         indices += ARRAY_SIZE(kvm_vz_get_one_regs);
1811
1812         if (cpu_guest_has_userlocal) {
1813                 index = KVM_REG_MIPS_CP0_USERLOCAL;
1814                 if (copy_to_user(indices, &index, sizeof(index)))
1815                         return -EFAULT;
1816                 ++indices;
1817         }
1818         if (cpu_guest_has_badinstr) {
1819                 index = KVM_REG_MIPS_CP0_BADINSTR;
1820                 if (copy_to_user(indices, &index, sizeof(index)))
1821                         return -EFAULT;
1822                 ++indices;
1823         }
1824         if (cpu_guest_has_badinstrp) {
1825                 index = KVM_REG_MIPS_CP0_BADINSTRP;
1826                 if (copy_to_user(indices, &index, sizeof(index)))
1827                         return -EFAULT;
1828                 ++indices;
1829         }
1830         if (cpu_guest_has_contextconfig) {
1831                 if (copy_to_user(indices, kvm_vz_get_one_regs_contextconfig,
1832                                  sizeof(kvm_vz_get_one_regs_contextconfig)))
1833                         return -EFAULT;
1834                 indices += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig);
1835         }
1836         if (cpu_guest_has_segments) {
1837                 if (copy_to_user(indices, kvm_vz_get_one_regs_segments,
1838                                  sizeof(kvm_vz_get_one_regs_segments)))
1839                         return -EFAULT;
1840                 indices += ARRAY_SIZE(kvm_vz_get_one_regs_segments);
1841         }
1842         if (cpu_guest_has_htw || cpu_guest_has_ldpte) {
1843                 if (copy_to_user(indices, kvm_vz_get_one_regs_htw,
1844                                  sizeof(kvm_vz_get_one_regs_htw)))
1845                         return -EFAULT;
1846                 indices += ARRAY_SIZE(kvm_vz_get_one_regs_htw);
1847         }
1848         if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar) {
1849                 for (i = 0; i < ARRAY_SIZE(vcpu->arch.maar); ++i) {
1850                         index = KVM_REG_MIPS_CP0_MAAR(i);
1851                         if (copy_to_user(indices, &index, sizeof(index)))
1852                                 return -EFAULT;
1853                         ++indices;
1854                 }
1855
1856                 index = KVM_REG_MIPS_CP0_MAARI;
1857                 if (copy_to_user(indices, &index, sizeof(index)))
1858                         return -EFAULT;
1859                 ++indices;
1860         }
1861         for (i = 0; i < 6; ++i) {
1862                 if (!cpu_guest_has_kscr(i + 2))
1863                         continue;
1864
1865                 if (copy_to_user(indices, &kvm_vz_get_one_regs_kscratch[i],
1866                                  sizeof(kvm_vz_get_one_regs_kscratch[i])))
1867                         return -EFAULT;
1868                 ++indices;
1869         }
1870
1871         return 0;
1872 }
1873
1874 static inline s64 entrylo_kvm_to_user(unsigned long v)
1875 {
1876         s64 mask, ret = v;
1877
1878         if (BITS_PER_LONG == 32) {
1879                 /*
1880                  * KVM API exposes 64-bit version of the register, so move the
1881                  * RI/XI bits up into place.
1882                  */
1883                 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI;
1884                 ret &= ~mask;
1885                 ret |= ((s64)v & mask) << 32;
1886         }
1887         return ret;
1888 }
1889
1890 static inline unsigned long entrylo_user_to_kvm(s64 v)
1891 {
1892         unsigned long mask, ret = v;
1893
1894         if (BITS_PER_LONG == 32) {
1895                 /*
1896                  * KVM API exposes 64-bit versiono of the register, so move the
1897                  * RI/XI bits down into place.
1898                  */
1899                 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI;
1900                 ret &= ~mask;
1901                 ret |= (v >> 32) & mask;
1902         }
1903         return ret;
1904 }
1905
1906 static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu,
1907                               const struct kvm_one_reg *reg,
1908                               s64 *v)
1909 {
1910         struct mips_coproc *cop0 = vcpu->arch.cop0;
1911         unsigned int idx;
1912
1913         switch (reg->id) {
1914         case KVM_REG_MIPS_CP0_INDEX:
1915                 *v = (long)read_gc0_index();
1916                 break;
1917         case KVM_REG_MIPS_CP0_ENTRYLO0:
1918                 *v = entrylo_kvm_to_user(read_gc0_entrylo0());
1919                 break;
1920         case KVM_REG_MIPS_CP0_ENTRYLO1:
1921                 *v = entrylo_kvm_to_user(read_gc0_entrylo1());
1922                 break;
1923         case KVM_REG_MIPS_CP0_CONTEXT:
1924                 *v = (long)read_gc0_context();
1925                 break;
1926         case KVM_REG_MIPS_CP0_CONTEXTCONFIG:
1927                 if (!cpu_guest_has_contextconfig)
1928                         return -EINVAL;
1929                 *v = read_gc0_contextconfig();
1930                 break;
1931         case KVM_REG_MIPS_CP0_USERLOCAL:
1932                 if (!cpu_guest_has_userlocal)
1933                         return -EINVAL;
1934                 *v = read_gc0_userlocal();
1935                 break;
1936 #ifdef CONFIG_64BIT
1937         case KVM_REG_MIPS_CP0_XCONTEXTCONFIG:
1938                 if (!cpu_guest_has_contextconfig)
1939                         return -EINVAL;
1940                 *v = read_gc0_xcontextconfig();
1941                 break;
1942 #endif
1943         case KVM_REG_MIPS_CP0_PAGEMASK:
1944                 *v = (long)read_gc0_pagemask();
1945                 break;
1946         case KVM_REG_MIPS_CP0_PAGEGRAIN:
1947                 *v = (long)read_gc0_pagegrain();
1948                 break;
1949         case KVM_REG_MIPS_CP0_SEGCTL0:
1950                 if (!cpu_guest_has_segments)
1951                         return -EINVAL;
1952                 *v = read_gc0_segctl0();
1953                 break;
1954         case KVM_REG_MIPS_CP0_SEGCTL1:
1955                 if (!cpu_guest_has_segments)
1956                         return -EINVAL;
1957                 *v = read_gc0_segctl1();
1958                 break;
1959         case KVM_REG_MIPS_CP0_SEGCTL2:
1960                 if (!cpu_guest_has_segments)
1961                         return -EINVAL;
1962                 *v = read_gc0_segctl2();
1963                 break;
1964         case KVM_REG_MIPS_CP0_PWBASE:
1965                 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
1966                         return -EINVAL;
1967                 *v = read_gc0_pwbase();
1968                 break;
1969         case KVM_REG_MIPS_CP0_PWFIELD:
1970                 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
1971                         return -EINVAL;
1972                 *v = read_gc0_pwfield();
1973                 break;
1974         case KVM_REG_MIPS_CP0_PWSIZE:
1975                 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
1976                         return -EINVAL;
1977                 *v = read_gc0_pwsize();
1978                 break;
1979         case KVM_REG_MIPS_CP0_WIRED:
1980                 *v = (long)read_gc0_wired();
1981                 break;
1982         case KVM_REG_MIPS_CP0_PWCTL:
1983                 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
1984                         return -EINVAL;
1985                 *v = read_gc0_pwctl();
1986                 break;
1987         case KVM_REG_MIPS_CP0_HWRENA:
1988                 *v = (long)read_gc0_hwrena();
1989                 break;
1990         case KVM_REG_MIPS_CP0_BADVADDR:
1991                 *v = (long)read_gc0_badvaddr();
1992                 break;
1993         case KVM_REG_MIPS_CP0_BADINSTR:
1994                 if (!cpu_guest_has_badinstr)
1995                         return -EINVAL;
1996                 *v = read_gc0_badinstr();
1997                 break;
1998         case KVM_REG_MIPS_CP0_BADINSTRP:
1999                 if (!cpu_guest_has_badinstrp)
2000                         return -EINVAL;
2001                 *v = read_gc0_badinstrp();
2002                 break;
2003         case KVM_REG_MIPS_CP0_COUNT:
2004                 *v = kvm_mips_read_count(vcpu);
2005                 break;
2006         case KVM_REG_MIPS_CP0_ENTRYHI:
2007                 *v = (long)read_gc0_entryhi();
2008                 break;
2009         case KVM_REG_MIPS_CP0_COMPARE:
2010                 *v = (long)read_gc0_compare();
2011                 break;
2012         case KVM_REG_MIPS_CP0_STATUS:
2013                 *v = (long)read_gc0_status();
2014                 break;
2015         case KVM_REG_MIPS_CP0_INTCTL:
2016                 *v = read_gc0_intctl();
2017                 break;
2018         case KVM_REG_MIPS_CP0_CAUSE:
2019                 *v = (long)read_gc0_cause();
2020                 break;
2021         case KVM_REG_MIPS_CP0_EPC:
2022                 *v = (long)read_gc0_epc();
2023                 break;
2024         case KVM_REG_MIPS_CP0_PRID:
2025                 switch (boot_cpu_type()) {
2026                 case CPU_CAVIUM_OCTEON3:
2027                         /* Octeon III has a read-only guest.PRid */
2028                         *v = read_gc0_prid();
2029                         break;
2030                 default:
2031                         *v = (long)kvm_read_c0_guest_prid(cop0);
2032                         break;
2033                 }
2034                 break;
2035         case KVM_REG_MIPS_CP0_EBASE:
2036                 *v = kvm_vz_read_gc0_ebase();
2037                 break;
2038         case KVM_REG_MIPS_CP0_CONFIG:
2039                 *v = read_gc0_config();
2040                 break;
2041         case KVM_REG_MIPS_CP0_CONFIG1:
2042                 if (!cpu_guest_has_conf1)
2043                         return -EINVAL;
2044                 *v = read_gc0_config1();
2045                 break;
2046         case KVM_REG_MIPS_CP0_CONFIG2:
2047                 if (!cpu_guest_has_conf2)
2048                         return -EINVAL;
2049                 *v = read_gc0_config2();
2050                 break;
2051         case KVM_REG_MIPS_CP0_CONFIG3:
2052                 if (!cpu_guest_has_conf3)
2053                         return -EINVAL;
2054                 *v = read_gc0_config3();
2055                 break;
2056         case KVM_REG_MIPS_CP0_CONFIG4:
2057                 if (!cpu_guest_has_conf4)
2058                         return -EINVAL;
2059                 *v = read_gc0_config4();
2060                 break;
2061         case KVM_REG_MIPS_CP0_CONFIG5:
2062                 if (!cpu_guest_has_conf5)
2063                         return -EINVAL;
2064                 *v = read_gc0_config5();
2065                 break;
2066         case KVM_REG_MIPS_CP0_CONFIG6:
2067                 *v = kvm_read_sw_gc0_config6(cop0);
2068                 break;
2069         case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f):
2070                 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
2071                         return -EINVAL;
2072                 idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0);
2073                 if (idx >= ARRAY_SIZE(vcpu->arch.maar))
2074                         return -EINVAL;
2075                 *v = vcpu->arch.maar[idx];
2076                 break;
2077         case KVM_REG_MIPS_CP0_MAARI:
2078                 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
2079                         return -EINVAL;
2080                 *v = kvm_read_sw_gc0_maari(vcpu->arch.cop0);
2081                 break;
2082 #ifdef CONFIG_64BIT
2083         case KVM_REG_MIPS_CP0_XCONTEXT:
2084                 *v = read_gc0_xcontext();
2085                 break;
2086 #endif
2087         case KVM_REG_MIPS_CP0_ERROREPC:
2088                 *v = (long)read_gc0_errorepc();
2089                 break;
2090         case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6:
2091                 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2;
2092                 if (!cpu_guest_has_kscr(idx))
2093                         return -EINVAL;
2094                 switch (idx) {
2095                 case 2:
2096                         *v = (long)read_gc0_kscratch1();
2097                         break;
2098                 case 3:
2099                         *v = (long)read_gc0_kscratch2();
2100                         break;
2101                 case 4:
2102                         *v = (long)read_gc0_kscratch3();
2103                         break;
2104                 case 5:
2105                         *v = (long)read_gc0_kscratch4();
2106                         break;
2107                 case 6:
2108                         *v = (long)read_gc0_kscratch5();
2109                         break;
2110                 case 7:
2111                         *v = (long)read_gc0_kscratch6();
2112                         break;
2113                 }
2114                 break;
2115         case KVM_REG_MIPS_COUNT_CTL:
2116                 *v = vcpu->arch.count_ctl;
2117                 break;
2118         case KVM_REG_MIPS_COUNT_RESUME:
2119                 *v = ktime_to_ns(vcpu->arch.count_resume);
2120                 break;
2121         case KVM_REG_MIPS_COUNT_HZ:
2122                 *v = vcpu->arch.count_hz;
2123                 break;
2124         default:
2125                 return -EINVAL;
2126         }
2127         return 0;
2128 }
2129
2130 static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu,
2131                               const struct kvm_one_reg *reg,
2132                               s64 v)
2133 {
2134         struct mips_coproc *cop0 = vcpu->arch.cop0;
2135         unsigned int idx;
2136         int ret = 0;
2137         unsigned int cur, change;
2138
2139         switch (reg->id) {
2140         case KVM_REG_MIPS_CP0_INDEX:
2141                 write_gc0_index(v);
2142                 break;
2143         case KVM_REG_MIPS_CP0_ENTRYLO0:
2144                 write_gc0_entrylo0(entrylo_user_to_kvm(v));
2145                 break;
2146         case KVM_REG_MIPS_CP0_ENTRYLO1:
2147                 write_gc0_entrylo1(entrylo_user_to_kvm(v));
2148                 break;
2149         case KVM_REG_MIPS_CP0_CONTEXT:
2150                 write_gc0_context(v);
2151                 break;
2152         case KVM_REG_MIPS_CP0_CONTEXTCONFIG:
2153                 if (!cpu_guest_has_contextconfig)
2154                         return -EINVAL;
2155                 write_gc0_contextconfig(v);
2156                 break;
2157         case KVM_REG_MIPS_CP0_USERLOCAL:
2158                 if (!cpu_guest_has_userlocal)
2159                         return -EINVAL;
2160                 write_gc0_userlocal(v);
2161                 break;
2162 #ifdef CONFIG_64BIT
2163         case KVM_REG_MIPS_CP0_XCONTEXTCONFIG:
2164                 if (!cpu_guest_has_contextconfig)
2165                         return -EINVAL;
2166                 write_gc0_xcontextconfig(v);
2167                 break;
2168 #endif
2169         case KVM_REG_MIPS_CP0_PAGEMASK:
2170                 write_gc0_pagemask(v);
2171                 break;
2172         case KVM_REG_MIPS_CP0_PAGEGRAIN:
2173                 write_gc0_pagegrain(v);
2174                 break;
2175         case KVM_REG_MIPS_CP0_SEGCTL0:
2176                 if (!cpu_guest_has_segments)
2177                         return -EINVAL;
2178                 write_gc0_segctl0(v);
2179                 break;
2180         case KVM_REG_MIPS_CP0_SEGCTL1:
2181                 if (!cpu_guest_has_segments)
2182                         return -EINVAL;
2183                 write_gc0_segctl1(v);
2184                 break;
2185         case KVM_REG_MIPS_CP0_SEGCTL2:
2186                 if (!cpu_guest_has_segments)
2187                         return -EINVAL;
2188                 write_gc0_segctl2(v);
2189                 break;
2190         case KVM_REG_MIPS_CP0_PWBASE:
2191                 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
2192                         return -EINVAL;
2193                 write_gc0_pwbase(v);
2194                 break;
2195         case KVM_REG_MIPS_CP0_PWFIELD:
2196                 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
2197                         return -EINVAL;
2198                 write_gc0_pwfield(v);
2199                 break;
2200         case KVM_REG_MIPS_CP0_PWSIZE:
2201                 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
2202                         return -EINVAL;
2203                 write_gc0_pwsize(v);
2204                 break;
2205         case KVM_REG_MIPS_CP0_WIRED:
2206                 change_gc0_wired(MIPSR6_WIRED_WIRED, v);
2207                 break;
2208         case KVM_REG_MIPS_CP0_PWCTL:
2209                 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte)
2210                         return -EINVAL;
2211                 write_gc0_pwctl(v);
2212                 break;
2213         case KVM_REG_MIPS_CP0_HWRENA:
2214                 write_gc0_hwrena(v);
2215                 break;
2216         case KVM_REG_MIPS_CP0_BADVADDR:
2217                 write_gc0_badvaddr(v);
2218                 break;
2219         case KVM_REG_MIPS_CP0_BADINSTR:
2220                 if (!cpu_guest_has_badinstr)
2221                         return -EINVAL;
2222                 write_gc0_badinstr(v);
2223                 break;
2224         case KVM_REG_MIPS_CP0_BADINSTRP:
2225                 if (!cpu_guest_has_badinstrp)
2226                         return -EINVAL;
2227                 write_gc0_badinstrp(v);
2228                 break;
2229         case KVM_REG_MIPS_CP0_COUNT:
2230                 kvm_mips_write_count(vcpu, v);
2231                 break;
2232         case KVM_REG_MIPS_CP0_ENTRYHI:
2233                 write_gc0_entryhi(v);
2234                 break;
2235         case KVM_REG_MIPS_CP0_COMPARE:
2236                 kvm_mips_write_compare(vcpu, v, false);
2237                 break;
2238         case KVM_REG_MIPS_CP0_STATUS:
2239                 write_gc0_status(v);
2240                 break;
2241         case KVM_REG_MIPS_CP0_INTCTL:
2242                 write_gc0_intctl(v);
2243                 break;
2244         case KVM_REG_MIPS_CP0_CAUSE:
2245                 /*
2246                  * If the timer is stopped or started (DC bit) it must look
2247                  * atomic with changes to the timer interrupt pending bit (TI).
2248                  * A timer interrupt should not happen in between.
2249                  */
2250                 if ((read_gc0_cause() ^ v) & CAUSEF_DC) {
2251                         if (v & CAUSEF_DC) {
2252                                 /* disable timer first */
2253                                 kvm_mips_count_disable_cause(vcpu);
2254                                 change_gc0_cause((u32)~CAUSEF_DC, v);
2255                         } else {
2256                                 /* enable timer last */
2257                                 change_gc0_cause((u32)~CAUSEF_DC, v);
2258                                 kvm_mips_count_enable_cause(vcpu);
2259                         }
2260                 } else {
2261                         write_gc0_cause(v);
2262                 }
2263                 break;
2264         case KVM_REG_MIPS_CP0_EPC:
2265                 write_gc0_epc(v);
2266                 break;
2267         case KVM_REG_MIPS_CP0_PRID:
2268                 switch (boot_cpu_type()) {
2269                 case CPU_CAVIUM_OCTEON3:
2270                         /* Octeon III has a guest.PRid, but its read-only */
2271                         break;
2272                 default:
2273                         kvm_write_c0_guest_prid(cop0, v);
2274                         break;
2275                 }
2276                 break;
2277         case KVM_REG_MIPS_CP0_EBASE:
2278                 kvm_vz_write_gc0_ebase(v);
2279                 break;
2280         case KVM_REG_MIPS_CP0_CONFIG:
2281                 cur = read_gc0_config();
2282                 change = (cur ^ v) & kvm_vz_config_user_wrmask(vcpu);
2283                 if (change) {
2284                         v = cur ^ change;
2285                         write_gc0_config(v);
2286                 }
2287                 break;
2288         case KVM_REG_MIPS_CP0_CONFIG1:
2289                 if (!cpu_guest_has_conf1)
2290                         break;
2291                 cur = read_gc0_config1();
2292                 change = (cur ^ v) & kvm_vz_config1_user_wrmask(vcpu);
2293                 if (change) {
2294                         v = cur ^ change;
2295                         write_gc0_config1(v);
2296                 }
2297                 break;
2298         case KVM_REG_MIPS_CP0_CONFIG2:
2299                 if (!cpu_guest_has_conf2)
2300                         break;
2301                 cur = read_gc0_config2();
2302                 change = (cur ^ v) & kvm_vz_config2_user_wrmask(vcpu);
2303                 if (change) {
2304                         v = cur ^ change;
2305                         write_gc0_config2(v);
2306                 }
2307                 break;
2308         case KVM_REG_MIPS_CP0_CONFIG3:
2309                 if (!cpu_guest_has_conf3)
2310                         break;
2311                 cur = read_gc0_config3();
2312                 change = (cur ^ v) & kvm_vz_config3_user_wrmask(vcpu);
2313                 if (change) {
2314                         v = cur ^ change;
2315                         write_gc0_config3(v);
2316                 }
2317                 break;
2318         case KVM_REG_MIPS_CP0_CONFIG4:
2319                 if (!cpu_guest_has_conf4)
2320                         break;
2321                 cur = read_gc0_config4();
2322                 change = (cur ^ v) & kvm_vz_config4_user_wrmask(vcpu);
2323                 if (change) {
2324                         v = cur ^ change;
2325                         write_gc0_config4(v);
2326                 }
2327                 break;
2328         case KVM_REG_MIPS_CP0_CONFIG5:
2329                 if (!cpu_guest_has_conf5)
2330                         break;
2331                 cur = read_gc0_config5();
2332                 change = (cur ^ v) & kvm_vz_config5_user_wrmask(vcpu);
2333                 if (change) {
2334                         v = cur ^ change;
2335                         write_gc0_config5(v);
2336                 }
2337                 break;
2338         case KVM_REG_MIPS_CP0_CONFIG6:
2339                 cur = kvm_read_sw_gc0_config6(cop0);
2340                 change = (cur ^ v) & kvm_vz_config6_user_wrmask(vcpu);
2341                 if (change) {
2342                         v = cur ^ change;
2343                         kvm_write_sw_gc0_config6(cop0, (int)v);
2344                 }
2345                 break;
2346         case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f):
2347                 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
2348                         return -EINVAL;
2349                 idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0);
2350                 if (idx >= ARRAY_SIZE(vcpu->arch.maar))
2351                         return -EINVAL;
2352                 vcpu->arch.maar[idx] = mips_process_maar(dmtc_op, v);
2353                 break;
2354         case KVM_REG_MIPS_CP0_MAARI:
2355                 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
2356                         return -EINVAL;
2357                 kvm_write_maari(vcpu, v);
2358                 break;
2359 #ifdef CONFIG_64BIT
2360         case KVM_REG_MIPS_CP0_XCONTEXT:
2361                 write_gc0_xcontext(v);
2362                 break;
2363 #endif
2364         case KVM_REG_MIPS_CP0_ERROREPC:
2365                 write_gc0_errorepc(v);
2366                 break;
2367         case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6:
2368                 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2;
2369                 if (!cpu_guest_has_kscr(idx))
2370                         return -EINVAL;
2371                 switch (idx) {
2372                 case 2:
2373                         write_gc0_kscratch1(v);
2374                         break;
2375                 case 3:
2376                         write_gc0_kscratch2(v);
2377                         break;
2378                 case 4:
2379                         write_gc0_kscratch3(v);
2380                         break;
2381                 case 5:
2382                         write_gc0_kscratch4(v);
2383                         break;
2384                 case 6:
2385                         write_gc0_kscratch5(v);
2386                         break;
2387                 case 7:
2388                         write_gc0_kscratch6(v);
2389                         break;
2390                 }
2391                 break;
2392         case KVM_REG_MIPS_COUNT_CTL:
2393                 ret = kvm_mips_set_count_ctl(vcpu, v);
2394                 break;
2395         case KVM_REG_MIPS_COUNT_RESUME:
2396                 ret = kvm_mips_set_count_resume(vcpu, v);
2397                 break;
2398         case KVM_REG_MIPS_COUNT_HZ:
2399                 ret = kvm_mips_set_count_hz(vcpu, v);
2400                 break;
2401         default:
2402                 return -EINVAL;
2403         }
2404         return ret;
2405 }
2406
2407 #define guestid_cache(cpu)      (cpu_data[cpu].guestid_cache)
2408 static void kvm_vz_get_new_guestid(unsigned long cpu, struct kvm_vcpu *vcpu)
2409 {
2410         unsigned long guestid = guestid_cache(cpu);
2411
2412         if (!(++guestid & GUESTID_MASK)) {
2413                 if (cpu_has_vtag_icache)
2414                         flush_icache_all();
2415
2416                 if (!guestid)           /* fix version if needed */
2417                         guestid = GUESTID_FIRST_VERSION;
2418
2419                 ++guestid;              /* guestid 0 reserved for root */
2420
2421                 /* start new guestid cycle */
2422                 kvm_vz_local_flush_roottlb_all_guests();
2423                 kvm_vz_local_flush_guesttlb_all();
2424         }
2425
2426         guestid_cache(cpu) = guestid;
2427 }
2428
2429 /* Returns 1 if the guest TLB may be clobbered */
2430 static int kvm_vz_check_requests(struct kvm_vcpu *vcpu, int cpu)
2431 {
2432         int ret = 0;
2433         int i;
2434
2435         if (!kvm_request_pending(vcpu))
2436                 return 0;
2437
2438         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2439                 if (cpu_has_guestid) {
2440                         /* Drop all GuestIDs for this VCPU */
2441                         for_each_possible_cpu(i)
2442                                 vcpu->arch.vzguestid[i] = 0;
2443                         /* This will clobber guest TLB contents too */
2444                         ret = 1;
2445                 }
2446                 /*
2447                  * For Root ASID Dealias (RAD) we don't do anything here, but we
2448                  * still need the request to ensure we recheck asid_flush_mask.
2449                  * We can still return 0 as only the root TLB will be affected
2450                  * by a root ASID flush.
2451                  */
2452         }
2453
2454         return ret;
2455 }
2456
2457 static void kvm_vz_vcpu_save_wired(struct kvm_vcpu *vcpu)
2458 {
2459         unsigned int wired = read_gc0_wired();
2460         struct kvm_mips_tlb *tlbs;
2461         int i;
2462
2463         /* Expand the wired TLB array if necessary */
2464         wired &= MIPSR6_WIRED_WIRED;
2465         if (wired > vcpu->arch.wired_tlb_limit) {
2466                 tlbs = krealloc(vcpu->arch.wired_tlb, wired *
2467                                 sizeof(*vcpu->arch.wired_tlb), GFP_ATOMIC);
2468                 if (WARN_ON(!tlbs)) {
2469                         /* Save whatever we can */
2470                         wired = vcpu->arch.wired_tlb_limit;
2471                 } else {
2472                         vcpu->arch.wired_tlb = tlbs;
2473                         vcpu->arch.wired_tlb_limit = wired;
2474                 }
2475         }
2476
2477         if (wired)
2478                 /* Save wired entries from the guest TLB */
2479                 kvm_vz_save_guesttlb(vcpu->arch.wired_tlb, 0, wired);
2480         /* Invalidate any dropped entries since last time */
2481         for (i = wired; i < vcpu->arch.wired_tlb_used; ++i) {
2482                 vcpu->arch.wired_tlb[i].tlb_hi = UNIQUE_GUEST_ENTRYHI(i);
2483                 vcpu->arch.wired_tlb[i].tlb_lo[0] = 0;
2484                 vcpu->arch.wired_tlb[i].tlb_lo[1] = 0;
2485                 vcpu->arch.wired_tlb[i].tlb_mask = 0;
2486         }
2487         vcpu->arch.wired_tlb_used = wired;
2488 }
2489
2490 static void kvm_vz_vcpu_load_wired(struct kvm_vcpu *vcpu)
2491 {
2492         /* Load wired entries into the guest TLB */
2493         if (vcpu->arch.wired_tlb)
2494                 kvm_vz_load_guesttlb(vcpu->arch.wired_tlb, 0,
2495                                      vcpu->arch.wired_tlb_used);
2496 }
2497
2498 static void kvm_vz_vcpu_load_tlb(struct kvm_vcpu *vcpu, int cpu)
2499 {
2500         struct kvm *kvm = vcpu->kvm;
2501         struct mm_struct *gpa_mm = &kvm->arch.gpa_mm;
2502         bool migrated;
2503
2504         /*
2505          * Are we entering guest context on a different CPU to last time?
2506          * If so, the VCPU's guest TLB state on this CPU may be stale.
2507          */
2508         migrated = (vcpu->arch.last_exec_cpu != cpu);
2509         vcpu->arch.last_exec_cpu = cpu;
2510
2511         /*
2512          * A vcpu's GuestID is set in GuestCtl1.ID when the vcpu is loaded and
2513          * remains set until another vcpu is loaded in.  As a rule GuestRID
2514          * remains zeroed when in root context unless the kernel is busy
2515          * manipulating guest tlb entries.
2516          */
2517         if (cpu_has_guestid) {
2518                 /*
2519                  * Check if our GuestID is of an older version and thus invalid.
2520                  *
2521                  * We also discard the stored GuestID if we've executed on
2522                  * another CPU, as the guest mappings may have changed without
2523                  * hypervisor knowledge.
2524                  */
2525                 if (migrated ||
2526                     (vcpu->arch.vzguestid[cpu] ^ guestid_cache(cpu)) &
2527                                         GUESTID_VERSION_MASK) {
2528                         kvm_vz_get_new_guestid(cpu, vcpu);
2529                         vcpu->arch.vzguestid[cpu] = guestid_cache(cpu);
2530                         trace_kvm_guestid_change(vcpu,
2531                                                  vcpu->arch.vzguestid[cpu]);
2532                 }
2533
2534                 /* Restore GuestID */
2535                 change_c0_guestctl1(GUESTID_MASK, vcpu->arch.vzguestid[cpu]);
2536         } else {
2537                 /*
2538                  * The Guest TLB only stores a single guest's TLB state, so
2539                  * flush it if another VCPU has executed on this CPU.
2540                  *
2541                  * We also flush if we've executed on another CPU, as the guest
2542                  * mappings may have changed without hypervisor knowledge.
2543                  */
2544                 if (migrated || last_exec_vcpu[cpu] != vcpu)
2545                         kvm_vz_local_flush_guesttlb_all();
2546                 last_exec_vcpu[cpu] = vcpu;
2547
2548                 /*
2549                  * Root ASID dealiases guest GPA mappings in the root TLB.
2550                  * Allocate new root ASID if needed.
2551                  */
2552                 if (cpumask_test_and_clear_cpu(cpu, &kvm->arch.asid_flush_mask))
2553                         get_new_mmu_context(gpa_mm);
2554                 else
2555                         check_mmu_context(gpa_mm);
2556         }
2557 }
2558
2559 static int kvm_vz_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2560 {
2561         struct mips_coproc *cop0 = vcpu->arch.cop0;
2562         bool migrated, all;
2563
2564         /*
2565          * Have we migrated to a different CPU?
2566          * If so, any old guest TLB state may be stale.
2567          */
2568         migrated = (vcpu->arch.last_sched_cpu != cpu);
2569
2570         /*
2571          * Was this the last VCPU to run on this CPU?
2572          * If not, any old guest state from this VCPU will have been clobbered.
2573          */
2574         all = migrated || (last_vcpu[cpu] != vcpu);
2575         last_vcpu[cpu] = vcpu;
2576
2577         /*
2578          * Restore CP0_Wired unconditionally as we clear it after use, and
2579          * restore wired guest TLB entries (while in guest context).
2580          */
2581         kvm_restore_gc0_wired(cop0);
2582         if (current->flags & PF_VCPU) {
2583                 tlbw_use_hazard();
2584                 kvm_vz_vcpu_load_tlb(vcpu, cpu);
2585                 kvm_vz_vcpu_load_wired(vcpu);
2586         }
2587
2588         /*
2589          * Restore timer state regardless, as e.g. Cause.TI can change over time
2590          * if left unmaintained.
2591          */
2592         kvm_vz_restore_timer(vcpu);
2593
2594         /* Set MC bit if we want to trace guest mode changes */
2595         if (kvm_trace_guest_mode_change)
2596                 set_c0_guestctl0(MIPS_GCTL0_MC);
2597         else
2598                 clear_c0_guestctl0(MIPS_GCTL0_MC);
2599
2600         /* Don't bother restoring registers multiple times unless necessary */
2601         if (!all)
2602                 return 0;
2603
2604         /*
2605          * Restore config registers first, as some implementations restrict
2606          * writes to other registers when the corresponding feature bits aren't
2607          * set. For example Status.CU1 cannot be set unless Config1.FP is set.
2608          */
2609         kvm_restore_gc0_config(cop0);
2610         if (cpu_guest_has_conf1)
2611                 kvm_restore_gc0_config1(cop0);
2612         if (cpu_guest_has_conf2)
2613                 kvm_restore_gc0_config2(cop0);
2614         if (cpu_guest_has_conf3)
2615                 kvm_restore_gc0_config3(cop0);
2616         if (cpu_guest_has_conf4)
2617                 kvm_restore_gc0_config4(cop0);
2618         if (cpu_guest_has_conf5)
2619                 kvm_restore_gc0_config5(cop0);
2620         if (cpu_guest_has_conf6)
2621                 kvm_restore_gc0_config6(cop0);
2622         if (cpu_guest_has_conf7)
2623                 kvm_restore_gc0_config7(cop0);
2624
2625         kvm_restore_gc0_index(cop0);
2626         kvm_restore_gc0_entrylo0(cop0);
2627         kvm_restore_gc0_entrylo1(cop0);
2628         kvm_restore_gc0_context(cop0);
2629         if (cpu_guest_has_contextconfig)
2630                 kvm_restore_gc0_contextconfig(cop0);
2631 #ifdef CONFIG_64BIT
2632         kvm_restore_gc0_xcontext(cop0);
2633         if (cpu_guest_has_contextconfig)
2634                 kvm_restore_gc0_xcontextconfig(cop0);
2635 #endif
2636         kvm_restore_gc0_pagemask(cop0);
2637         kvm_restore_gc0_pagegrain(cop0);
2638         kvm_restore_gc0_hwrena(cop0);
2639         kvm_restore_gc0_badvaddr(cop0);
2640         kvm_restore_gc0_entryhi(cop0);
2641         kvm_restore_gc0_status(cop0);
2642         kvm_restore_gc0_intctl(cop0);
2643         kvm_restore_gc0_epc(cop0);
2644         kvm_vz_write_gc0_ebase(kvm_read_sw_gc0_ebase(cop0));
2645         if (cpu_guest_has_userlocal)
2646                 kvm_restore_gc0_userlocal(cop0);
2647
2648         kvm_restore_gc0_errorepc(cop0);
2649
2650         /* restore KScratch registers if enabled in guest */
2651         if (cpu_guest_has_conf4) {
2652                 if (cpu_guest_has_kscr(2))
2653                         kvm_restore_gc0_kscratch1(cop0);
2654                 if (cpu_guest_has_kscr(3))
2655                         kvm_restore_gc0_kscratch2(cop0);
2656                 if (cpu_guest_has_kscr(4))
2657                         kvm_restore_gc0_kscratch3(cop0);
2658                 if (cpu_guest_has_kscr(5))
2659                         kvm_restore_gc0_kscratch4(cop0);
2660                 if (cpu_guest_has_kscr(6))
2661                         kvm_restore_gc0_kscratch5(cop0);
2662                 if (cpu_guest_has_kscr(7))
2663                         kvm_restore_gc0_kscratch6(cop0);
2664         }
2665
2666         if (cpu_guest_has_badinstr)
2667                 kvm_restore_gc0_badinstr(cop0);
2668         if (cpu_guest_has_badinstrp)
2669                 kvm_restore_gc0_badinstrp(cop0);
2670
2671         if (cpu_guest_has_segments) {
2672                 kvm_restore_gc0_segctl0(cop0);
2673                 kvm_restore_gc0_segctl1(cop0);
2674                 kvm_restore_gc0_segctl2(cop0);
2675         }
2676
2677         /* restore HTW registers */
2678         if (cpu_guest_has_htw || cpu_guest_has_ldpte) {
2679                 kvm_restore_gc0_pwbase(cop0);
2680                 kvm_restore_gc0_pwfield(cop0);
2681                 kvm_restore_gc0_pwsize(cop0);
2682                 kvm_restore_gc0_pwctl(cop0);
2683         }
2684
2685         /* restore Root.GuestCtl2 from unused Guest guestctl2 register */
2686         if (cpu_has_guestctl2)
2687                 write_c0_guestctl2(
2688                         cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL]);
2689
2690         /*
2691          * We should clear linked load bit to break interrupted atomics. This
2692          * prevents a SC on the next VCPU from succeeding by matching a LL on
2693          * the previous VCPU.
2694          */
2695         if (vcpu->kvm->created_vcpus > 1)
2696                 write_gc0_lladdr(0);
2697
2698         return 0;
2699 }
2700
2701 static int kvm_vz_vcpu_put(struct kvm_vcpu *vcpu, int cpu)
2702 {
2703         struct mips_coproc *cop0 = vcpu->arch.cop0;
2704
2705         if (current->flags & PF_VCPU)
2706                 kvm_vz_vcpu_save_wired(vcpu);
2707
2708         kvm_lose_fpu(vcpu);
2709
2710         kvm_save_gc0_index(cop0);
2711         kvm_save_gc0_entrylo0(cop0);
2712         kvm_save_gc0_entrylo1(cop0);
2713         kvm_save_gc0_context(cop0);
2714         if (cpu_guest_has_contextconfig)
2715                 kvm_save_gc0_contextconfig(cop0);
2716 #ifdef CONFIG_64BIT
2717         kvm_save_gc0_xcontext(cop0);
2718         if (cpu_guest_has_contextconfig)
2719                 kvm_save_gc0_xcontextconfig(cop0);
2720 #endif
2721         kvm_save_gc0_pagemask(cop0);
2722         kvm_save_gc0_pagegrain(cop0);
2723         kvm_save_gc0_wired(cop0);
2724         /* allow wired TLB entries to be overwritten */
2725         clear_gc0_wired(MIPSR6_WIRED_WIRED);
2726         kvm_save_gc0_hwrena(cop0);
2727         kvm_save_gc0_badvaddr(cop0);
2728         kvm_save_gc0_entryhi(cop0);
2729         kvm_save_gc0_status(cop0);
2730         kvm_save_gc0_intctl(cop0);
2731         kvm_save_gc0_epc(cop0);
2732         kvm_write_sw_gc0_ebase(cop0, kvm_vz_read_gc0_ebase());
2733         if (cpu_guest_has_userlocal)
2734                 kvm_save_gc0_userlocal(cop0);
2735
2736         /* only save implemented config registers */
2737         kvm_save_gc0_config(cop0);
2738         if (cpu_guest_has_conf1)
2739                 kvm_save_gc0_config1(cop0);
2740         if (cpu_guest_has_conf2)
2741                 kvm_save_gc0_config2(cop0);
2742         if (cpu_guest_has_conf3)
2743                 kvm_save_gc0_config3(cop0);
2744         if (cpu_guest_has_conf4)
2745                 kvm_save_gc0_config4(cop0);
2746         if (cpu_guest_has_conf5)
2747                 kvm_save_gc0_config5(cop0);
2748         if (cpu_guest_has_conf6)
2749                 kvm_save_gc0_config6(cop0);
2750         if (cpu_guest_has_conf7)
2751                 kvm_save_gc0_config7(cop0);
2752
2753         kvm_save_gc0_errorepc(cop0);
2754
2755         /* save KScratch registers if enabled in guest */
2756         if (cpu_guest_has_conf4) {
2757                 if (cpu_guest_has_kscr(2))
2758                         kvm_save_gc0_kscratch1(cop0);
2759                 if (cpu_guest_has_kscr(3))
2760                         kvm_save_gc0_kscratch2(cop0);
2761                 if (cpu_guest_has_kscr(4))
2762                         kvm_save_gc0_kscratch3(cop0);
2763                 if (cpu_guest_has_kscr(5))
2764                         kvm_save_gc0_kscratch4(cop0);
2765                 if (cpu_guest_has_kscr(6))
2766                         kvm_save_gc0_kscratch5(cop0);
2767                 if (cpu_guest_has_kscr(7))
2768                         kvm_save_gc0_kscratch6(cop0);
2769         }
2770
2771         if (cpu_guest_has_badinstr)
2772                 kvm_save_gc0_badinstr(cop0);
2773         if (cpu_guest_has_badinstrp)
2774                 kvm_save_gc0_badinstrp(cop0);
2775
2776         if (cpu_guest_has_segments) {
2777                 kvm_save_gc0_segctl0(cop0);
2778                 kvm_save_gc0_segctl1(cop0);
2779                 kvm_save_gc0_segctl2(cop0);
2780         }
2781
2782         /* save HTW registers if enabled in guest */
2783         if (cpu_guest_has_ldpte || (cpu_guest_has_htw &&
2784             kvm_read_sw_gc0_config3(cop0) & MIPS_CONF3_PW)) {
2785                 kvm_save_gc0_pwbase(cop0);
2786                 kvm_save_gc0_pwfield(cop0);
2787                 kvm_save_gc0_pwsize(cop0);
2788                 kvm_save_gc0_pwctl(cop0);
2789         }
2790
2791         kvm_vz_save_timer(vcpu);
2792
2793         /* save Root.GuestCtl2 in unused Guest guestctl2 register */
2794         if (cpu_has_guestctl2)
2795                 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] =
2796                         read_c0_guestctl2();
2797
2798         return 0;
2799 }
2800
2801 /**
2802  * kvm_vz_resize_guest_vtlb() - Attempt to resize guest VTLB.
2803  * @size:       Number of guest VTLB entries (0 < @size <= root VTLB entries).
2804  *
2805  * Attempt to resize the guest VTLB by writing guest Config registers. This is
2806  * necessary for cores with a shared root/guest TLB to avoid overlap with wired
2807  * entries in the root VTLB.
2808  *
2809  * Returns:     The resulting guest VTLB size.
2810  */
2811 static unsigned int kvm_vz_resize_guest_vtlb(unsigned int size)
2812 {
2813         unsigned int config4 = 0, ret = 0, limit;
2814
2815         /* Write MMUSize - 1 into guest Config registers */
2816         if (cpu_guest_has_conf1)
2817                 change_gc0_config1(MIPS_CONF1_TLBS,
2818                                    (size - 1) << MIPS_CONF1_TLBS_SHIFT);
2819         if (cpu_guest_has_conf4) {
2820                 config4 = read_gc0_config4();
2821                 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) ==
2822                     MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) {
2823                         config4 &= ~MIPS_CONF4_VTLBSIZEEXT;
2824                         config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) <<
2825                                 MIPS_CONF4_VTLBSIZEEXT_SHIFT;
2826                 } else if ((config4 & MIPS_CONF4_MMUEXTDEF) ==
2827                            MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) {
2828                         config4 &= ~MIPS_CONF4_MMUSIZEEXT;
2829                         config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) <<
2830                                 MIPS_CONF4_MMUSIZEEXT_SHIFT;
2831                 }
2832                 write_gc0_config4(config4);
2833         }
2834
2835         /*
2836          * Set Guest.Wired.Limit = 0 (no limit up to Guest.MMUSize-1), unless it
2837          * would exceed Root.Wired.Limit (clearing Guest.Wired.Wired so write
2838          * not dropped)
2839          */
2840         if (cpu_has_mips_r6) {
2841                 limit = (read_c0_wired() & MIPSR6_WIRED_LIMIT) >>
2842                                                 MIPSR6_WIRED_LIMIT_SHIFT;
2843                 if (size - 1 <= limit)
2844                         limit = 0;
2845                 write_gc0_wired(limit << MIPSR6_WIRED_LIMIT_SHIFT);
2846         }
2847
2848         /* Read back MMUSize - 1 */
2849         back_to_back_c0_hazard();
2850         if (cpu_guest_has_conf1)
2851                 ret = (read_gc0_config1() & MIPS_CONF1_TLBS) >>
2852                                                 MIPS_CONF1_TLBS_SHIFT;
2853         if (config4) {
2854                 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) ==
2855                     MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT)
2856                         ret |= ((config4 & MIPS_CONF4_VTLBSIZEEXT) >>
2857                                 MIPS_CONF4_VTLBSIZEEXT_SHIFT) <<
2858                                 MIPS_CONF1_TLBS_SIZE;
2859                 else if ((config4 & MIPS_CONF4_MMUEXTDEF) ==
2860                          MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT)
2861                         ret |= ((config4 & MIPS_CONF4_MMUSIZEEXT) >>
2862                                 MIPS_CONF4_MMUSIZEEXT_SHIFT) <<
2863                                 MIPS_CONF1_TLBS_SIZE;
2864         }
2865         return ret + 1;
2866 }
2867
2868 static int kvm_vz_hardware_enable(void)
2869 {
2870         unsigned int mmu_size, guest_mmu_size, ftlb_size;
2871         u64 guest_cvmctl, cvmvmconfig;
2872
2873         switch (current_cpu_type()) {
2874         case CPU_CAVIUM_OCTEON3:
2875                 /* Set up guest timer/perfcount IRQ lines */
2876                 guest_cvmctl = read_gc0_cvmctl();
2877                 guest_cvmctl &= ~CVMCTL_IPTI;
2878                 guest_cvmctl |= 7ull << CVMCTL_IPTI_SHIFT;
2879                 guest_cvmctl &= ~CVMCTL_IPPCI;
2880                 guest_cvmctl |= 6ull << CVMCTL_IPPCI_SHIFT;
2881                 write_gc0_cvmctl(guest_cvmctl);
2882
2883                 cvmvmconfig = read_c0_cvmvmconfig();
2884                 /* No I/O hole translation. */
2885                 cvmvmconfig |= CVMVMCONF_DGHT;
2886                 /* Halve the root MMU size */
2887                 mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1)
2888                             >> CVMVMCONF_MMUSIZEM1_S) + 1;
2889                 guest_mmu_size = mmu_size / 2;
2890                 mmu_size -= guest_mmu_size;
2891                 cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1;
2892                 cvmvmconfig |= mmu_size - 1;
2893                 write_c0_cvmvmconfig(cvmvmconfig);
2894
2895                 /* Update our records */
2896                 current_cpu_data.tlbsize = mmu_size;
2897                 current_cpu_data.tlbsizevtlb = mmu_size;
2898                 current_cpu_data.guest.tlbsize = guest_mmu_size;
2899
2900                 /* Flush moved entries in new (guest) context */
2901                 kvm_vz_local_flush_guesttlb_all();
2902                 break;
2903         default:
2904                 /*
2905                  * ImgTec cores tend to use a shared root/guest TLB. To avoid
2906                  * overlap of root wired and guest entries, the guest TLB may
2907                  * need resizing.
2908                  */
2909                 mmu_size = current_cpu_data.tlbsizevtlb;
2910                 ftlb_size = current_cpu_data.tlbsize - mmu_size;
2911
2912                 /* Try switching to maximum guest VTLB size for flush */
2913                 guest_mmu_size = kvm_vz_resize_guest_vtlb(mmu_size);
2914                 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size;
2915                 kvm_vz_local_flush_guesttlb_all();
2916
2917                 /*
2918                  * Reduce to make space for root wired entries and at least 2
2919                  * root non-wired entries. This does assume that long-term wired
2920                  * entries won't be added later.
2921                  */
2922                 guest_mmu_size = mmu_size - num_wired_entries() - 2;
2923                 guest_mmu_size = kvm_vz_resize_guest_vtlb(guest_mmu_size);
2924                 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size;
2925
2926                 /*
2927                  * Write the VTLB size, but if another CPU has already written,
2928                  * check it matches or we won't provide a consistent view to the
2929                  * guest. If this ever happens it suggests an asymmetric number
2930                  * of wired entries.
2931                  */
2932                 if (cmpxchg(&kvm_vz_guest_vtlb_size, 0, guest_mmu_size) &&
2933                     WARN(guest_mmu_size != kvm_vz_guest_vtlb_size,
2934                          "Available guest VTLB size mismatch"))
2935                         return -EINVAL;
2936                 break;
2937         }
2938
2939         /*
2940          * Enable virtualization features granting guest direct control of
2941          * certain features:
2942          * CP0=1:       Guest coprocessor 0 context.
2943          * AT=Guest:    Guest MMU.
2944          * CG=1:        Hit (virtual address) CACHE operations (optional).
2945          * CF=1:        Guest Config registers.
2946          * CGI=1:       Indexed flush CACHE operations (optional).
2947          */
2948         write_c0_guestctl0(MIPS_GCTL0_CP0 |
2949                            (MIPS_GCTL0_AT_GUEST << MIPS_GCTL0_AT_SHIFT) |
2950                            MIPS_GCTL0_CG | MIPS_GCTL0_CF);
2951         if (cpu_has_guestctl0ext) {
2952                 if (current_cpu_type() != CPU_LOONGSON64)
2953                         set_c0_guestctl0ext(MIPS_GCTL0EXT_CGI);
2954                 else
2955                         clear_c0_guestctl0ext(MIPS_GCTL0EXT_CGI);
2956         }
2957
2958         if (cpu_has_guestid) {
2959                 write_c0_guestctl1(0);
2960                 kvm_vz_local_flush_roottlb_all_guests();
2961
2962                 GUESTID_MASK = current_cpu_data.guestid_mask;
2963                 GUESTID_FIRST_VERSION = GUESTID_MASK + 1;
2964                 GUESTID_VERSION_MASK = ~GUESTID_MASK;
2965
2966                 current_cpu_data.guestid_cache = GUESTID_FIRST_VERSION;
2967         }
2968
2969         /* clear any pending injected virtual guest interrupts */
2970         if (cpu_has_guestctl2)
2971                 clear_c0_guestctl2(0x3f << 10);
2972
2973 #ifdef CONFIG_CPU_LOONGSON64
2974         /* Control guest CCA attribute */
2975         if (cpu_has_csr())
2976                 csr_writel(csr_readl(0xffffffec) | 0x1, 0xffffffec);
2977 #endif
2978
2979         return 0;
2980 }
2981
2982 static void kvm_vz_hardware_disable(void)
2983 {
2984         u64 cvmvmconfig;
2985         unsigned int mmu_size;
2986
2987         /* Flush any remaining guest TLB entries */
2988         kvm_vz_local_flush_guesttlb_all();
2989
2990         switch (current_cpu_type()) {
2991         case CPU_CAVIUM_OCTEON3:
2992                 /*
2993                  * Allocate whole TLB for root. Existing guest TLB entries will
2994                  * change ownership to the root TLB. We should be safe though as
2995                  * they've already been flushed above while in guest TLB.
2996                  */
2997                 cvmvmconfig = read_c0_cvmvmconfig();
2998                 mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1)
2999                             >> CVMVMCONF_MMUSIZEM1_S) + 1;
3000                 cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1;
3001                 cvmvmconfig |= mmu_size - 1;
3002                 write_c0_cvmvmconfig(cvmvmconfig);
3003
3004                 /* Update our records */
3005                 current_cpu_data.tlbsize = mmu_size;
3006                 current_cpu_data.tlbsizevtlb = mmu_size;
3007                 current_cpu_data.guest.tlbsize = 0;
3008
3009                 /* Flush moved entries in new (root) context */
3010                 local_flush_tlb_all();
3011                 break;
3012         }
3013
3014         if (cpu_has_guestid) {
3015                 write_c0_guestctl1(0);
3016                 kvm_vz_local_flush_roottlb_all_guests();
3017         }
3018 }
3019
3020 static int kvm_vz_check_extension(struct kvm *kvm, long ext)
3021 {
3022         int r;
3023
3024         switch (ext) {
3025         case KVM_CAP_MIPS_VZ:
3026                 /* we wouldn't be here unless cpu_has_vz */
3027                 r = 1;
3028                 break;
3029 #ifdef CONFIG_64BIT
3030         case KVM_CAP_MIPS_64BIT:
3031                 /* We support 64-bit registers/operations and addresses */
3032                 r = 2;
3033                 break;
3034 #endif
3035         case KVM_CAP_IOEVENTFD:
3036                 r = 1;
3037                 break;
3038         default:
3039                 r = 0;
3040                 break;
3041         }
3042
3043         return r;
3044 }
3045
3046 static int kvm_vz_vcpu_init(struct kvm_vcpu *vcpu)
3047 {
3048         int i;
3049
3050         for_each_possible_cpu(i)
3051                 vcpu->arch.vzguestid[i] = 0;
3052
3053         return 0;
3054 }
3055
3056 static void kvm_vz_vcpu_uninit(struct kvm_vcpu *vcpu)
3057 {
3058         int cpu;
3059
3060         /*
3061          * If the VCPU is freed and reused as another VCPU, we don't want the
3062          * matching pointer wrongly hanging around in last_vcpu[] or
3063          * last_exec_vcpu[].
3064          */
3065         for_each_possible_cpu(cpu) {
3066                 if (last_vcpu[cpu] == vcpu)
3067                         last_vcpu[cpu] = NULL;
3068                 if (last_exec_vcpu[cpu] == vcpu)
3069                         last_exec_vcpu[cpu] = NULL;
3070         }
3071 }
3072
3073 static int kvm_vz_vcpu_setup(struct kvm_vcpu *vcpu)
3074 {
3075         struct mips_coproc *cop0 = vcpu->arch.cop0;
3076         unsigned long count_hz = 100*1000*1000; /* default to 100 MHz */
3077
3078         /*
3079          * Start off the timer at the same frequency as the host timer, but the
3080          * soft timer doesn't handle frequencies greater than 1GHz yet.
3081          */
3082         if (mips_hpt_frequency && mips_hpt_frequency <= NSEC_PER_SEC)
3083                 count_hz = mips_hpt_frequency;
3084         kvm_mips_init_count(vcpu, count_hz);
3085
3086         /*
3087          * Initialize guest register state to valid architectural reset state.
3088          */
3089
3090         /* PageGrain */
3091         if (cpu_has_mips_r5 || cpu_has_mips_r6)
3092                 kvm_write_sw_gc0_pagegrain(cop0, PG_RIE | PG_XIE | PG_IEC);
3093         /* Wired */
3094         if (cpu_has_mips_r6)
3095                 kvm_write_sw_gc0_wired(cop0,
3096                                        read_gc0_wired() & MIPSR6_WIRED_LIMIT);
3097         /* Status */
3098         kvm_write_sw_gc0_status(cop0, ST0_BEV | ST0_ERL);
3099         if (cpu_has_mips_r5 || cpu_has_mips_r6)
3100                 kvm_change_sw_gc0_status(cop0, ST0_FR, read_gc0_status());
3101         /* IntCtl */
3102         kvm_write_sw_gc0_intctl(cop0, read_gc0_intctl() &
3103                                 (INTCTLF_IPFDC | INTCTLF_IPPCI | INTCTLF_IPTI));
3104         /* PRId */
3105         kvm_write_sw_gc0_prid(cop0, boot_cpu_data.processor_id);
3106         /* EBase */
3107         kvm_write_sw_gc0_ebase(cop0, (s32)0x80000000 | vcpu->vcpu_id);
3108         /* Config */
3109         kvm_save_gc0_config(cop0);
3110         /* architecturally writable (e.g. from guest) */
3111         kvm_change_sw_gc0_config(cop0, CONF_CM_CMASK,
3112                                  _page_cachable_default >> _CACHE_SHIFT);
3113         /* architecturally read only, but maybe writable from root */
3114         kvm_change_sw_gc0_config(cop0, MIPS_CONF_MT, read_c0_config());
3115         if (cpu_guest_has_conf1) {
3116                 kvm_set_sw_gc0_config(cop0, MIPS_CONF_M);
3117                 /* Config1 */
3118                 kvm_save_gc0_config1(cop0);
3119                 /* architecturally read only, but maybe writable from root */
3120                 kvm_clear_sw_gc0_config1(cop0, MIPS_CONF1_C2    |
3121                                                MIPS_CONF1_MD    |
3122                                                MIPS_CONF1_PC    |
3123                                                MIPS_CONF1_WR    |
3124                                                MIPS_CONF1_CA    |
3125                                                MIPS_CONF1_FP);
3126         }
3127         if (cpu_guest_has_conf2) {
3128                 kvm_set_sw_gc0_config1(cop0, MIPS_CONF_M);
3129                 /* Config2 */
3130                 kvm_save_gc0_config2(cop0);
3131         }
3132         if (cpu_guest_has_conf3) {
3133                 kvm_set_sw_gc0_config2(cop0, MIPS_CONF_M);
3134                 /* Config3 */
3135                 kvm_save_gc0_config3(cop0);
3136                 /* architecturally writable (e.g. from guest) */
3137                 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_ISA_OE);
3138                 /* architecturally read only, but maybe writable from root */
3139                 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_MSA   |
3140                                                MIPS_CONF3_BPG   |
3141                                                MIPS_CONF3_ULRI  |
3142                                                MIPS_CONF3_DSP   |
3143                                                MIPS_CONF3_CTXTC |
3144                                                MIPS_CONF3_ITL   |
3145                                                MIPS_CONF3_LPA   |
3146                                                MIPS_CONF3_VEIC  |
3147                                                MIPS_CONF3_VINT  |
3148                                                MIPS_CONF3_SP    |
3149                                                MIPS_CONF3_CDMM  |
3150                                                MIPS_CONF3_MT    |
3151                                                MIPS_CONF3_SM    |
3152                                                MIPS_CONF3_TL);
3153         }
3154         if (cpu_guest_has_conf4) {
3155                 kvm_set_sw_gc0_config3(cop0, MIPS_CONF_M);
3156                 /* Config4 */
3157                 kvm_save_gc0_config4(cop0);
3158         }
3159         if (cpu_guest_has_conf5) {
3160                 kvm_set_sw_gc0_config4(cop0, MIPS_CONF_M);
3161                 /* Config5 */
3162                 kvm_save_gc0_config5(cop0);
3163                 /* architecturally writable (e.g. from guest) */
3164                 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_K     |
3165                                                MIPS_CONF5_CV    |
3166                                                MIPS_CONF5_MSAEN |
3167                                                MIPS_CONF5_UFE   |
3168                                                MIPS_CONF5_FRE   |
3169                                                MIPS_CONF5_SBRI  |
3170                                                MIPS_CONF5_UFR);
3171                 /* architecturally read only, but maybe writable from root */
3172                 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_MRP);
3173         }
3174
3175         if (cpu_guest_has_contextconfig) {
3176                 /* ContextConfig */
3177                 kvm_write_sw_gc0_contextconfig(cop0, 0x007ffff0);
3178 #ifdef CONFIG_64BIT
3179                 /* XContextConfig */
3180                 /* bits SEGBITS-13+3:4 set */
3181                 kvm_write_sw_gc0_xcontextconfig(cop0,
3182                                         ((1ull << (cpu_vmbits - 13)) - 1) << 4);
3183 #endif
3184         }
3185
3186         /* Implementation dependent, use the legacy layout */
3187         if (cpu_guest_has_segments) {
3188                 /* SegCtl0, SegCtl1, SegCtl2 */
3189                 kvm_write_sw_gc0_segctl0(cop0, 0x00200010);
3190                 kvm_write_sw_gc0_segctl1(cop0, 0x00000002 |
3191                                 (_page_cachable_default >> _CACHE_SHIFT) <<
3192                                                 (16 + MIPS_SEGCFG_C_SHIFT));
3193                 kvm_write_sw_gc0_segctl2(cop0, 0x00380438);
3194         }
3195
3196         /* reset HTW registers */
3197         if (cpu_guest_has_htw && (cpu_has_mips_r5 || cpu_has_mips_r6)) {
3198                 /* PWField */
3199                 kvm_write_sw_gc0_pwfield(cop0, 0x0c30c302);
3200                 /* PWSize */
3201                 kvm_write_sw_gc0_pwsize(cop0, 1 << MIPS_PWSIZE_PTW_SHIFT);
3202         }
3203
3204         /* start with no pending virtual guest interrupts */
3205         if (cpu_has_guestctl2)
3206                 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 0;
3207
3208         /* Put PC at reset vector */
3209         vcpu->arch.pc = CKSEG1ADDR(0x1fc00000);
3210
3211         return 0;
3212 }
3213
3214 static void kvm_vz_flush_shadow_all(struct kvm *kvm)
3215 {
3216         if (cpu_has_guestid) {
3217                 /* Flush GuestID for each VCPU individually */
3218                 kvm_flush_remote_tlbs(kvm);
3219         } else {
3220                 /*
3221                  * For each CPU there is a single GPA ASID used by all VCPUs in
3222                  * the VM, so it doesn't make sense for the VCPUs to handle
3223                  * invalidation of these ASIDs individually.
3224                  *
3225                  * Instead mark all CPUs as needing ASID invalidation in
3226                  * asid_flush_mask, and just use kvm_flush_remote_tlbs(kvm) to
3227                  * kick any running VCPUs so they check asid_flush_mask.
3228                  */
3229                 cpumask_setall(&kvm->arch.asid_flush_mask);
3230                 kvm_flush_remote_tlbs(kvm);
3231         }
3232 }
3233
3234 static void kvm_vz_flush_shadow_memslot(struct kvm *kvm,
3235                                         const struct kvm_memory_slot *slot)
3236 {
3237         kvm_vz_flush_shadow_all(kvm);
3238 }
3239
3240 static void kvm_vz_vcpu_reenter(struct kvm_vcpu *vcpu)
3241 {
3242         int cpu = smp_processor_id();
3243         int preserve_guest_tlb;
3244
3245         preserve_guest_tlb = kvm_vz_check_requests(vcpu, cpu);
3246
3247         if (preserve_guest_tlb)
3248                 kvm_vz_vcpu_save_wired(vcpu);
3249
3250         kvm_vz_vcpu_load_tlb(vcpu, cpu);
3251
3252         if (preserve_guest_tlb)
3253                 kvm_vz_vcpu_load_wired(vcpu);
3254 }
3255
3256 static int kvm_vz_vcpu_run(struct kvm_vcpu *vcpu)
3257 {
3258         int cpu = smp_processor_id();
3259         int r;
3260
3261         kvm_vz_acquire_htimer(vcpu);
3262         /* Check if we have any exceptions/interrupts pending */
3263         kvm_mips_deliver_interrupts(vcpu, read_gc0_cause());
3264
3265         kvm_vz_check_requests(vcpu, cpu);
3266         kvm_vz_vcpu_load_tlb(vcpu, cpu);
3267         kvm_vz_vcpu_load_wired(vcpu);
3268
3269         r = vcpu->arch.vcpu_run(vcpu);
3270
3271         kvm_vz_vcpu_save_wired(vcpu);
3272
3273         return r;
3274 }
3275
3276 static struct kvm_mips_callbacks kvm_vz_callbacks = {
3277         .handle_cop_unusable = kvm_trap_vz_handle_cop_unusable,
3278         .handle_tlb_mod = kvm_trap_vz_handle_tlb_st_miss,
3279         .handle_tlb_ld_miss = kvm_trap_vz_handle_tlb_ld_miss,
3280         .handle_tlb_st_miss = kvm_trap_vz_handle_tlb_st_miss,
3281         .handle_addr_err_st = kvm_trap_vz_no_handler,
3282         .handle_addr_err_ld = kvm_trap_vz_no_handler,
3283         .handle_syscall = kvm_trap_vz_no_handler,
3284         .handle_res_inst = kvm_trap_vz_no_handler,
3285         .handle_break = kvm_trap_vz_no_handler,
3286         .handle_msa_disabled = kvm_trap_vz_handle_msa_disabled,
3287         .handle_guest_exit = kvm_trap_vz_handle_guest_exit,
3288
3289         .hardware_enable = kvm_vz_hardware_enable,
3290         .hardware_disable = kvm_vz_hardware_disable,
3291         .check_extension = kvm_vz_check_extension,
3292         .vcpu_init = kvm_vz_vcpu_init,
3293         .vcpu_uninit = kvm_vz_vcpu_uninit,
3294         .vcpu_setup = kvm_vz_vcpu_setup,
3295         .flush_shadow_all = kvm_vz_flush_shadow_all,
3296         .flush_shadow_memslot = kvm_vz_flush_shadow_memslot,
3297         .gva_to_gpa = kvm_vz_gva_to_gpa_cb,
3298         .queue_timer_int = kvm_vz_queue_timer_int_cb,
3299         .dequeue_timer_int = kvm_vz_dequeue_timer_int_cb,
3300         .queue_io_int = kvm_vz_queue_io_int_cb,
3301         .dequeue_io_int = kvm_vz_dequeue_io_int_cb,
3302         .irq_deliver = kvm_vz_irq_deliver_cb,
3303         .irq_clear = kvm_vz_irq_clear_cb,
3304         .num_regs = kvm_vz_num_regs,
3305         .copy_reg_indices = kvm_vz_copy_reg_indices,
3306         .get_one_reg = kvm_vz_get_one_reg,
3307         .set_one_reg = kvm_vz_set_one_reg,
3308         .vcpu_load = kvm_vz_vcpu_load,
3309         .vcpu_put = kvm_vz_vcpu_put,
3310         .vcpu_run = kvm_vz_vcpu_run,
3311         .vcpu_reenter = kvm_vz_vcpu_reenter,
3312 };
3313
3314 int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks)
3315 {
3316         if (!cpu_has_vz)
3317                 return -ENODEV;
3318
3319         /*
3320          * VZ requires at least 2 KScratch registers, so it should have been
3321          * possible to allocate pgd_reg.
3322          */
3323         if (WARN(pgd_reg == -1,
3324                  "pgd_reg not allocated even though cpu_has_vz\n"))
3325                 return -ENODEV;
3326
3327         pr_info("Starting KVM with MIPS VZ extensions\n");
3328
3329         *install_callbacks = &kvm_vz_callbacks;
3330         return 0;
3331 }