arch/x86/kvm/svm.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * AMD SVM support
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   8  *
   9  * Authors:
  10  *   Yaniv Kamay  <yaniv@qumranet.com>
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) "SVM: " fmt
  19
  20 #include <linux/kvm_host.h>
  21
  22 #include "irq.h"
  23 #include "mmu.h"
  24 #include "kvm_cache_regs.h"
  25 #include "x86.h"
  26 #include "cpuid.h"
  27 #include "pmu.h"
  28
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/kernel.h>
  32 #include <linux/vmalloc.h>
  33 #include <linux/highmem.h>
  34 #include <linux/sched.h>
  35 #include <linux/trace_events.h>
  36 #include <linux/slab.h>
  37 #include <linux/amd-iommu.h>
  38 #include <linux/hashtable.h>
  39 #include <linux/frame.h>
  40 #include <linux/psp-sev.h>
  41 #include <linux/file.h>
  42 #include <linux/pagemap.h>
  43 #include <linux/swap.h>
  44
  45 #include <asm/apic.h>
  46 #include <asm/perf_event.h>
  47 #include <asm/tlbflush.h>
  48 #include <asm/desc.h>
  49 #include <asm/debugreg.h>
  50 #include <asm/kvm_para.h>
  51 #include <asm/irq_remapping.h>
  52 #include <asm/spec-ctrl.h>
  53
  54 #include <asm/virtext.h>
  55 #include "trace.h"
  56
  57 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  58
  59 MODULE_AUTHOR("Qumranet");
  60 MODULE_LICENSE("GPL");
  61
  62 static const struct x86_cpu_id svm_cpu_id[] = {
  63         X86_FEATURE_MATCH(X86_FEATURE_SVM),
  64         {}
  65 };
  66 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  67
  68 #define IOPM_ALLOC_ORDER 2
  69 #define MSRPM_ALLOC_ORDER 1
  70
  71 #define SEG_TYPE_LDT 2
  72 #define SEG_TYPE_BUSY_TSS16 3
  73
  74 #define SVM_FEATURE_NPT            (1 <<  0)
  75 #define SVM_FEATURE_LBRV           (1 <<  1)
  76 #define SVM_FEATURE_SVML           (1 <<  2)
  77 #define SVM_FEATURE_NRIP           (1 <<  3)
  78 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  79 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  80 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  81 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  82 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  83
  84 #define SVM_AVIC_DOORBELL       0xc001011b
  85
  86 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  87 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  88 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  89
  90 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  91
  92 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  93 #define TSC_RATIO_MIN           0x0000000000000001ULL
  94 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  95
  96 #define AVIC_HPA_MASK   ~((0xFFFULL << 52) | 0xFFF)
  97
  98 /*
  99  * 0xff is broadcast, so the max index allowed for physical APIC ID
 100  * table is 0xfe.  APIC IDs above 0xff are reserved.
 101  */
 102 #define AVIC_MAX_PHYSICAL_ID_COUNT      255
 103
 104 #define AVIC_UNACCEL_ACCESS_WRITE_MASK          1
 105 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK         0xFF0
 106 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK         0xFFFFFFFF
 107
 108 /* AVIC GATAG is encoded using VM and VCPU IDs */
 109 #define AVIC_VCPU_ID_BITS               8
 110 #define AVIC_VCPU_ID_MASK               ((1 << AVIC_VCPU_ID_BITS) - 1)
 111
 112 #define AVIC_VM_ID_BITS                 24
 113 #define AVIC_VM_ID_NR                   (1 << AVIC_VM_ID_BITS)
 114 #define AVIC_VM_ID_MASK                 ((1 << AVIC_VM_ID_BITS) - 1)
 115
 116 #define AVIC_GATAG(x, y)                (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
 117                                                 (y & AVIC_VCPU_ID_MASK))
 118 #define AVIC_GATAG_TO_VMID(x)           ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
 119 #define AVIC_GATAG_TO_VCPUID(x)         (x & AVIC_VCPU_ID_MASK)
 120
 121 static bool erratum_383_found __read_mostly;
 122
 123 static const u32 host_save_user_msrs[] = {
 124 #ifdef CONFIG_X86_64
 125         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
 126         MSR_FS_BASE,
 127 #endif
 128         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 129         MSR_TSC_AUX,
 130 };
 131
 132 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
 133
 134 struct kvm_sev_info {
 135         bool active;            /* SEV enabled guest */
 136         unsigned int asid;      /* ASID used for this guest */
 137         unsigned int handle;    /* SEV firmware handle */
 138         int fd;                 /* SEV device fd */
 139         unsigned long pages_locked; /* Number of pages locked */
 140         struct list_head regions_list;  /* List of registered regions */
 141 };
 142
 143 struct kvm_svm {
 144         struct kvm kvm;
 145
 146         /* Struct members for AVIC */
 147         u32 avic_vm_id;
 148         u32 ldr_mode;
 149         struct page *avic_logical_id_table_page;
 150         struct page *avic_physical_id_table_page;
 151         struct hlist_node hnode;
 152
 153         struct kvm_sev_info sev_info;
 154 };
 155
 156 struct kvm_vcpu;
 157
 158 struct nested_state {
 159         struct vmcb *hsave;
 160         u64 hsave_msr;
 161         u64 vm_cr_msr;
 162         u64 vmcb;
 163
 164         /* These are the merged vectors */
 165         u32 *msrpm;
 166
 167         /* gpa pointers to the real vectors */
 168         u64 vmcb_msrpm;
 169         u64 vmcb_iopm;
 170
 171         /* A VMEXIT is required but not yet emulated */
 172         bool exit_required;
 173
 174         /* cache for intercepts of the guest */
 175         u32 intercept_cr;
 176         u32 intercept_dr;
 177         u32 intercept_exceptions;
 178         u64 intercept;
 179
 180         /* Nested Paging related state */
 181         u64 nested_cr3;
 182 };
 183
 184 #define MSRPM_OFFSETS   16
 185 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 186
 187 /*
 188  * Set osvw_len to higher value when updated Revision Guides
 189  * are published and we know what the new status bits are
 190  */
 191 static uint64_t osvw_len = 4, osvw_status;
 192
 193 struct vcpu_svm {
 194         struct kvm_vcpu vcpu;
 195         struct vmcb *vmcb;
 196         unsigned long vmcb_pa;
 197         struct svm_cpu_data *svm_data;
 198         uint64_t asid_generation;
 199         uint64_t sysenter_esp;
 200         uint64_t sysenter_eip;
 201         uint64_t tsc_aux;
 202
 203         u64 msr_decfg;
 204
 205         u64 next_rip;
 206
 207         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 208         struct {
 209                 u16 fs;
 210                 u16 gs;
 211                 u16 ldt;
 212                 u64 gs_base;
 213         } host;
 214
 215         u64 spec_ctrl;
 216         /*
 217          * Contains guest-controlled bits of VIRT_SPEC_CTRL, which will be
 218          * translated into the appropriate L2_CFG bits on the host to
 219          * perform speculative control.
 220          */
 221         u64 virt_spec_ctrl;
 222
 223         u32 *msrpm;
 224
 225         ulong nmi_iret_rip;
 226
 227         struct nested_state nested;
 228
 229         bool nmi_singlestep;
 230         u64 nmi_singlestep_guest_rflags;
 231
 232         unsigned int3_injected;
 233         unsigned long int3_rip;
 234
 235         /* cached guest cpuid flags for faster access */
 236         bool nrips_enabled      : 1;
 237
 238         u32 ldr_reg;
 239         struct page *avic_backing_page;
 240         u64 *avic_physical_id_cache;
 241         bool avic_is_running;
 242
 243         /*
 244          * Per-vcpu list of struct amd_svm_iommu_ir:
 245          * This is used mainly to store interrupt remapping information used
 246          * when update the vcpu affinity. This avoids the need to scan for
 247          * IRTE and try to match ga_tag in the IOMMU driver.
 248          */
 249         struct list_head ir_list;
 250         spinlock_t ir_list_lock;
 251
 252         /* which host CPU was used for running this vcpu */
 253         unsigned int last_cpu;
 254 };
 255
 256 /*
 257  * This is a wrapper of struct amd_iommu_ir_data.
 258  */
 259 struct amd_svm_iommu_ir {
 260         struct list_head node;  /* Used by SVM for per-vcpu ir_list */
 261         void *data;             /* Storing pointer to struct amd_ir_data */
 262 };
 263
 264 #define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK    (0xFF)
 265 #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK                (1 << 31)
 266
 267 #define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK    (0xFFULL)
 268 #define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK        (0xFFFFFFFFFFULL << 12)
 269 #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK          (1ULL << 62)
 270 #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK               (1ULL << 63)
 271
 272 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 273 #define TSC_RATIO_DEFAULT       0x0100000000ULL
 274
 275 #define MSR_INVALID                     0xffffffffU
 276
 277 static const struct svm_direct_access_msrs {
 278         u32 index;   /* Index of the MSR */
 279         bool always; /* True if intercept is always on */
 280 } direct_access_msrs[] = {
 281         { .index = MSR_STAR,                            .always = true  },
 282         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 283 #ifdef CONFIG_X86_64
 284         { .index = MSR_GS_BASE,                         .always = true  },
 285         { .index = MSR_FS_BASE,                         .always = true  },
 286         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 287         { .index = MSR_LSTAR,                           .always = true  },
 288         { .index = MSR_CSTAR,                           .always = true  },
 289         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 290 #endif
 291         { .index = MSR_IA32_SPEC_CTRL,                  .always = false },
 292         { .index = MSR_IA32_PRED_CMD,                   .always = false },
 293         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 294         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 295         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 296         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 297         { .index = MSR_INVALID,                         .always = false },
 298 };
 299
 300 /* enable NPT for AMD64 and X86 with PAE */
 301 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 302 static bool npt_enabled = true;
 303 #else
 304 static bool npt_enabled;
 305 #endif
 306
 307 /*
 308  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 309  * pause_filter_count: On processors that support Pause filtering(indicated
 310  *      by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
 311  *      count value. On VMRUN this value is loaded into an internal counter.
 312  *      Each time a pause instruction is executed, this counter is decremented
 313  *      until it reaches zero at which time a #VMEXIT is generated if pause
 314  *      intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
 315  *      Intercept Filtering for more details.
 316  *      This also indicate if ple logic enabled.
 317  *
 318  * pause_filter_thresh: In addition, some processor families support advanced
 319  *      pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
 320  *      the amount of time a guest is allowed to execute in a pause loop.
 321  *      In this mode, a 16-bit pause filter threshold field is added in the
 322  *      VMCB. The threshold value is a cycle count that is used to reset the
 323  *      pause counter. As with simple pause filtering, VMRUN loads the pause
 324  *      count value from VMCB into an internal counter. Then, on each pause
 325  *      instruction the hardware checks the elapsed number of cycles since
 326  *      the most recent pause instruction against the pause filter threshold.
 327  *      If the elapsed cycle count is greater than the pause filter threshold,
 328  *      then the internal pause count is reloaded from the VMCB and execution
 329  *      continues. If the elapsed cycle count is less than the pause filter
 330  *      threshold, then the internal pause count is decremented. If the count
 331  *      value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
 332  *      triggered. If advanced pause filtering is supported and pause filter
 333  *      threshold field is set to zero, the filter will operate in the simpler,
 334  *      count only mode.
 335  */
 336
 337 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
 338 module_param(pause_filter_thresh, ushort, 0444);
 339
 340 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
 341 module_param(pause_filter_count, ushort, 0444);
 342
 343 /* Default doubles per-vcpu window every exit. */
 344 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 345 module_param(pause_filter_count_grow, ushort, 0444);
 346
 347 /* Default resets per-vcpu window every exit to pause_filter_count. */
 348 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 349 module_param(pause_filter_count_shrink, ushort, 0444);
 350
 351 /* Default is to compute the maximum so we can never overflow. */
 352 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
 353 module_param(pause_filter_count_max, ushort, 0444);
 354
 355 /* allow nested paging (virtualized MMU) for all guests */
 356 static int npt = true;
 357 module_param(npt, int, S_IRUGO);
 358
 359 /* allow nested virtualization in KVM/SVM */
 360 static int nested = true;
 361 module_param(nested, int, S_IRUGO);
 362
 363 /* enable / disable AVIC */
 364 static int avic;
 365 #ifdef CONFIG_X86_LOCAL_APIC
 366 module_param(avic, int, S_IRUGO);
 367 #endif
 368
 369 /* enable/disable Virtual VMLOAD VMSAVE */
 370 static int vls = true;
 371 module_param(vls, int, 0444);
 372
 373 /* enable/disable Virtual GIF */
 374 static int vgif = true;
 375 module_param(vgif, int, 0444);
 376
 377 /* enable/disable SEV support */
 378 static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
 379 module_param(sev, int, 0444);
 380
 381 static u8 rsm_ins_bytes[] = "\x0f\xaa";
 382
 383 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 384 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa);
 385 static void svm_complete_interrupts(struct vcpu_svm *svm);
 386
 387 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 388 static int nested_svm_intercept(struct vcpu_svm *svm);
 389 static int nested_svm_vmexit(struct vcpu_svm *svm);
 390 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 391                                       bool has_error_code, u32 error_code);
 392
 393 enum {
 394         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
 395                             pause filter count */
 396         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
 397         VMCB_ASID,       /* ASID */
 398         VMCB_INTR,       /* int_ctl, int_vector */
 399         VMCB_NPT,        /* npt_en, nCR3, gPAT */
 400         VMCB_CR,         /* CR0, CR3, CR4, EFER */
 401         VMCB_DR,         /* DR6, DR7 */
 402         VMCB_DT,         /* GDT, IDT */
 403         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
 404         VMCB_CR2,        /* CR2 only */
 405         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
 406         VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
 407                           * AVIC PHYSICAL_TABLE pointer,
 408                           * AVIC LOGICAL_TABLE pointer
 409                           */
 410         VMCB_DIRTY_MAX,
 411 };
 412
 413 /* TPR and CR2 are always written before VMRUN */
 414 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
 415
 416 #define VMCB_AVIC_APIC_BAR_MASK         0xFFFFFFFFFF000ULL
 417
 418 static unsigned int max_sev_asid;
 419 static unsigned int min_sev_asid;
 420 static unsigned long *sev_asid_bitmap;
 421 #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
 422
 423 struct enc_region {
 424         struct list_head list;
 425         unsigned long npages;
 426         struct page **pages;
 427         unsigned long uaddr;
 428         unsigned long size;
 429 };
 430
 431
 432 static inline struct kvm_svm *to_kvm_svm(struct kvm *kvm)
 433 {
 434         return container_of(kvm, struct kvm_svm, kvm);
 435 }
 436
 437 static inline bool svm_sev_enabled(void)
 438 {
 439         return max_sev_asid;
 440 }
 441
 442 static inline bool sev_guest(struct kvm *kvm)
 443 {
 444         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 445
 446         return sev->active;
 447 }
 448
 449 static inline int sev_get_asid(struct kvm *kvm)
 450 {
 451         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 452
 453         return sev->asid;
 454 }
 455
 456 static inline void mark_all_dirty(struct vmcb *vmcb)
 457 {
 458         vmcb->control.clean = 0;
 459 }
 460
 461 static inline void mark_all_clean(struct vmcb *vmcb)
 462 {
 463         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
 464                                & ~VMCB_ALWAYS_DIRTY_MASK;
 465 }
 466
 467 static inline void mark_dirty(struct vmcb *vmcb, int bit)
 468 {
 469         vmcb->control.clean &= ~(1 << bit);
 470 }
 471
 472 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 473 {
 474         return container_of(vcpu, struct vcpu_svm, vcpu);
 475 }
 476
 477 static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
 478 {
 479         svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
 480         mark_dirty(svm->vmcb, VMCB_AVIC);
 481 }
 482
 483 static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
 484 {
 485         struct vcpu_svm *svm = to_svm(vcpu);
 486         u64 *entry = svm->avic_physical_id_cache;
 487
 488         if (!entry)
 489                 return false;
 490
 491         return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
 492 }
 493
 494 static void recalc_intercepts(struct vcpu_svm *svm)
 495 {
 496         struct vmcb_control_area *c, *h;
 497         struct nested_state *g;
 498
 499         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 500
 501         if (!is_guest_mode(&svm->vcpu))
 502                 return;
 503
 504         c = &svm->vmcb->control;
 505         h = &svm->nested.hsave->control;
 506         g = &svm->nested;
 507
 508         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 509         c->intercept_dr = h->intercept_dr | g->intercept_dr;
 510         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 511         c->intercept = h->intercept | g->intercept;
 512 }
 513
 514 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 515 {
 516         if (is_guest_mode(&svm->vcpu))
 517                 return svm->nested.hsave;
 518         else
 519                 return svm->vmcb;
 520 }
 521
 522 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 523 {
 524         struct vmcb *vmcb = get_host_vmcb(svm);
 525
 526         vmcb->control.intercept_cr |= (1U << bit);
 527
 528         recalc_intercepts(svm);
 529 }
 530
 531 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 532 {
 533         struct vmcb *vmcb = get_host_vmcb(svm);
 534
 535         vmcb->control.intercept_cr &= ~(1U << bit);
 536
 537         recalc_intercepts(svm);
 538 }
 539
 540 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 541 {
 542         struct vmcb *vmcb = get_host_vmcb(svm);
 543
 544         return vmcb->control.intercept_cr & (1U << bit);
 545 }
 546
 547 static inline void set_dr_intercepts(struct vcpu_svm *svm)
 548 {
 549         struct vmcb *vmcb = get_host_vmcb(svm);
 550
 551         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
 552                 | (1 << INTERCEPT_DR1_READ)
 553                 | (1 << INTERCEPT_DR2_READ)
 554                 | (1 << INTERCEPT_DR3_READ)
 555                 | (1 << INTERCEPT_DR4_READ)
 556                 | (1 << INTERCEPT_DR5_READ)
 557                 | (1 << INTERCEPT_DR6_READ)
 558                 | (1 << INTERCEPT_DR7_READ)
 559                 | (1 << INTERCEPT_DR0_WRITE)
 560                 | (1 << INTERCEPT_DR1_WRITE)
 561                 | (1 << INTERCEPT_DR2_WRITE)
 562                 | (1 << INTERCEPT_DR3_WRITE)
 563                 | (1 << INTERCEPT_DR4_WRITE)
 564                 | (1 << INTERCEPT_DR5_WRITE)
 565                 | (1 << INTERCEPT_DR6_WRITE)
 566                 | (1 << INTERCEPT_DR7_WRITE);
 567
 568         recalc_intercepts(svm);
 569 }
 570
 571 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
 572 {
 573         struct vmcb *vmcb = get_host_vmcb(svm);
 574
 575         vmcb->control.intercept_dr = 0;
 576
 577         recalc_intercepts(svm);
 578 }
 579
 580 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
 581 {
 582         struct vmcb *vmcb = get_host_vmcb(svm);
 583
 584         vmcb->control.intercept_exceptions |= (1U << bit);
 585
 586         recalc_intercepts(svm);
 587 }
 588
 589 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 590 {
 591         struct vmcb *vmcb = get_host_vmcb(svm);
 592
 593         vmcb->control.intercept_exceptions &= ~(1U << bit);
 594
 595         recalc_intercepts(svm);
 596 }
 597
 598 static inline void set_intercept(struct vcpu_svm *svm, int bit)
 599 {
 600         struct vmcb *vmcb = get_host_vmcb(svm);
 601
 602         vmcb->control.intercept |= (1ULL << bit);
 603
 604         recalc_intercepts(svm);
 605 }
 606
 607 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
 608 {
 609         struct vmcb *vmcb = get_host_vmcb(svm);
 610
 611         vmcb->control.intercept &= ~(1ULL << bit);
 612
 613         recalc_intercepts(svm);
 614 }
 615
 616 static inline bool vgif_enabled(struct vcpu_svm *svm)
 617 {
 618         return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK);
 619 }
 620
 621 static inline void enable_gif(struct vcpu_svm *svm)
 622 {
 623         if (vgif_enabled(svm))
 624                 svm->vmcb->control.int_ctl |= V_GIF_MASK;
 625         else
 626                 svm->vcpu.arch.hflags |= HF_GIF_MASK;
 627 }
 628
 629 static inline void disable_gif(struct vcpu_svm *svm)
 630 {
 631         if (vgif_enabled(svm))
 632                 svm->vmcb->control.int_ctl &= ~V_GIF_MASK;
 633         else
 634                 svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 635 }
 636
 637 static inline bool gif_set(struct vcpu_svm *svm)
 638 {
 639         if (vgif_enabled(svm))
 640                 return !!(svm->vmcb->control.int_ctl & V_GIF_MASK);
 641         else
 642                 return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 643 }
 644
 645 static unsigned long iopm_base;
 646
 647 struct kvm_ldttss_desc {
 648         u16 limit0;
 649         u16 base0;
 650         unsigned base1:8, type:5, dpl:2, p:1;
 651         unsigned limit1:4, zero0:3, g:1, base2:8;
 652         u32 base3;
 653         u32 zero1;
 654 } __attribute__((packed));
 655
 656 struct svm_cpu_data {
 657         int cpu;
 658
 659         u64 asid_generation;
 660         u32 max_asid;
 661         u32 next_asid;
 662         u32 min_asid;
 663         struct kvm_ldttss_desc *tss_desc;
 664
 665         struct page *save_area;
 666         struct vmcb *current_vmcb;
 667
 668         /* index = sev_asid, value = vmcb pointer */
 669         struct vmcb **sev_vmcbs;
 670 };
 671
 672 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 673
 674 struct svm_init_data {
 675         int cpu;
 676         int r;
 677 };
 678
 679 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 680
 681 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 682 #define MSRS_RANGE_SIZE 2048
 683 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 684
 685 static u32 svm_msrpm_offset(u32 msr)
 686 {
 687         u32 offset;
 688         int i;
 689
 690         for (i = 0; i < NUM_MSR_MAPS; i++) {
 691                 if (msr < msrpm_ranges[i] ||
 692                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 693                         continue;
 694
 695                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 696                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 697
 698                 /* Now we have the u8 offset - but need the u32 offset */
 699                 return offset / 4;
 700         }
 701
 702         /* MSR not in any range */
 703         return MSR_INVALID;
 704 }
 705
 706 #define MAX_INST_SIZE 15
 707
 708 static inline void clgi(void)
 709 {
 710         asm volatile (__ex(SVM_CLGI));
 711 }
 712
 713 static inline void stgi(void)
 714 {
 715         asm volatile (__ex(SVM_STGI));
 716 }
 717
 718 static inline void invlpga(unsigned long addr, u32 asid)
 719 {
 720         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
 721 }
 722
 723 static int get_npt_level(struct kvm_vcpu *vcpu)
 724 {
 725 #ifdef CONFIG_X86_64
 726         return PT64_ROOT_4LEVEL;
 727 #else
 728         return PT32E_ROOT_LEVEL;
 729 #endif
 730 }
 731
 732 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 733 {
 734         vcpu->arch.efer = efer;
 735         if (!npt_enabled && !(efer & EFER_LMA))
 736                 efer &= ~EFER_LME;
 737
 738         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 739         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 740 }
 741
 742 static int is_external_interrupt(u32 info)
 743 {
 744         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 745         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 746 }
 747
 748 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 749 {
 750         struct vcpu_svm *svm = to_svm(vcpu);
 751         u32 ret = 0;
 752
 753         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 754                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 755         return ret;
 756 }
 757
 758 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 759 {
 760         struct vcpu_svm *svm = to_svm(vcpu);
 761
 762         if (mask == 0)
 763                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 764         else
 765                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 766
 767 }
 768
 769 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 770 {
 771         struct vcpu_svm *svm = to_svm(vcpu);
 772
 773         if (svm->vmcb->control.next_rip != 0) {
 774                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 775                 svm->next_rip = svm->vmcb->control.next_rip;
 776         }
 777
 778         if (!svm->next_rip) {
 779                 if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
 780                                 EMULATE_DONE)
 781                         printk(KERN_DEBUG "%s: NOP\n", __func__);
 782                 return;
 783         }
 784         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 785                 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
 786                        __func__, kvm_rip_read(vcpu), svm->next_rip);
 787
 788         kvm_rip_write(vcpu, svm->next_rip);
 789         svm_set_interrupt_shadow(vcpu, 0);
 790 }
 791
 792 static void svm_queue_exception(struct kvm_vcpu *vcpu)
 793 {
 794         struct vcpu_svm *svm = to_svm(vcpu);
 795         unsigned nr = vcpu->arch.exception.nr;
 796         bool has_error_code = vcpu->arch.exception.has_error_code;
 797         bool reinject = vcpu->arch.exception.injected;
 798         u32 error_code = vcpu->arch.exception.error_code;
 799
 800         /*
 801          * If we are within a nested VM we'd better #VMEXIT and let the guest
 802          * handle the exception
 803          */
 804         if (!reinject &&
 805             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 806                 return;
 807
 808         if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
 809                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 810
 811                 /*
 812                  * For guest debugging where we have to reinject #BP if some
 813                  * INT3 is guest-owned:
 814                  * Emulate nRIP by moving RIP forward. Will fail if injection
 815                  * raises a fault that is not intercepted. Still better than
 816                  * failing in all cases.
 817                  */
 818                 skip_emulated_instruction(&svm->vcpu);
 819                 rip = kvm_rip_read(&svm->vcpu);
 820                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 821                 svm->int3_injected = rip - old_rip;
 822         }
 823
 824         svm->vmcb->control.event_inj = nr
 825                 | SVM_EVTINJ_VALID
 826                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 827                 | SVM_EVTINJ_TYPE_EXEPT;
 828         svm->vmcb->control.event_inj_err = error_code;
 829 }
 830
 831 static void svm_init_erratum_383(void)
 832 {
 833         u32 low, high;
 834         int err;
 835         u64 val;
 836
 837         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 838                 return;
 839
 840         /* Use _safe variants to not break nested virtualization */
 841         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 842         if (err)
 843                 return;
 844
 845         val |= (1ULL << 47);
 846
 847         low  = lower_32_bits(val);
 848         high = upper_32_bits(val);
 849
 850         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 851
 852         erratum_383_found = true;
 853 }
 854
 855 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 856 {
 857         /*
 858          * Guests should see errata 400 and 415 as fixed (assuming that
 859          * HLT and IO instructions are intercepted).
 860          */
 861         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 862         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 863
 864         /*
 865          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 866          * all osvw.status bits inside that length, including bit 0 (which is
 867          * reserved for erratum 298), are valid. However, if host processor's
 868          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 869          * be conservative here and therefore we tell the guest that erratum 298
 870          * is present (because we really don't know).
 871          */
 872         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 873                 vcpu->arch.osvw.status |= 1;
 874 }
 875
 876 static int has_svm(void)
 877 {
 878         const char *msg;
 879
 880         if (!cpu_has_svm(&msg)) {
 881                 printk(KERN_INFO "has_svm: %s\n", msg);
 882                 return 0;
 883         }
 884
 885         return 1;
 886 }
 887
 888 static void svm_hardware_disable(void)
 889 {
 890         /* Make sure we clean up behind us */
 891         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
 892                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 893
 894         cpu_svm_disable();
 895
 896         amd_pmu_disable_virt();
 897 }
 898
 899 static int svm_hardware_enable(void)
 900 {
 901
 902         struct svm_cpu_data *sd;
 903         uint64_t efer;
 904         struct desc_struct *gdt;
 905         int me = raw_smp_processor_id();
 906
 907         rdmsrl(MSR_EFER, efer);
 908         if (efer & EFER_SVME)
 909                 return -EBUSY;
 910
 911         if (!has_svm()) {
 912                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 913                 return -EINVAL;
 914         }
 915         sd = per_cpu(svm_data, me);
 916         if (!sd) {
 917                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 918                 return -EINVAL;
 919         }
 920
 921         sd->asid_generation = 1;
 922         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 923         sd->next_asid = sd->max_asid + 1;
 924         sd->min_asid = max_sev_asid + 1;
 925
 926         gdt = get_current_gdt_rw();
 927         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 928
 929         wrmsrl(MSR_EFER, efer | EFER_SVME);
 930
 931         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 932
 933         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 934                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 935                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 936         }
 937
 938
 939         /*
 940          * Get OSVW bits.
 941          *
 942          * Note that it is possible to have a system with mixed processor
 943          * revisions and therefore different OSVW bits. If bits are not the same
 944          * on different processors then choose the worst case (i.e. if erratum
 945          * is present on one processor and not on another then assume that the
 946          * erratum is present everywhere).
 947          */
 948         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 949                 uint64_t len, status = 0;
 950                 int err;
 951
 952                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 953                 if (!err)
 954                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 955                                                       &err);
 956
 957                 if (err)
 958                         osvw_status = osvw_len = 0;
 959                 else {
 960                         if (len < osvw_len)
 961                                 osvw_len = len;
 962                         osvw_status |= status;
 963                         osvw_status &= (1ULL << osvw_len) - 1;
 964                 }
 965         } else
 966                 osvw_status = osvw_len = 0;
 967
 968         svm_init_erratum_383();
 969
 970         amd_pmu_enable_virt();
 971
 972         return 0;
 973 }
 974
 975 static void svm_cpu_uninit(int cpu)
 976 {
 977         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 978
 979         if (!sd)
 980                 return;
 981
 982         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
 983         kfree(sd->sev_vmcbs);
 984         __free_page(sd->save_area);
 985         kfree(sd);
 986 }
 987
 988 static int svm_cpu_init(int cpu)
 989 {
 990         struct svm_cpu_data *sd;
 991         int r;
 992
 993         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
 994         if (!sd)
 995                 return -ENOMEM;
 996         sd->cpu = cpu;
 997         r = -ENOMEM;
 998         sd->save_area = alloc_page(GFP_KERNEL);
 999         if (!sd->save_area)
1000                 goto err_1;
1001
1002         if (svm_sev_enabled()) {
1003                 r = -ENOMEM;
1004                 sd->sev_vmcbs = kmalloc((max_sev_asid + 1) * sizeof(void *), GFP_KERNEL);
1005                 if (!sd->sev_vmcbs)
1006                         goto err_1;
1007         }
1008
1009         per_cpu(svm_data, cpu) = sd;
1010
1011         return 0;
1012
1013 err_1:
1014         kfree(sd);
1015         return r;
1016
1017 }
1018
1019 static bool valid_msr_intercept(u32 index)
1020 {
1021         int i;
1022
1023         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
1024                 if (direct_access_msrs[i].index == index)
1025                         return true;
1026
1027         return false;
1028 }
1029
1030 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
1031 {
1032         u8 bit_write;
1033         unsigned long tmp;
1034         u32 offset;
1035         u32 *msrpm;
1036
1037         msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
1038                                       to_svm(vcpu)->msrpm;
1039
1040         offset    = svm_msrpm_offset(msr);
1041         bit_write = 2 * (msr & 0x0f) + 1;
1042         tmp       = msrpm[offset];
1043
1044         BUG_ON(offset == MSR_INVALID);
1045
1046         return !!test_bit(bit_write,  &tmp);
1047 }
1048
1049 static void set_msr_interception(u32 *msrpm, unsigned msr,
1050                                  int read, int write)
1051 {
1052         u8 bit_read, bit_write;
1053         unsigned long tmp;
1054         u32 offset;
1055
1056         /*
1057          * If this warning triggers extend the direct_access_msrs list at the
1058          * beginning of the file
1059          */
1060         WARN_ON(!valid_msr_intercept(msr));
1061
1062         offset    = svm_msrpm_offset(msr);
1063         bit_read  = 2 * (msr & 0x0f);
1064         bit_write = 2 * (msr & 0x0f) + 1;
1065         tmp       = msrpm[offset];
1066
1067         BUG_ON(offset == MSR_INVALID);
1068
1069         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
1070         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
1071
1072         msrpm[offset] = tmp;
1073 }
1074
1075 static void svm_vcpu_init_msrpm(u32 *msrpm)
1076 {
1077         int i;
1078
1079         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
1080
1081         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1082                 if (!direct_access_msrs[i].always)
1083                         continue;
1084
1085                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
1086         }
1087 }
1088
1089 static void add_msr_offset(u32 offset)
1090 {
1091         int i;
1092
1093         for (i = 0; i < MSRPM_OFFSETS; ++i) {
1094
1095                 /* Offset already in list? */
1096                 if (msrpm_offsets[i] == offset)
1097                         return;
1098
1099                 /* Slot used by another offset? */
1100                 if (msrpm_offsets[i] != MSR_INVALID)
1101                         continue;
1102
1103                 /* Add offset to list */
1104                 msrpm_offsets[i] = offset;
1105
1106                 return;
1107         }
1108
1109         /*
1110          * If this BUG triggers the msrpm_offsets table has an overflow. Just
1111          * increase MSRPM_OFFSETS in this case.
1112          */
1113         BUG();
1114 }
1115
1116 static void init_msrpm_offsets(void)
1117 {
1118         int i;
1119
1120         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
1121
1122         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1123                 u32 offset;
1124
1125                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
1126                 BUG_ON(offset == MSR_INVALID);
1127
1128                 add_msr_offset(offset);
1129         }
1130 }
1131
1132 static void svm_enable_lbrv(struct vcpu_svm *svm)
1133 {
1134         u32 *msrpm = svm->msrpm;
1135
1136         svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1137         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1138         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1139         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1140         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1141 }
1142
1143 static void svm_disable_lbrv(struct vcpu_svm *svm)
1144 {
1145         u32 *msrpm = svm->msrpm;
1146
1147         svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1148         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1149         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1150         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1151         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1152 }
1153
1154 static void disable_nmi_singlestep(struct vcpu_svm *svm)
1155 {
1156         svm->nmi_singlestep = false;
1157
1158         if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1159                 /* Clear our flags if they were not set by the guest */
1160                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1161                         svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1162                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1163                         svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1164         }
1165 }
1166
1167 /* Note:
1168  * This hash table is used to map VM_ID to a struct kvm_svm,
1169  * when handling AMD IOMMU GALOG notification to schedule in
1170  * a particular vCPU.
1171  */
1172 #define SVM_VM_DATA_HASH_BITS   8
1173 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
1174 static u32 next_vm_id = 0;
1175 static bool next_vm_id_wrapped = 0;
1176 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
1177
1178 /* Note:
1179  * This function is called from IOMMU driver to notify
1180  * SVM to schedule in a particular vCPU of a particular VM.
1181  */
1182 static int avic_ga_log_notifier(u32 ga_tag)
1183 {
1184         unsigned long flags;
1185         struct kvm_svm *kvm_svm;
1186         struct kvm_vcpu *vcpu = NULL;
1187         u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
1188         u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
1189
1190         pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
1191
1192         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1193         hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
1194                 if (kvm_svm->avic_vm_id != vm_id)
1195                         continue;
1196                 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
1197                 break;
1198         }
1199         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1200
1201         /* Note:
1202          * At this point, the IOMMU should have already set the pending
1203          * bit in the vAPIC backing page. So, we just need to schedule
1204          * in the vcpu.
1205          */
1206         if (vcpu)
1207                 kvm_vcpu_wake_up(vcpu);
1208
1209         return 0;
1210 }
1211
1212 static __init int sev_hardware_setup(void)
1213 {
1214         struct sev_user_data_status *status;
1215         int rc;
1216
1217         /* Maximum number of encrypted guests supported simultaneously */
1218         max_sev_asid = cpuid_ecx(0x8000001F);
1219
1220         if (!max_sev_asid)
1221                 return 1;
1222
1223         /* Minimum ASID value that should be used for SEV guest */
1224         min_sev_asid = cpuid_edx(0x8000001F);
1225
1226         /* Initialize SEV ASID bitmap */
1227         sev_asid_bitmap = kcalloc(BITS_TO_LONGS(max_sev_asid),
1228                                 sizeof(unsigned long), GFP_KERNEL);
1229         if (!sev_asid_bitmap)
1230                 return 1;
1231
1232         status = kmalloc(sizeof(*status), GFP_KERNEL);
1233         if (!status)
1234                 return 1;
1235
1236         /*
1237          * Check SEV platform status.
1238          *
1239          * PLATFORM_STATUS can be called in any state, if we failed to query
1240          * the PLATFORM status then either PSP firmware does not support SEV
1241          * feature or SEV firmware is dead.
1242          */
1243         rc = sev_platform_status(status, NULL);
1244         if (rc)
1245                 goto err;
1246
1247         pr_info("SEV supported\n");
1248
1249 err:
1250         kfree(status);
1251         return rc;
1252 }
1253
1254 static void grow_ple_window(struct kvm_vcpu *vcpu)
1255 {
1256         struct vcpu_svm *svm = to_svm(vcpu);
1257         struct vmcb_control_area *control = &svm->vmcb->control;
1258         int old = control->pause_filter_count;
1259
1260         control->pause_filter_count = __grow_ple_window(old,
1261                                                         pause_filter_count,
1262                                                         pause_filter_count_grow,
1263                                                         pause_filter_count_max);
1264
1265         if (control->pause_filter_count != old)
1266                 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1267
1268         trace_kvm_ple_window_grow(vcpu->vcpu_id,
1269                                   control->pause_filter_count, old);
1270 }
1271
1272 static void shrink_ple_window(struct kvm_vcpu *vcpu)
1273 {
1274         struct vcpu_svm *svm = to_svm(vcpu);
1275         struct vmcb_control_area *control = &svm->vmcb->control;
1276         int old = control->pause_filter_count;
1277
1278         control->pause_filter_count =
1279                                 __shrink_ple_window(old,
1280                                                     pause_filter_count,
1281                                                     pause_filter_count_shrink,
1282                                                     pause_filter_count);
1283         if (control->pause_filter_count != old)
1284                 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1285
1286         trace_kvm_ple_window_shrink(vcpu->vcpu_id,
1287                                     control->pause_filter_count, old);
1288 }
1289
1290 static __init int svm_hardware_setup(void)
1291 {
1292         int cpu;
1293         struct page *iopm_pages;
1294         void *iopm_va;
1295         int r;
1296
1297         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
1298
1299         if (!iopm_pages)
1300                 return -ENOMEM;
1301
1302         iopm_va = page_address(iopm_pages);
1303         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
1304         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
1305
1306         init_msrpm_offsets();
1307
1308         if (boot_cpu_has(X86_FEATURE_NX))
1309                 kvm_enable_efer_bits(EFER_NX);
1310
1311         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
1312                 kvm_enable_efer_bits(EFER_FFXSR);
1313
1314         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1315                 kvm_has_tsc_control = true;
1316                 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
1317                 kvm_tsc_scaling_ratio_frac_bits = 32;
1318         }
1319
1320         /* Check for pause filtering support */
1321         if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1322                 pause_filter_count = 0;
1323                 pause_filter_thresh = 0;
1324         } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
1325                 pause_filter_thresh = 0;
1326         }
1327
1328         if (nested) {
1329                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
1330                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
1331         }
1332
1333         if (sev) {
1334                 if (boot_cpu_has(X86_FEATURE_SEV) &&
1335                     IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
1336                         r = sev_hardware_setup();
1337                         if (r)
1338                                 sev = false;
1339                 } else {
1340                         sev = false;
1341                 }
1342         }
1343
1344         for_each_possible_cpu(cpu) {
1345                 r = svm_cpu_init(cpu);
1346                 if (r)
1347                         goto err;
1348         }
1349
1350         if (!boot_cpu_has(X86_FEATURE_NPT))
1351                 npt_enabled = false;
1352
1353         if (npt_enabled && !npt) {
1354                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
1355                 npt_enabled = false;
1356         }
1357
1358         if (npt_enabled) {
1359                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
1360                 kvm_enable_tdp();
1361         } else
1362                 kvm_disable_tdp();
1363
1364         if (avic) {
1365                 if (!npt_enabled ||
1366                     !boot_cpu_has(X86_FEATURE_AVIC) ||
1367                     !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
1368                         avic = false;
1369                 } else {
1370                         pr_info("AVIC enabled\n");
1371
1372                         amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1373                 }
1374         }
1375
1376         if (vls) {
1377                 if (!npt_enabled ||
1378                     !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1379                     !IS_ENABLED(CONFIG_X86_64)) {
1380                         vls = false;
1381                 } else {
1382                         pr_info("Virtual VMLOAD VMSAVE supported\n");
1383                 }
1384         }
1385
1386         if (vgif) {
1387                 if (!boot_cpu_has(X86_FEATURE_VGIF))
1388                         vgif = false;
1389                 else
1390                         pr_info("Virtual GIF supported\n");
1391         }
1392
1393         return 0;
1394
1395 err:
1396         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
1397         iopm_base = 0;
1398         return r;
1399 }
1400
1401 static __exit void svm_hardware_unsetup(void)
1402 {
1403         int cpu;
1404
1405         if (svm_sev_enabled())
1406                 kfree(sev_asid_bitmap);
1407
1408         for_each_possible_cpu(cpu)
1409                 svm_cpu_uninit(cpu);
1410
1411         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
1412         iopm_base = 0;
1413 }
1414
1415 static void init_seg(struct vmcb_seg *seg)
1416 {
1417         seg->selector = 0;
1418         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1419                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1420         seg->limit = 0xffff;
1421         seg->base = 0;
1422 }
1423
1424 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1425 {
1426         seg->selector = 0;
1427         seg->attrib = SVM_SELECTOR_P_MASK | type;
1428         seg->limit = 0xffff;
1429         seg->base = 0;
1430 }
1431
1432 static u64 svm_read_l1_tsc_offset(struct kvm_vcpu *vcpu)
1433 {
1434         struct vcpu_svm *svm = to_svm(vcpu);
1435
1436         if (is_guest_mode(vcpu))
1437                 return svm->nested.hsave->control.tsc_offset;
1438
1439         return vcpu->arch.tsc_offset;
1440 }
1441
1442 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1443 {
1444         struct vcpu_svm *svm = to_svm(vcpu);
1445         u64 g_tsc_offset = 0;
1446
1447         if (is_guest_mode(vcpu)) {
1448                 /* Write L1's TSC offset.  */
1449                 g_tsc_offset = svm->vmcb->control.tsc_offset -
1450                                svm->nested.hsave->control.tsc_offset;
1451                 svm->nested.hsave->control.tsc_offset = offset;
1452         } else
1453                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1454                                            svm->vmcb->control.tsc_offset,
1455                                            offset);
1456
1457         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1458
1459         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1460 }
1461
1462 static void avic_init_vmcb(struct vcpu_svm *svm)
1463 {
1464         struct vmcb *vmcb = svm->vmcb;
1465         struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
1466         phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
1467         phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
1468         phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
1469
1470         vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
1471         vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
1472         vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
1473         vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
1474         vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
1475 }
1476
1477 static void init_vmcb(struct vcpu_svm *svm)
1478 {
1479         struct vmcb_control_area *control = &svm->vmcb->control;
1480         struct vmcb_save_area *save = &svm->vmcb->save;
1481
1482         svm->vcpu.arch.hflags = 0;
1483
1484         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1485         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1486         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1487         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1488         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1489         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1490         if (!kvm_vcpu_apicv_active(&svm->vcpu))
1491                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1492
1493         set_dr_intercepts(svm);
1494
1495         set_exception_intercept(svm, PF_VECTOR);
1496         set_exception_intercept(svm, UD_VECTOR);
1497         set_exception_intercept(svm, MC_VECTOR);
1498         set_exception_intercept(svm, AC_VECTOR);
1499         set_exception_intercept(svm, DB_VECTOR);
1500         /*
1501          * Guest access to VMware backdoor ports could legitimately
1502          * trigger #GP because of TSS I/O permission bitmap.
1503          * We intercept those #GP and allow access to them anyway
1504          * as VMware does.
1505          */
1506         if (enable_vmware_backdoor)
1507                 set_exception_intercept(svm, GP_VECTOR);
1508
1509         set_intercept(svm, INTERCEPT_INTR);
1510         set_intercept(svm, INTERCEPT_NMI);
1511         set_intercept(svm, INTERCEPT_SMI);
1512         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1513         set_intercept(svm, INTERCEPT_RDPMC);
1514         set_intercept(svm, INTERCEPT_CPUID);
1515         set_intercept(svm, INTERCEPT_INVD);
1516         set_intercept(svm, INTERCEPT_INVLPG);
1517         set_intercept(svm, INTERCEPT_INVLPGA);
1518         set_intercept(svm, INTERCEPT_IOIO_PROT);
1519         set_intercept(svm, INTERCEPT_MSR_PROT);
1520         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1521         set_intercept(svm, INTERCEPT_SHUTDOWN);
1522         set_intercept(svm, INTERCEPT_VMRUN);
1523         set_intercept(svm, INTERCEPT_VMMCALL);
1524         set_intercept(svm, INTERCEPT_VMLOAD);
1525         set_intercept(svm, INTERCEPT_VMSAVE);
1526         set_intercept(svm, INTERCEPT_STGI);
1527         set_intercept(svm, INTERCEPT_CLGI);
1528         set_intercept(svm, INTERCEPT_SKINIT);
1529         set_intercept(svm, INTERCEPT_WBINVD);
1530         set_intercept(svm, INTERCEPT_XSETBV);
1531         set_intercept(svm, INTERCEPT_RSM);
1532
1533         if (!kvm_mwait_in_guest(svm->vcpu.kvm)) {
1534                 set_intercept(svm, INTERCEPT_MONITOR);
1535                 set_intercept(svm, INTERCEPT_MWAIT);
1536         }
1537
1538         if (!kvm_hlt_in_guest(svm->vcpu.kvm))
1539                 set_intercept(svm, INTERCEPT_HLT);
1540
1541         control->iopm_base_pa = __sme_set(iopm_base);
1542         control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1543         control->int_ctl = V_INTR_MASKING_MASK;
1544
1545         init_seg(&save->es);
1546         init_seg(&save->ss);
1547         init_seg(&save->ds);
1548         init_seg(&save->fs);
1549         init_seg(&save->gs);
1550
1551         save->cs.selector = 0xf000;
1552         save->cs.base = 0xffff0000;
1553         /* Executable/Readable Code Segment */
1554         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1555                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1556         save->cs.limit = 0xffff;
1557
1558         save->gdtr.limit = 0xffff;
1559         save->idtr.limit = 0xffff;
1560
1561         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1562         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1563
1564         svm_set_efer(&svm->vcpu, 0);
1565         save->dr6 = 0xffff0ff0;
1566         kvm_set_rflags(&svm->vcpu, 2);
1567         save->rip = 0x0000fff0;
1568         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1569
1570         /*
1571          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1572          * It also updates the guest-visible cr0 value.
1573          */
1574         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1575         kvm_mmu_reset_context(&svm->vcpu);
1576
1577         save->cr4 = X86_CR4_PAE;
1578         /* rdx = ?? */
1579
1580         if (npt_enabled) {
1581                 /* Setup VMCB for Nested Paging */
1582                 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1583                 clr_intercept(svm, INTERCEPT_INVLPG);
1584                 clr_exception_intercept(svm, PF_VECTOR);
1585                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1586                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1587                 save->g_pat = svm->vcpu.arch.pat;
1588                 save->cr3 = 0;
1589                 save->cr4 = 0;
1590         }
1591         svm->asid_generation = 0;
1592
1593         svm->nested.vmcb = 0;
1594         svm->vcpu.arch.hflags = 0;
1595
1596         if (pause_filter_count) {
1597                 control->pause_filter_count = pause_filter_count;
1598                 if (pause_filter_thresh)
1599                         control->pause_filter_thresh = pause_filter_thresh;
1600                 set_intercept(svm, INTERCEPT_PAUSE);
1601         } else {
1602                 clr_intercept(svm, INTERCEPT_PAUSE);
1603         }
1604
1605         if (kvm_vcpu_apicv_active(&svm->vcpu))
1606                 avic_init_vmcb(svm);
1607
1608         /*
1609          * If hardware supports Virtual VMLOAD VMSAVE then enable it
1610          * in VMCB and clear intercepts to avoid #VMEXIT.
1611          */
1612         if (vls) {
1613                 clr_intercept(svm, INTERCEPT_VMLOAD);
1614                 clr_intercept(svm, INTERCEPT_VMSAVE);
1615                 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1616         }
1617
1618         if (vgif) {
1619                 clr_intercept(svm, INTERCEPT_STGI);
1620                 clr_intercept(svm, INTERCEPT_CLGI);
1621                 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1622         }
1623
1624         if (sev_guest(svm->vcpu.kvm)) {
1625                 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1626                 clr_exception_intercept(svm, UD_VECTOR);
1627         }
1628
1629         mark_all_dirty(svm->vmcb);
1630
1631         enable_gif(svm);
1632
1633 }
1634
1635 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
1636                                        unsigned int index)
1637 {
1638         u64 *avic_physical_id_table;
1639         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
1640
1641         if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
1642                 return NULL;
1643
1644         avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page);
1645
1646         return &avic_physical_id_table[index];
1647 }
1648
1649 /**
1650  * Note:
1651  * AVIC hardware walks the nested page table to check permissions,
1652  * but does not use the SPA address specified in the leaf page
1653  * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
1654  * field of the VMCB. Therefore, we set up the
1655  * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
1656  */
1657 static int avic_init_access_page(struct kvm_vcpu *vcpu)
1658 {
1659         struct kvm *kvm = vcpu->kvm;
1660         int ret;
1661
1662         if (kvm->arch.apic_access_page_done)
1663                 return 0;
1664
1665         ret = x86_set_memory_region(kvm,
1666                                     APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
1667                                     APIC_DEFAULT_PHYS_BASE,
1668                                     PAGE_SIZE);
1669         if (ret)
1670                 return ret;
1671
1672         kvm->arch.apic_access_page_done = true;
1673         return 0;
1674 }
1675
1676 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
1677 {
1678         int ret;
1679         u64 *entry, new_entry;
1680         int id = vcpu->vcpu_id;
1681         struct vcpu_svm *svm = to_svm(vcpu);
1682
1683         ret = avic_init_access_page(vcpu);
1684         if (ret)
1685                 return ret;
1686
1687         if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
1688                 return -EINVAL;
1689
1690         if (!svm->vcpu.arch.apic->regs)
1691                 return -EINVAL;
1692
1693         svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);
1694
1695         /* Setting AVIC backing page address in the phy APIC ID table */
1696         entry = avic_get_physical_id_entry(vcpu, id);
1697         if (!entry)
1698                 return -EINVAL;
1699
1700         new_entry = READ_ONCE(*entry);
1701         new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
1702                               AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
1703                               AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
1704         WRITE_ONCE(*entry, new_entry);
1705
1706         svm->avic_physical_id_cache = entry;
1707
1708         return 0;
1709 }
1710
1711 static void __sev_asid_free(int asid)
1712 {
1713         struct svm_cpu_data *sd;
1714         int cpu, pos;
1715
1716         pos = asid - 1;
1717         clear_bit(pos, sev_asid_bitmap);
1718
1719         for_each_possible_cpu(cpu) {
1720                 sd = per_cpu(svm_data, cpu);
1721                 sd->sev_vmcbs[pos] = NULL;
1722         }
1723 }
1724
1725 static void sev_asid_free(struct kvm *kvm)
1726 {
1727         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1728
1729         __sev_asid_free(sev->asid);
1730 }
1731
1732 static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
1733 {
1734         struct sev_data_decommission *decommission;
1735         struct sev_data_deactivate *data;
1736
1737         if (!handle)
1738                 return;
1739
1740         data = kzalloc(sizeof(*data), GFP_KERNEL);
1741         if (!data)
1742                 return;
1743
1744         /* deactivate handle */
1745         data->handle = handle;
1746         sev_guest_deactivate(data, NULL);
1747
1748         wbinvd_on_all_cpus();
1749         sev_guest_df_flush(NULL);
1750         kfree(data);
1751
1752         decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
1753         if (!decommission)
1754                 return;
1755
1756         /* decommission handle */
1757         decommission->handle = handle;
1758         sev_guest_decommission(decommission, NULL);
1759
1760         kfree(decommission);
1761 }
1762
1763 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
1764                                     unsigned long ulen, unsigned long *n,
1765                                     int write)
1766 {
1767         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1768         unsigned long npages, npinned, size;
1769         unsigned long locked, lock_limit;
1770         struct page **pages;
1771         int first, last;
1772
1773         /* Calculate number of pages. */
1774         first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
1775         last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
1776         npages = (last - first + 1);
1777
1778         locked = sev->pages_locked + npages;
1779         lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1780         if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
1781                 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
1782                 return NULL;
1783         }
1784
1785         /* Avoid using vmalloc for smaller buffers. */
1786         size = npages * sizeof(struct page *);
1787         if (size > PAGE_SIZE)
1788                 pages = vmalloc(size);
1789         else
1790                 pages = kmalloc(size, GFP_KERNEL);
1791
1792         if (!pages)
1793                 return NULL;
1794
1795         /* Pin the user virtual address. */
1796         npinned = get_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages);
1797         if (npinned != npages) {
1798                 pr_err("SEV: Failure locking %lu pages.\n", npages);
1799                 goto err;
1800         }
1801
1802         *n = npages;
1803         sev->pages_locked = locked;
1804
1805         return pages;
1806
1807 err:
1808         if (npinned > 0)
1809                 release_pages(pages, npinned);
1810
1811         kvfree(pages);
1812         return NULL;
1813 }
1814
1815 static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
1816                              unsigned long npages)
1817 {
1818         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1819
1820         release_pages(pages, npages);
1821         kvfree(pages);
1822         sev->pages_locked -= npages;
1823 }
1824
1825 static void sev_clflush_pages(struct page *pages[], unsigned long npages)
1826 {
1827         uint8_t *page_virtual;
1828         unsigned long i;
1829
1830         if (npages == 0 || pages == NULL)
1831                 return;
1832
1833         for (i = 0; i < npages; i++) {
1834                 page_virtual = kmap_atomic(pages[i]);
1835                 clflush_cache_range(page_virtual, PAGE_SIZE);
1836                 kunmap_atomic(page_virtual);
1837         }
1838 }
1839
1840 static void __unregister_enc_region_locked(struct kvm *kvm,
1841                                            struct enc_region *region)
1842 {
1843         /*
1844          * The guest may change the memory encryption attribute from C=0 -> C=1
1845          * or vice versa for this memory range. Lets make sure caches are
1846          * flushed to ensure that guest data gets written into memory with
1847          * correct C-bit.
1848          */
1849         sev_clflush_pages(region->pages, region->npages);
1850
1851         sev_unpin_memory(kvm, region->pages, region->npages);
1852         list_del(&region->list);
1853         kfree(region);
1854 }
1855
1856 static struct kvm *svm_vm_alloc(void)
1857 {
1858         struct kvm_svm *kvm_svm = kzalloc(sizeof(struct kvm_svm), GFP_KERNEL);
1859         return &kvm_svm->kvm;
1860 }
1861
1862 static void svm_vm_free(struct kvm *kvm)
1863 {
1864         kfree(to_kvm_svm(kvm));
1865 }
1866
1867 static void sev_vm_destroy(struct kvm *kvm)
1868 {
1869         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1870         struct list_head *head = &sev->regions_list;
1871         struct list_head *pos, *q;
1872
1873         if (!sev_guest(kvm))
1874                 return;
1875
1876         mutex_lock(&kvm->lock);
1877
1878         /*
1879          * if userspace was terminated before unregistering the memory regions
1880          * then lets unpin all the registered memory.
1881          */
1882         if (!list_empty(head)) {
1883                 list_for_each_safe(pos, q, head) {
1884                         __unregister_enc_region_locked(kvm,
1885                                 list_entry(pos, struct enc_region, list));
1886                 }
1887         }
1888
1889         mutex_unlock(&kvm->lock);
1890
1891         sev_unbind_asid(kvm, sev->handle);
1892         sev_asid_free(kvm);
1893 }
1894
1895 static void avic_vm_destroy(struct kvm *kvm)
1896 {
1897         unsigned long flags;
1898         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
1899
1900         if (!avic)
1901                 return;
1902
1903         if (kvm_svm->avic_logical_id_table_page)
1904                 __free_page(kvm_svm->avic_logical_id_table_page);
1905         if (kvm_svm->avic_physical_id_table_page)
1906                 __free_page(kvm_svm->avic_physical_id_table_page);
1907
1908         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1909         hash_del(&kvm_svm->hnode);
1910         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1911 }
1912
1913 static void svm_vm_destroy(struct kvm *kvm)
1914 {
1915         avic_vm_destroy(kvm);
1916         sev_vm_destroy(kvm);
1917 }
1918
1919 static int avic_vm_init(struct kvm *kvm)
1920 {
1921         unsigned long flags;
1922         int err = -ENOMEM;
1923         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
1924         struct kvm_svm *k2;
1925         struct page *p_page;
1926         struct page *l_page;
1927         u32 vm_id;
1928
1929         if (!avic)
1930                 return 0;
1931
1932         /* Allocating physical APIC ID table (4KB) */
1933         p_page = alloc_page(GFP_KERNEL);
1934         if (!p_page)
1935                 goto free_avic;
1936
1937         kvm_svm->avic_physical_id_table_page = p_page;
1938         clear_page(page_address(p_page));
1939
1940         /* Allocating logical APIC ID table (4KB) */
1941         l_page = alloc_page(GFP_KERNEL);
1942         if (!l_page)
1943                 goto free_avic;
1944
1945         kvm_svm->avic_logical_id_table_page = l_page;
1946         clear_page(page_address(l_page));
1947
1948         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1949  again:
1950         vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
1951         if (vm_id == 0) { /* id is 1-based, zero is not okay */
1952                 next_vm_id_wrapped = 1;
1953                 goto again;
1954         }
1955         /* Is it still in use? Only possible if wrapped at least once */
1956         if (next_vm_id_wrapped) {
1957                 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
1958                         if (k2->avic_vm_id == vm_id)
1959                                 goto again;
1960                 }
1961         }
1962         kvm_svm->avic_vm_id = vm_id;
1963         hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
1964         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1965
1966         return 0;
1967
1968 free_avic:
1969         avic_vm_destroy(kvm);
1970         return err;
1971 }
1972
1973 static inline int
1974 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
1975 {
1976         int ret = 0;
1977         unsigned long flags;
1978         struct amd_svm_iommu_ir *ir;
1979         struct vcpu_svm *svm = to_svm(vcpu);
1980
1981         if (!kvm_arch_has_assigned_device(vcpu->kvm))
1982                 return 0;
1983
1984         /*
1985          * Here, we go through the per-vcpu ir_list to update all existing
1986          * interrupt remapping table entry targeting this vcpu.
1987          */
1988         spin_lock_irqsave(&svm->ir_list_lock, flags);
1989
1990         if (list_empty(&svm->ir_list))
1991                 goto out;
1992
1993         list_for_each_entry(ir, &svm->ir_list, node) {
1994                 ret = amd_iommu_update_ga(cpu, r, ir->data);
1995                 if (ret)
1996                         break;
1997         }
1998 out:
1999         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
2000         return ret;
2001 }
2002
2003 static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2004 {
2005         u64 entry;
2006         /* ID = 0xff (broadcast), ID > 0xff (reserved) */
2007         int h_physical_id = kvm_cpu_get_apicid(cpu);
2008         struct vcpu_svm *svm = to_svm(vcpu);
2009
2010         if (!kvm_vcpu_apicv_active(vcpu))
2011                 return;
2012
2013         if (WARN_ON(h_physical_id >= AVIC_MAX_PHYSICAL_ID_COUNT))
2014                 return;
2015
2016         entry = READ_ONCE(*(svm->avic_physical_id_cache));
2017         WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
2018
2019         entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
2020         entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
2021
2022         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2023         if (svm->avic_is_running)
2024                 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2025
2026         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
2027         avic_update_iommu_vcpu_affinity(vcpu, h_physical_id,
2028                                         svm->avic_is_running);
2029 }
2030
2031 static void avic_vcpu_put(struct kvm_vcpu *vcpu)
2032 {
2033         u64 entry;
2034         struct vcpu_svm *svm = to_svm(vcpu);
2035
2036         if (!kvm_vcpu_apicv_active(vcpu))
2037                 return;
2038
2039         entry = READ_ONCE(*(svm->avic_physical_id_cache));
2040         if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
2041                 avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
2042
2043         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2044         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
2045 }
2046
2047 /**
2048  * This function is called during VCPU halt/unhalt.
2049  */
2050 static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
2051 {
2052         struct vcpu_svm *svm = to_svm(vcpu);
2053
2054         svm->avic_is_running = is_run;
2055         if (is_run)
2056                 avic_vcpu_load(vcpu, vcpu->cpu);
2057         else
2058                 avic_vcpu_put(vcpu);
2059 }
2060
2061 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
2062 {
2063         struct vcpu_svm *svm = to_svm(vcpu);
2064         u32 dummy;
2065         u32 eax = 1;
2066
2067         vcpu->arch.microcode_version = 0x01000065;
2068         svm->spec_ctrl = 0;
2069         svm->virt_spec_ctrl = 0;
2070
2071         if (!init_event) {
2072                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
2073                                            MSR_IA32_APICBASE_ENABLE;
2074                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
2075                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
2076         }
2077         init_vmcb(svm);
2078
2079         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true);
2080         kvm_register_write(vcpu, VCPU_REGS_RDX, eax);
2081
2082         if (kvm_vcpu_apicv_active(vcpu) && !init_event)
2083                 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
2084 }
2085
2086 static int avic_init_vcpu(struct vcpu_svm *svm)
2087 {
2088         int ret;
2089
2090         if (!kvm_vcpu_apicv_active(&svm->vcpu))
2091                 return 0;
2092
2093         ret = avic_init_backing_page(&svm->vcpu);
2094         if (ret)
2095                 return ret;
2096
2097         INIT_LIST_HEAD(&svm->ir_list);
2098         spin_lock_init(&svm->ir_list_lock);
2099
2100         return ret;
2101 }
2102
2103 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
2104 {
2105         struct vcpu_svm *svm;
2106         struct page *page;
2107         struct page *msrpm_pages;
2108         struct page *hsave_page;
2109         struct page *nested_msrpm_pages;
2110         int err;
2111
2112         svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2113         if (!svm) {
2114                 err = -ENOMEM;
2115                 goto out;
2116         }
2117
2118         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
2119         if (err)
2120                 goto free_svm;
2121
2122         err = -ENOMEM;
2123         page = alloc_page(GFP_KERNEL);
2124         if (!page)
2125                 goto uninit;
2126
2127         msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
2128         if (!msrpm_pages)
2129                 goto free_page1;
2130
2131         nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
2132         if (!nested_msrpm_pages)
2133                 goto free_page2;
2134
2135         hsave_page = alloc_page(GFP_KERNEL);
2136         if (!hsave_page)
2137                 goto free_page3;
2138
2139         err = avic_init_vcpu(svm);
2140         if (err)
2141                 goto free_page4;
2142
2143         /* We initialize this flag to true to make sure that the is_running
2144          * bit would be set the first time the vcpu is loaded.
2145          */
2146         svm->avic_is_running = true;
2147
2148         svm->nested.hsave = page_address(hsave_page);
2149
2150         svm->msrpm = page_address(msrpm_pages);
2151         svm_vcpu_init_msrpm(svm->msrpm);
2152
2153         svm->nested.msrpm = page_address(nested_msrpm_pages);
2154         svm_vcpu_init_msrpm(svm->nested.msrpm);
2155
2156         svm->vmcb = page_address(page);
2157         clear_page(svm->vmcb);
2158         svm->vmcb_pa = __sme_set(page_to_pfn(page) << PAGE_SHIFT);
2159         svm->asid_generation = 0;
2160         init_vmcb(svm);
2161
2162         svm_init_osvw(&svm->vcpu);
2163
2164         return &svm->vcpu;
2165
2166 free_page4:
2167         __free_page(hsave_page);
2168 free_page3:
2169         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
2170 free_page2:
2171         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
2172 free_page1:
2173         __free_page(page);
2174 uninit:
2175         kvm_vcpu_uninit(&svm->vcpu);
2176 free_svm:
2177         kmem_cache_free(kvm_vcpu_cache, svm);
2178 out:
2179         return ERR_PTR(err);
2180 }
2181
2182 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
2183 {
2184         struct vcpu_svm *svm = to_svm(vcpu);
2185
2186         __free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT));
2187         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
2188         __free_page(virt_to_page(svm->nested.hsave));
2189         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
2190         kvm_vcpu_uninit(vcpu);
2191         kmem_cache_free(kvm_vcpu_cache, svm);
2192         /*
2193          * The vmcb page can be recycled, causing a false negative in
2194          * svm_vcpu_load(). So do a full IBPB now.
2195          */
2196         indirect_branch_prediction_barrier();
2197 }
2198
2199 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2200 {
2201         struct vcpu_svm *svm = to_svm(vcpu);
2202         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2203         int i;
2204
2205         if (unlikely(cpu != vcpu->cpu)) {
2206                 svm->asid_generation = 0;
2207                 mark_all_dirty(svm->vmcb);
2208         }
2209
2210 #ifdef CONFIG_X86_64
2211         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
2212 #endif
2213         savesegment(fs, svm->host.fs);
2214         savesegment(gs, svm->host.gs);
2215         svm->host.ldt = kvm_read_ldt();
2216
2217         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2218                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2219
2220         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
2221                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
2222                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
2223                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
2224                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
2225                 }
2226         }
2227         /* This assumes that the kernel never uses MSR_TSC_AUX */
2228         if (static_cpu_has(X86_FEATURE_RDTSCP))
2229                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
2230
2231         if (sd->current_vmcb != svm->vmcb) {
2232                 sd->current_vmcb = svm->vmcb;
2233                 indirect_branch_prediction_barrier();
2234         }
2235         avic_vcpu_load(vcpu, cpu);
2236 }
2237
2238 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
2239 {
2240         struct vcpu_svm *svm = to_svm(vcpu);
2241         int i;
2242
2243         avic_vcpu_put(vcpu);
2244
2245         ++vcpu->stat.host_state_reload;
2246         kvm_load_ldt(svm->host.ldt);
2247 #ifdef CONFIG_X86_64
2248         loadsegment(fs, svm->host.fs);
2249         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
2250         load_gs_index(svm->host.gs);
2251 #else
2252 #ifdef CONFIG_X86_32_LAZY_GS
2253         loadsegment(gs, svm->host.gs);
2254 #endif
2255 #endif
2256         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2257                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2258 }
2259
2260 static void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
2261 {
2262         avic_set_running(vcpu, false);
2263 }
2264
2265 static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu)
2266 {
2267         avic_set_running(vcpu, true);
2268 }
2269
2270 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
2271 {
2272         struct vcpu_svm *svm = to_svm(vcpu);
2273         unsigned long rflags = svm->vmcb->save.rflags;
2274
2275         if (svm->nmi_singlestep) {
2276                 /* Hide our flags if they were not set by the guest */
2277                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
2278                         rflags &= ~X86_EFLAGS_TF;
2279                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
2280                         rflags &= ~X86_EFLAGS_RF;
2281         }
2282         return rflags;
2283 }
2284
2285 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
2286 {
2287         if (to_svm(vcpu)->nmi_singlestep)
2288                 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
2289
2290        /*
2291         * Any change of EFLAGS.VM is accompanied by a reload of SS
2292         * (caused by either a task switch or an inter-privilege IRET),
2293         * so we do not need to update the CPL here.
2294         */
2295         to_svm(vcpu)->vmcb->save.rflags = rflags;
2296 }
2297
2298 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2299 {
2300         switch (reg) {
2301         case VCPU_EXREG_PDPTR:
2302                 BUG_ON(!npt_enabled);
2303                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
2304                 break;
2305         default:
2306                 BUG();
2307         }
2308 }
2309
2310 static void svm_set_vintr(struct vcpu_svm *svm)
2311 {
2312         set_intercept(svm, INTERCEPT_VINTR);
2313 }
2314
2315 static void svm_clear_vintr(struct vcpu_svm *svm)
2316 {
2317         clr_intercept(svm, INTERCEPT_VINTR);
2318 }
2319
2320 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
2321 {
2322         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2323
2324         switch (seg) {
2325         case VCPU_SREG_CS: return &save->cs;
2326         case VCPU_SREG_DS: return &save->ds;
2327         case VCPU_SREG_ES: return &save->es;
2328         case VCPU_SREG_FS: return &save->fs;
2329         case VCPU_SREG_GS: return &save->gs;
2330         case VCPU_SREG_SS: return &save->ss;
2331         case VCPU_SREG_TR: return &save->tr;
2332         case VCPU_SREG_LDTR: return &save->ldtr;
2333         }
2334         BUG();
2335         return NULL;
2336 }
2337
2338 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
2339 {
2340         struct vmcb_seg *s = svm_seg(vcpu, seg);
2341
2342         return s->base;
2343 }
2344
2345 static void svm_get_segment(struct kvm_vcpu *vcpu,
2346                             struct kvm_segment *var, int seg)
2347 {
2348         struct vmcb_seg *s = svm_seg(vcpu, seg);
2349
2350         var->base = s->base;
2351         var->limit = s->limit;
2352         var->selector = s->selector;
2353         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
2354         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
2355         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
2356         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
2357         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
2358         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
2359         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
2360
2361         /*
2362          * AMD CPUs circa 2014 track the G bit for all segments except CS.
2363          * However, the SVM spec states that the G bit is not observed by the
2364          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
2365          * So let's synthesize a legal G bit for all segments, this helps
2366          * running KVM nested. It also helps cross-vendor migration, because
2367          * Intel's vmentry has a check on the 'G' bit.
2368          */
2369         var->g = s->limit > 0xfffff;
2370
2371         /*
2372          * AMD's VMCB does not have an explicit unusable field, so emulate it
2373          * for cross vendor migration purposes by "not present"
2374          */
2375         var->unusable = !var->present;
2376
2377         switch (seg) {
2378         case VCPU_SREG_TR:
2379                 /*
2380                  * Work around a bug where the busy flag in the tr selector
2381                  * isn't exposed
2382                  */
2383                 var->type |= 0x2;
2384                 break;
2385         case VCPU_SREG_DS:
2386         case VCPU_SREG_ES:
2387         case VCPU_SREG_FS:
2388         case VCPU_SREG_GS:
2389                 /*
2390                  * The accessed bit must always be set in the segment
2391                  * descriptor cache, although it can be cleared in the
2392                  * descriptor, the cached bit always remains at 1. Since
2393                  * Intel has a check on this, set it here to support
2394                  * cross-vendor migration.
2395                  */
2396                 if (!var->unusable)
2397                         var->type |= 0x1;
2398                 break;
2399         case VCPU_SREG_SS:
2400                 /*
2401                  * On AMD CPUs sometimes the DB bit in the segment
2402                  * descriptor is left as 1, although the whole segment has
2403                  * been made unusable. Clear it here to pass an Intel VMX
2404                  * entry check when cross vendor migrating.
2405                  */
2406                 if (var->unusable)
2407                         var->db = 0;
2408                 /* This is symmetric with svm_set_segment() */
2409                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
2410                 break;
2411         }
2412 }
2413
2414 static int svm_get_cpl(struct kvm_vcpu *vcpu)
2415 {
2416         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2417
2418         return save->cpl;
2419 }
2420
2421 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2422 {
2423         struct vcpu_svm *svm = to_svm(vcpu);
2424
2425         dt->size = svm->vmcb->save.idtr.limit;
2426         dt->address = svm->vmcb->save.idtr.base;
2427 }
2428
2429 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2430 {
2431         struct vcpu_svm *svm = to_svm(vcpu);
2432
2433         svm->vmcb->save.idtr.limit = dt->size;
2434         svm->vmcb->save.idtr.base = dt->address ;
2435         mark_dirty(svm->vmcb, VMCB_DT);
2436 }
2437
2438 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2439 {
2440         struct vcpu_svm *svm = to_svm(vcpu);
2441
2442         dt->size = svm->vmcb->save.gdtr.limit;
2443         dt->address = svm->vmcb->save.gdtr.base;
2444 }
2445
2446 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2447 {
2448         struct vcpu_svm *svm = to_svm(vcpu);
2449
2450         svm->vmcb->save.gdtr.limit = dt->size;
2451         svm->vmcb->save.gdtr.base = dt->address ;
2452         mark_dirty(svm->vmcb, VMCB_DT);
2453 }
2454
2455 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
2456 {
2457 }
2458
2459 static void svm_decache_cr3(struct kvm_vcpu *vcpu)
2460 {
2461 }
2462
2463 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
2464 {
2465 }
2466
2467 static void update_cr0_intercept(struct vcpu_svm *svm)
2468 {
2469         ulong gcr0 = svm->vcpu.arch.cr0;
2470         u64 *hcr0 = &svm->vmcb->save.cr0;
2471
2472         *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
2473                 | (gcr0 & SVM_CR0_SELECTIVE_MASK);
2474
2475         mark_dirty(svm->vmcb, VMCB_CR);
2476
2477         if (gcr0 == *hcr0) {
2478                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
2479                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2480         } else {
2481                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
2482                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2483         }
2484 }
2485
2486 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
2487 {
2488         struct vcpu_svm *svm = to_svm(vcpu);
2489
2490 #ifdef CONFIG_X86_64
2491         if (vcpu->arch.efer & EFER_LME) {
2492                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
2493                         vcpu->arch.efer |= EFER_LMA;
2494                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
2495                 }
2496
2497                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
2498                         vcpu->arch.efer &= ~EFER_LMA;
2499                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
2500                 }
2501         }
2502 #endif
2503         vcpu->arch.cr0 = cr0;
2504
2505         if (!npt_enabled)
2506                 cr0 |= X86_CR0_PG | X86_CR0_WP;
2507
2508         /*
2509          * re-enable caching here because the QEMU bios
2510          * does not do it - this results in some delay at
2511          * reboot
2512          */
2513         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
2514                 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
2515         svm->vmcb->save.cr0 = cr0;
2516         mark_dirty(svm->vmcb, VMCB_CR);
2517         update_cr0_intercept(svm);
2518 }
2519
2520 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
2521 {
2522         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
2523         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
2524
2525         if (cr4 & X86_CR4_VMXE)
2526                 return 1;
2527
2528         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
2529                 svm_flush_tlb(vcpu, true);
2530
2531         vcpu->arch.cr4 = cr4;
2532         if (!npt_enabled)
2533                 cr4 |= X86_CR4_PAE;
2534         cr4 |= host_cr4_mce;
2535         to_svm(vcpu)->vmcb->save.cr4 = cr4;
2536         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
2537         return 0;
2538 }
2539
2540 static void svm_set_segment(struct kvm_vcpu *vcpu,
2541                             struct kvm_segment *var, int seg)
2542 {
2543         struct vcpu_svm *svm = to_svm(vcpu);
2544         struct vmcb_seg *s = svm_seg(vcpu, seg);
2545
2546         s->base = var->base;
2547         s->limit = var->limit;
2548         s->selector = var->selector;
2549         s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
2550         s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
2551         s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
2552         s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
2553         s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
2554         s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
2555         s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
2556         s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
2557
2558         /*
2559          * This is always accurate, except if SYSRET returned to a segment
2560          * with SS.DPL != 3.  Intel does not have this quirk, and always
2561          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
2562          * would entail passing the CPL to userspace and back.
2563          */
2564         if (seg == VCPU_SREG_SS)
2565                 /* This is symmetric with svm_get_segment() */
2566                 svm->vmcb->save.cpl = (var->dpl & 3);
2567
2568         mark_dirty(svm->vmcb, VMCB_SEG);
2569 }
2570
2571 static void update_bp_intercept(struct kvm_vcpu *vcpu)
2572 {
2573         struct vcpu_svm *svm = to_svm(vcpu);
2574
2575         clr_exception_intercept(svm, BP_VECTOR);
2576
2577         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
2578                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2579                         set_exception_intercept(svm, BP_VECTOR);
2580         } else
2581                 vcpu->guest_debug = 0;
2582 }
2583
2584 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
2585 {
2586         if (sd->next_asid > sd->max_asid) {
2587                 ++sd->asid_generation;
2588                 sd->next_asid = sd->min_asid;
2589                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
2590         }
2591
2592         svm->asid_generation = sd->asid_generation;
2593         svm->vmcb->control.asid = sd->next_asid++;
2594
2595         mark_dirty(svm->vmcb, VMCB_ASID);
2596 }
2597
2598 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
2599 {
2600         return to_svm(vcpu)->vmcb->save.dr6;
2601 }
2602
2603 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
2604 {
2605         struct vcpu_svm *svm = to_svm(vcpu);
2606
2607         svm->vmcb->save.dr6 = value;
2608         mark_dirty(svm->vmcb, VMCB_DR);
2609 }
2610
2611 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2612 {
2613         struct vcpu_svm *svm = to_svm(vcpu);
2614
2615         get_debugreg(vcpu->arch.db[0], 0);
2616         get_debugreg(vcpu->arch.db[1], 1);
2617         get_debugreg(vcpu->arch.db[2], 2);
2618         get_debugreg(vcpu->arch.db[3], 3);
2619         vcpu->arch.dr6 = svm_get_dr6(vcpu);
2620         vcpu->arch.dr7 = svm->vmcb->save.dr7;
2621
2622         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2623         set_dr_intercepts(svm);
2624 }
2625
2626 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2627 {
2628         struct vcpu_svm *svm = to_svm(vcpu);
2629
2630         svm->vmcb->save.dr7 = value;
2631         mark_dirty(svm->vmcb, VMCB_DR);
2632 }
2633
2634 static int pf_interception(struct vcpu_svm *svm)
2635 {
2636         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2637         u64 error_code = svm->vmcb->control.exit_info_1;
2638
2639         return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
2640                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2641                         svm->vmcb->control.insn_bytes : NULL,
2642                         svm->vmcb->control.insn_len);
2643 }
2644
2645 static int npf_interception(struct vcpu_svm *svm)
2646 {
2647         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2648         u64 error_code = svm->vmcb->control.exit_info_1;
2649
2650         trace_kvm_page_fault(fault_address, error_code);
2651         return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
2652                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2653                         svm->vmcb->control.insn_bytes : NULL,
2654                         svm->vmcb->control.insn_len);
2655 }
2656
2657 static int db_interception(struct vcpu_svm *svm)
2658 {
2659         struct kvm_run *kvm_run = svm->vcpu.run;
2660
2661         if (!(svm->vcpu.guest_debug &
2662               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2663                 !svm->nmi_singlestep) {
2664                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
2665                 return 1;
2666         }
2667
2668         if (svm->nmi_singlestep) {
2669                 disable_nmi_singlestep(svm);
2670         }
2671
2672         if (svm->vcpu.guest_debug &
2673             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2674                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2675                 kvm_run->debug.arch.pc =
2676                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2677                 kvm_run->debug.arch.exception = DB_VECTOR;
2678                 return 0;
2679         }
2680
2681         return 1;
2682 }
2683
2684 static int bp_interception(struct vcpu_svm *svm)
2685 {
2686         struct kvm_run *kvm_run = svm->vcpu.run;
2687
2688         kvm_run->exit_reason = KVM_EXIT_DEBUG;
2689         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2690         kvm_run->debug.arch.exception = BP_VECTOR;
2691         return 0;
2692 }
2693
2694 static int ud_interception(struct vcpu_svm *svm)
2695 {
2696         return handle_ud(&svm->vcpu);
2697 }
2698
2699 static int ac_interception(struct vcpu_svm *svm)
2700 {
2701         kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
2702         return 1;
2703 }
2704
2705 static int gp_interception(struct vcpu_svm *svm)
2706 {
2707         struct kvm_vcpu *vcpu = &svm->vcpu;
2708         u32 error_code = svm->vmcb->control.exit_info_1;
2709         int er;
2710
2711         WARN_ON_ONCE(!enable_vmware_backdoor);
2712
2713         er = emulate_instruction(vcpu,
2714                 EMULTYPE_VMWARE | EMULTYPE_NO_UD_ON_FAIL);
2715         if (er == EMULATE_USER_EXIT)
2716                 return 0;
2717         else if (er != EMULATE_DONE)
2718                 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2719         return 1;
2720 }
2721
2722 static bool is_erratum_383(void)
2723 {
2724         int err, i;
2725         u64 value;
2726
2727         if (!erratum_383_found)
2728                 return false;
2729
2730         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2731         if (err)
2732                 return false;
2733
2734         /* Bit 62 may or may not be set for this mce */
2735         value &= ~(1ULL << 62);
2736
2737         if (value != 0xb600000000010015ULL)
2738                 return false;
2739
2740         /* Clear MCi_STATUS registers */
2741         for (i = 0; i < 6; ++i)
2742                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2743
2744         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2745         if (!err) {
2746                 u32 low, high;
2747
2748                 value &= ~(1ULL << 2);
2749                 low    = lower_32_bits(value);
2750                 high   = upper_32_bits(value);
2751
2752                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2753         }
2754
2755         /* Flush tlb to evict multi-match entries */
2756         __flush_tlb_all();
2757
2758         return true;
2759 }
2760
2761 static void svm_handle_mce(struct vcpu_svm *svm)
2762 {
2763         if (is_erratum_383()) {
2764                 /*
2765                  * Erratum 383 triggered. Guest state is corrupt so kill the
2766                  * guest.
2767                  */
2768                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
2769
2770                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
2771
2772                 return;
2773         }
2774
2775         /*
2776          * On an #MC intercept the MCE handler is not called automatically in
2777          * the host. So do it by hand here.
2778          */
2779         asm volatile (
2780                 "int $0x12\n");
2781         /* not sure if we ever come back to this point */
2782
2783         return;
2784 }
2785
2786 static int mc_interception(struct vcpu_svm *svm)
2787 {
2788         return 1;
2789 }
2790
2791 static int shutdown_interception(struct vcpu_svm *svm)
2792 {
2793         struct kvm_run *kvm_run = svm->vcpu.run;
2794
2795         /*
2796          * VMCB is undefined after a SHUTDOWN intercept
2797          * so reinitialize it.
2798          */
2799         clear_page(svm->vmcb);
2800         init_vmcb(svm);
2801
2802         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2803         return 0;
2804 }
2805
2806 static int io_interception(struct vcpu_svm *svm)
2807 {
2808         struct kvm_vcpu *vcpu = &svm->vcpu;
2809         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2810         int size, in, string;
2811         unsigned port;
2812
2813         ++svm->vcpu.stat.io_exits;
2814         string = (io_info & SVM_IOIO_STR_MASK) != 0;
2815         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2816         if (string)
2817                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
2818
2819         port = io_info >> 16;
2820         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2821         svm->next_rip = svm->vmcb->control.exit_info_2;
2822
2823         return kvm_fast_pio(&svm->vcpu, size, port, in);
2824 }
2825
2826 static int nmi_interception(struct vcpu_svm *svm)
2827 {
2828         return 1;
2829 }
2830
2831 static int intr_interception(struct vcpu_svm *svm)
2832 {
2833         ++svm->vcpu.stat.irq_exits;
2834         return 1;
2835 }
2836
2837 static int nop_on_interception(struct vcpu_svm *svm)
2838 {
2839         return 1;
2840 }
2841
2842 static int halt_interception(struct vcpu_svm *svm)
2843 {
2844         svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
2845         return kvm_emulate_halt(&svm->vcpu);
2846 }
2847
2848 static int vmmcall_interception(struct vcpu_svm *svm)
2849 {
2850         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2851         return kvm_emulate_hypercall(&svm->vcpu);
2852 }
2853
2854 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
2855 {
2856         struct vcpu_svm *svm = to_svm(vcpu);
2857
2858         return svm->nested.nested_cr3;
2859 }
2860
2861 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
2862 {
2863         struct vcpu_svm *svm = to_svm(vcpu);
2864         u64 cr3 = svm->nested.nested_cr3;
2865         u64 pdpte;
2866         int ret;
2867
2868         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte,
2869                                        offset_in_page(cr3) + index * 8, 8);
2870         if (ret)
2871                 return 0;
2872         return pdpte;
2873 }
2874
2875 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
2876                                    unsigned long root)
2877 {
2878         struct vcpu_svm *svm = to_svm(vcpu);
2879
2880         svm->vmcb->control.nested_cr3 = __sme_set(root);
2881         mark_dirty(svm->vmcb, VMCB_NPT);
2882         svm_flush_tlb(vcpu, true);
2883 }
2884
2885 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
2886                                        struct x86_exception *fault)
2887 {
2888         struct vcpu_svm *svm = to_svm(vcpu);
2889
2890         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
2891                 /*
2892                  * TODO: track the cause of the nested page fault, and
2893                  * correctly fill in the high bits of exit_info_1.
2894                  */
2895                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
2896                 svm->vmcb->control.exit_code_hi = 0;
2897                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
2898                 svm->vmcb->control.exit_info_2 = fault->address;
2899         }
2900
2901         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
2902         svm->vmcb->control.exit_info_1 |= fault->error_code;
2903
2904         /*
2905          * The present bit is always zero for page structure faults on real
2906          * hardware.
2907          */
2908         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
2909                 svm->vmcb->control.exit_info_1 &= ~1;
2910
2911         nested_svm_vmexit(svm);
2912 }
2913
2914 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
2915 {
2916         WARN_ON(mmu_is_nested(vcpu));
2917         kvm_init_shadow_mmu(vcpu);
2918         vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
2919         vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
2920         vcpu->arch.mmu.get_pdptr         = nested_svm_get_tdp_pdptr;
2921         vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
2922         vcpu->arch.mmu.shadow_root_level = get_npt_level(vcpu);
2923         reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
2924         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
2925 }
2926
2927 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
2928 {
2929         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
2930 }
2931
2932 static int nested_svm_check_permissions(struct vcpu_svm *svm)
2933 {
2934         if (!(svm->vcpu.arch.efer & EFER_SVME) ||
2935             !is_paging(&svm->vcpu)) {
2936                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2937                 return 1;
2938         }
2939
2940         if (svm->vmcb->save.cpl) {
2941                 kvm_inject_gp(&svm->vcpu, 0);
2942                 return 1;
2943         }
2944
2945         return 0;
2946 }
2947
2948 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
2949                                       bool has_error_code, u32 error_code)
2950 {
2951         int vmexit;
2952
2953         if (!is_guest_mode(&svm->vcpu))
2954                 return 0;
2955
2956         vmexit = nested_svm_intercept(svm);
2957         if (vmexit != NESTED_EXIT_DONE)
2958                 return 0;
2959
2960         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
2961         svm->vmcb->control.exit_code_hi = 0;
2962         svm->vmcb->control.exit_info_1 = error_code;
2963
2964         /*
2965          * FIXME: we should not write CR2 when L1 intercepts an L2 #PF exception.
2966          * The fix is to add the ancillary datum (CR2 or DR6) to structs
2967          * kvm_queued_exception and kvm_vcpu_events, so that CR2 and DR6 can be
2968          * written only when inject_pending_event runs (DR6 would written here
2969          * too).  This should be conditional on a new capability---if the
2970          * capability is disabled, kvm_multiple_exception would write the
2971          * ancillary information to CR2 or DR6, for backwards ABI-compatibility.
2972          */
2973         if (svm->vcpu.arch.exception.nested_apf)
2974                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
2975         else
2976                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
2977
2978         svm->nested.exit_required = true;
2979         return vmexit;
2980 }
2981
2982 /* This function returns true if it is save to enable the irq window */
2983 static inline bool nested_svm_intr(struct vcpu_svm *svm)
2984 {
2985         if (!is_guest_mode(&svm->vcpu))
2986                 return true;
2987
2988         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2989                 return true;
2990
2991         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
2992                 return false;
2993
2994         /*
2995          * if vmexit was already requested (by intercepted exception
2996          * for instance) do not overwrite it with "external interrupt"
2997          * vmexit.
2998          */
2999         if (svm->nested.exit_required)
3000                 return false;
3001
3002         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
3003         svm->vmcb->control.exit_info_1 = 0;
3004         svm->vmcb->control.exit_info_2 = 0;
3005
3006         if (svm->nested.intercept & 1ULL) {
3007                 /*
3008                  * The #vmexit can't be emulated here directly because this
3009                  * code path runs with irqs and preemption disabled. A
3010                  * #vmexit emulation might sleep. Only signal request for
3011                  * the #vmexit here.
3012                  */
3013                 svm->nested.exit_required = true;
3014                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
3015                 return false;
3016         }
3017
3018         return true;
3019 }
3020
3021 /* This function returns true if it is save to enable the nmi window */
3022 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
3023 {
3024         if (!is_guest_mode(&svm->vcpu))
3025                 return true;
3026
3027         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
3028                 return true;
3029
3030         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
3031         svm->nested.exit_required = true;
3032
3033         return false;
3034 }
3035
3036 static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
3037 {
3038         struct page *page;
3039
3040         might_sleep();
3041
3042         page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
3043         if (is_error_page(page))
3044                 goto error;
3045
3046         *_page = page;
3047
3048         return kmap(page);
3049
3050 error:
3051         kvm_inject_gp(&svm->vcpu, 0);
3052
3053         return NULL;
3054 }
3055
3056 static void nested_svm_unmap(struct page *page)
3057 {
3058         kunmap(page);
3059         kvm_release_page_dirty(page);
3060 }
3061
3062 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
3063 {
3064         unsigned port, size, iopm_len;
3065         u16 val, mask;
3066         u8 start_bit;
3067         u64 gpa;
3068
3069         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
3070                 return NESTED_EXIT_HOST;
3071
3072         port = svm->vmcb->control.exit_info_1 >> 16;
3073         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
3074                 SVM_IOIO_SIZE_SHIFT;
3075         gpa  = svm->nested.vmcb_iopm + (port / 8);
3076         start_bit = port % 8;
3077         iopm_len = (start_bit + size > 8) ? 2 : 1;
3078         mask = (0xf >> (4 - size)) << start_bit;
3079         val = 0;
3080
3081         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
3082                 return NESTED_EXIT_DONE;
3083
3084         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
3085 }
3086
3087 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
3088 {
3089         u32 offset, msr, value;
3090         int write, mask;
3091
3092         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3093                 return NESTED_EXIT_HOST;
3094
3095         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
3096         offset = svm_msrpm_offset(msr);
3097         write  = svm->vmcb->control.exit_info_1 & 1;
3098         mask   = 1 << ((2 * (msr & 0xf)) + write);
3099
3100         if (offset == MSR_INVALID)
3101                 return NESTED_EXIT_DONE;
3102
3103         /* Offset is in 32 bit units but need in 8 bit units */
3104         offset *= 4;
3105
3106         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
3107                 return NESTED_EXIT_DONE;
3108
3109         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
3110 }
3111
3112 /* DB exceptions for our internal use must not cause vmexit */
3113 static int nested_svm_intercept_db(struct vcpu_svm *svm)
3114 {
3115         unsigned long dr6;
3116
3117         /* if we're not singlestepping, it's not ours */
3118         if (!svm->nmi_singlestep)
3119                 return NESTED_EXIT_DONE;
3120
3121         /* if it's not a singlestep exception, it's not ours */
3122         if (kvm_get_dr(&svm->vcpu, 6, &dr6))
3123                 return NESTED_EXIT_DONE;
3124         if (!(dr6 & DR6_BS))
3125                 return NESTED_EXIT_DONE;
3126
3127         /* if the guest is singlestepping, it should get the vmexit */
3128         if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) {
3129                 disable_nmi_singlestep(svm);
3130                 return NESTED_EXIT_DONE;
3131         }
3132
3133         /* it's ours, the nested hypervisor must not see this one */
3134         return NESTED_EXIT_HOST;
3135 }
3136
3137 static int nested_svm_exit_special(struct vcpu_svm *svm)
3138 {
3139         u32 exit_code = svm->vmcb->control.exit_code;
3140
3141         switch (exit_code) {
3142         case SVM_EXIT_INTR:
3143         case SVM_EXIT_NMI:
3144         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
3145                 return NESTED_EXIT_HOST;
3146         case SVM_EXIT_NPF:
3147                 /* For now we are always handling NPFs when using them */
3148                 if (npt_enabled)
3149                         return NESTED_EXIT_HOST;
3150                 break;
3151         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
3152                 /* When we're shadowing, trap PFs, but not async PF */
3153                 if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0)
3154                         return NESTED_EXIT_HOST;
3155                 break;
3156         default:
3157                 break;
3158         }
3159
3160         return NESTED_EXIT_CONTINUE;
3161 }
3162
3163 /*
3164  * If this function returns true, this #vmexit was already handled
3165  */
3166 static int nested_svm_intercept(struct vcpu_svm *svm)
3167 {
3168         u32 exit_code = svm->vmcb->control.exit_code;
3169         int vmexit = NESTED_EXIT_HOST;
3170
3171         switch (exit_code) {
3172         case SVM_EXIT_MSR:
3173                 vmexit = nested_svm_exit_handled_msr(svm);
3174                 break;
3175         case SVM_EXIT_IOIO:
3176                 vmexit = nested_svm_intercept_ioio(svm);
3177                 break;
3178         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
3179                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
3180                 if (svm->nested.intercept_cr & bit)
3181                         vmexit = NESTED_EXIT_DONE;
3182                 break;
3183         }
3184         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
3185                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
3186                 if (svm->nested.intercept_dr & bit)
3187                         vmexit = NESTED_EXIT_DONE;
3188                 break;
3189         }
3190         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
3191                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
3192                 if (svm->nested.intercept_exceptions & excp_bits) {
3193                         if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR)
3194                                 vmexit = nested_svm_intercept_db(svm);
3195                         else
3196                                 vmexit = NESTED_EXIT_DONE;
3197                 }
3198                 /* async page fault always cause vmexit */
3199                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
3200                          svm->vcpu.arch.exception.nested_apf != 0)
3201                         vmexit = NESTED_EXIT_DONE;
3202                 break;
3203         }
3204         case SVM_EXIT_ERR: {
3205                 vmexit = NESTED_EXIT_DONE;
3206                 break;
3207         }
3208         default: {
3209                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
3210                 if (svm->nested.intercept & exit_bits)
3211                         vmexit = NESTED_EXIT_DONE;
3212         }
3213         }
3214
3215         return vmexit;
3216 }
3217
3218 static int nested_svm_exit_handled(struct vcpu_svm *svm)
3219 {
3220         int vmexit;
3221
3222         vmexit = nested_svm_intercept(svm);
3223
3224         if (vmexit == NESTED_EXIT_DONE)
3225                 nested_svm_vmexit(svm);
3226
3227         return vmexit;
3228 }
3229
3230 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
3231 {
3232         struct vmcb_control_area *dst  = &dst_vmcb->control;
3233         struct vmcb_control_area *from = &from_vmcb->control;
3234
3235         dst->intercept_cr         = from->intercept_cr;
3236         dst->intercept_dr         = from->intercept_dr;
3237         dst->intercept_exceptions = from->intercept_exceptions;
3238         dst->intercept            = from->intercept;
3239         dst->iopm_base_pa         = from->iopm_base_pa;
3240         dst->msrpm_base_pa        = from->msrpm_base_pa;
3241         dst->tsc_offset           = from->tsc_offset;
3242         dst->asid                 = from->asid;
3243         dst->tlb_ctl              = from->tlb_ctl;
3244         dst->int_ctl              = from->int_ctl;
3245         dst->int_vector           = from->int_vector;
3246         dst->int_state            = from->int_state;
3247         dst->exit_code            = from->exit_code;
3248         dst->exit_code_hi         = from->exit_code_hi;
3249         dst->exit_info_1          = from->exit_info_1;
3250         dst->exit_info_2          = from->exit_info_2;
3251         dst->exit_int_info        = from->exit_int_info;
3252         dst->exit_int_info_err    = from->exit_int_info_err;
3253         dst->nested_ctl           = from->nested_ctl;
3254         dst->event_inj            = from->event_inj;
3255         dst->event_inj_err        = from->event_inj_err;
3256         dst->nested_cr3           = from->nested_cr3;
3257         dst->virt_ext              = from->virt_ext;
3258 }
3259
3260 static int nested_svm_vmexit(struct vcpu_svm *svm)
3261 {
3262         struct vmcb *nested_vmcb;
3263         struct vmcb *hsave = svm->nested.hsave;
3264         struct vmcb *vmcb = svm->vmcb;
3265         struct page *page;
3266
3267         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
3268                                        vmcb->control.exit_info_1,
3269                                        vmcb->control.exit_info_2,
3270                                        vmcb->control.exit_int_info,
3271                                        vmcb->control.exit_int_info_err,
3272                                        KVM_ISA_SVM);
3273
3274         nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
3275         if (!nested_vmcb)
3276                 return 1;
3277
3278         /* Exit Guest-Mode */
3279         leave_guest_mode(&svm->vcpu);
3280         svm->nested.vmcb = 0;
3281
3282         /* Give the current vmcb to the guest */
3283         disable_gif(svm);
3284
3285         nested_vmcb->save.es     = vmcb->save.es;
3286         nested_vmcb->save.cs     = vmcb->save.cs;
3287         nested_vmcb->save.ss     = vmcb->save.ss;
3288         nested_vmcb->save.ds     = vmcb->save.ds;
3289         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
3290         nested_vmcb->save.idtr   = vmcb->save.idtr;
3291         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
3292         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
3293         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
3294         nested_vmcb->save.cr2    = vmcb->save.cr2;
3295         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
3296         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
3297         nested_vmcb->save.rip    = vmcb->save.rip;
3298         nested_vmcb->save.rsp    = vmcb->save.rsp;
3299         nested_vmcb->save.rax    = vmcb->save.rax;
3300         nested_vmcb->save.dr7    = vmcb->save.dr7;
3301         nested_vmcb->save.dr6    = vmcb->save.dr6;
3302         nested_vmcb->save.cpl    = vmcb->save.cpl;
3303
3304         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
3305         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
3306         nested_vmcb->control.int_state         = vmcb->control.int_state;
3307         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
3308         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
3309         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
3310         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
3311         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
3312         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
3313
3314         if (svm->nrips_enabled)
3315                 nested_vmcb->control.next_rip  = vmcb->control.next_rip;
3316
3317         /*
3318          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
3319          * to make sure that we do not lose injected events. So check event_inj
3320          * here and copy it to exit_int_info if it is valid.
3321          * Exit_int_info and event_inj can't be both valid because the case
3322          * below only happens on a VMRUN instruction intercept which has
3323          * no valid exit_int_info set.
3324          */
3325         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
3326                 struct vmcb_control_area *nc = &nested_vmcb->control;
3327
3328                 nc->exit_int_info     = vmcb->control.event_inj;
3329                 nc->exit_int_info_err = vmcb->control.event_inj_err;
3330         }
3331
3332         nested_vmcb->control.tlb_ctl           = 0;
3333         nested_vmcb->control.event_inj         = 0;
3334         nested_vmcb->control.event_inj_err     = 0;
3335
3336         /* We always set V_INTR_MASKING and remember the old value in hflags */
3337         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
3338                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3339
3340         /* Restore the original control entries */
3341         copy_vmcb_control_area(vmcb, hsave);
3342
3343         svm->vcpu.arch.tsc_offset = svm->vmcb->control.tsc_offset;
3344         kvm_clear_exception_queue(&svm->vcpu);
3345         kvm_clear_interrupt_queue(&svm->vcpu);
3346
3347         svm->nested.nested_cr3 = 0;
3348
3349         /* Restore selected save entries */
3350         svm->vmcb->save.es = hsave->save.es;
3351         svm->vmcb->save.cs = hsave->save.cs;
3352         svm->vmcb->save.ss = hsave->save.ss;
3353         svm->vmcb->save.ds = hsave->save.ds;
3354         svm->vmcb->save.gdtr = hsave->save.gdtr;
3355         svm->vmcb->save.idtr = hsave->save.idtr;
3356         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
3357         svm_set_efer(&svm->vcpu, hsave->save.efer);
3358         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
3359         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
3360         if (npt_enabled) {
3361                 svm->vmcb->save.cr3 = hsave->save.cr3;
3362                 svm->vcpu.arch.cr3 = hsave->save.cr3;
3363         } else {
3364                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
3365         }
3366         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
3367         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
3368         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
3369         svm->vmcb->save.dr7 = 0;
3370         svm->vmcb->save.cpl = 0;
3371         svm->vmcb->control.exit_int_info = 0;
3372
3373         mark_all_dirty(svm->vmcb);
3374
3375         nested_svm_unmap(page);
3376
3377         nested_svm_uninit_mmu_context(&svm->vcpu);
3378         kvm_mmu_reset_context(&svm->vcpu);
3379         kvm_mmu_load(&svm->vcpu);
3380
3381         return 0;
3382 }
3383
3384 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
3385 {
3386         /*
3387          * This function merges the msr permission bitmaps of kvm and the
3388          * nested vmcb. It is optimized in that it only merges the parts where
3389          * the kvm msr permission bitmap may contain zero bits
3390          */
3391         int i;
3392
3393         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3394                 return true;
3395
3396         for (i = 0; i < MSRPM_OFFSETS; i++) {
3397                 u32 value, p;
3398                 u64 offset;
3399
3400                 if (msrpm_offsets[i] == 0xffffffff)
3401                         break;
3402
3403                 p      = msrpm_offsets[i];
3404                 offset = svm->nested.vmcb_msrpm + (p * 4);
3405
3406                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
3407                         return false;
3408
3409                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
3410         }
3411
3412         svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
3413
3414         return true;
3415 }
3416
3417 static bool nested_vmcb_checks(struct vmcb *vmcb)
3418 {
3419         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
3420                 return false;
3421
3422         if (vmcb->control.asid == 0)
3423                 return false;
3424
3425         if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) &&
3426             !npt_enabled)
3427                 return false;
3428
3429         return true;
3430 }
3431
3432 static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa,
3433                                  struct vmcb *nested_vmcb, struct page *page)
3434 {
3435         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
3436                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
3437         else
3438                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
3439
3440         if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) {
3441                 kvm_mmu_unload(&svm->vcpu);
3442                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
3443                 nested_svm_init_mmu_context(&svm->vcpu);
3444         }
3445
3446         /* Load the nested guest state */
3447         svm->vmcb->save.es = nested_vmcb->save.es;
3448         svm->vmcb->save.cs = nested_vmcb->save.cs;
3449         svm->vmcb->save.ss = nested_vmcb->save.ss;
3450         svm->vmcb->save.ds = nested_vmcb->save.ds;
3451         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
3452         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
3453         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
3454         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
3455         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
3456         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
3457         if (npt_enabled) {
3458                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
3459                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
3460         } else
3461                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
3462
3463         /* Guest paging mode is active - reset mmu */
3464         kvm_mmu_reset_context(&svm->vcpu);
3465
3466         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
3467         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
3468         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
3469         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
3470
3471         /* In case we don't even reach vcpu_run, the fields are not updated */
3472         svm->vmcb->save.rax = nested_vmcb->save.rax;
3473         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
3474         svm->vmcb->save.rip = nested_vmcb->save.rip;
3475         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
3476         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
3477         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
3478
3479         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
3480         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
3481
3482         /* cache intercepts */
3483         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
3484         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
3485         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
3486         svm->nested.intercept            = nested_vmcb->control.intercept;
3487
3488         svm_flush_tlb(&svm->vcpu, true);
3489         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
3490         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
3491                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
3492         else
3493                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
3494
3495         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
3496                 /* We only want the cr8 intercept bits of the guest */
3497                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
3498                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3499         }
3500
3501         /* We don't want to see VMMCALLs from a nested guest */
3502         clr_intercept(svm, INTERCEPT_VMMCALL);
3503
3504         svm->vcpu.arch.tsc_offset += nested_vmcb->control.tsc_offset;
3505         svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset;
3506
3507         svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext;
3508         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
3509         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
3510         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
3511         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
3512
3513         nested_svm_unmap(page);
3514
3515         /* Enter Guest-Mode */
3516         enter_guest_mode(&svm->vcpu);
3517
3518         /*
3519          * Merge guest and host intercepts - must be called  with vcpu in
3520          * guest-mode to take affect here
3521          */
3522         recalc_intercepts(svm);
3523
3524         svm->nested.vmcb = vmcb_gpa;
3525
3526         enable_gif(svm);
3527
3528         mark_all_dirty(svm->vmcb);
3529 }
3530
3531 static bool nested_svm_vmrun(struct vcpu_svm *svm)
3532 {
3533         struct vmcb *nested_vmcb;
3534         struct vmcb *hsave = svm->nested.hsave;
3535         struct vmcb *vmcb = svm->vmcb;
3536         struct page *page;
3537         u64 vmcb_gpa;
3538
3539         vmcb_gpa = svm->vmcb->save.rax;
3540
3541         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3542         if (!nested_vmcb)
3543                 return false;
3544
3545         if (!nested_vmcb_checks(nested_vmcb)) {
3546                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
3547                 nested_vmcb->control.exit_code_hi = 0;
3548                 nested_vmcb->control.exit_info_1  = 0;
3549                 nested_vmcb->control.exit_info_2  = 0;
3550
3551                 nested_svm_unmap(page);
3552
3553                 return false;
3554         }
3555
3556         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
3557                                nested_vmcb->save.rip,
3558                                nested_vmcb->control.int_ctl,
3559                                nested_vmcb->control.event_inj,
3560                                nested_vmcb->control.nested_ctl);
3561
3562         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
3563                                     nested_vmcb->control.intercept_cr >> 16,
3564                                     nested_vmcb->control.intercept_exceptions,
3565                                     nested_vmcb->control.intercept);
3566
3567         /* Clear internal status */
3568         kvm_clear_exception_queue(&svm->vcpu);
3569         kvm_clear_interrupt_queue(&svm->vcpu);
3570
3571         /*
3572          * Save the old vmcb, so we don't need to pick what we save, but can
3573          * restore everything when a VMEXIT occurs
3574          */
3575         hsave->save.es     = vmcb->save.es;
3576         hsave->save.cs     = vmcb->save.cs;
3577         hsave->save.ss     = vmcb->save.ss;
3578         hsave->save.ds     = vmcb->save.ds;
3579         hsave->save.gdtr   = vmcb->save.gdtr;
3580         hsave->save.idtr   = vmcb->save.idtr;
3581         hsave->save.efer   = svm->vcpu.arch.efer;
3582         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
3583         hsave->save.cr4    = svm->vcpu.arch.cr4;
3584         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
3585         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
3586         hsave->save.rsp    = vmcb->save.rsp;
3587         hsave->save.rax    = vmcb->save.rax;
3588         if (npt_enabled)
3589                 hsave->save.cr3    = vmcb->save.cr3;
3590         else
3591                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
3592
3593         copy_vmcb_control_area(hsave, vmcb);
3594
3595         enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, page);
3596
3597         return true;
3598 }
3599
3600 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
3601 {
3602         to_vmcb->save.fs = from_vmcb->save.fs;
3603         to_vmcb->save.gs = from_vmcb->save.gs;
3604         to_vmcb->save.tr = from_vmcb->save.tr;
3605         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
3606         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
3607         to_vmcb->save.star = from_vmcb->save.star;
3608         to_vmcb->save.lstar = from_vmcb->save.lstar;
3609         to_vmcb->save.cstar = from_vmcb->save.cstar;
3610         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
3611         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
3612         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
3613         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
3614 }
3615
3616 static int vmload_interception(struct vcpu_svm *svm)
3617 {
3618         struct vmcb *nested_vmcb;
3619         struct page *page;
3620         int ret;
3621
3622         if (nested_svm_check_permissions(svm))
3623                 return 1;
3624
3625         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3626         if (!nested_vmcb)
3627                 return 1;
3628
3629         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3630         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3631
3632         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
3633         nested_svm_unmap(page);
3634
3635         return ret;
3636 }
3637
3638 static int vmsave_interception(struct vcpu_svm *svm)
3639 {
3640         struct vmcb *nested_vmcb;
3641         struct page *page;
3642         int ret;
3643
3644         if (nested_svm_check_permissions(svm))
3645                 return 1;
3646
3647         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3648         if (!nested_vmcb)
3649                 return 1;
3650
3651         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3652         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3653
3654         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
3655         nested_svm_unmap(page);
3656
3657         return ret;
3658 }
3659
3660 static int vmrun_interception(struct vcpu_svm *svm)
3661 {
3662         if (nested_svm_check_permissions(svm))
3663                 return 1;
3664
3665         /* Save rip after vmrun instruction */
3666         kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
3667
3668         if (!nested_svm_vmrun(svm))
3669                 return 1;
3670
3671         if (!nested_svm_vmrun_msrpm(svm))
3672                 goto failed;
3673
3674         return 1;
3675
3676 failed:
3677
3678         svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
3679         svm->vmcb->control.exit_code_hi = 0;
3680         svm->vmcb->control.exit_info_1  = 0;
3681         svm->vmcb->control.exit_info_2  = 0;
3682
3683         nested_svm_vmexit(svm);
3684
3685         return 1;
3686 }
3687
3688 static int stgi_interception(struct vcpu_svm *svm)
3689 {
3690         int ret;
3691
3692         if (nested_svm_check_permissions(svm))
3693                 return 1;
3694
3695         /*
3696          * If VGIF is enabled, the STGI intercept is only added to
3697          * detect the opening of the SMI/NMI window; remove it now.
3698          */
3699         if (vgif_enabled(svm))
3700                 clr_intercept(svm, INTERCEPT_STGI);
3701
3702         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3703         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3704         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3705
3706         enable_gif(svm);
3707
3708         return ret;
3709 }
3710
3711 static int clgi_interception(struct vcpu_svm *svm)
3712 {
3713         int ret;
3714
3715         if (nested_svm_check_permissions(svm))
3716                 return 1;
3717
3718         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3719         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3720
3721         disable_gif(svm);
3722
3723         /* After a CLGI no interrupts should come */
3724         if (!kvm_vcpu_apicv_active(&svm->vcpu)) {
3725                 svm_clear_vintr(svm);
3726                 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3727                 mark_dirty(svm->vmcb, VMCB_INTR);
3728         }
3729
3730         return ret;
3731 }
3732
3733 static int invlpga_interception(struct vcpu_svm *svm)
3734 {
3735         struct kvm_vcpu *vcpu = &svm->vcpu;
3736
3737         trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
3738                           kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3739
3740         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
3741         kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3742
3743         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3744         return kvm_skip_emulated_instruction(&svm->vcpu);
3745 }
3746
3747 static int skinit_interception(struct vcpu_svm *svm)
3748 {
3749         trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3750
3751         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3752         return 1;
3753 }
3754
3755 static int wbinvd_interception(struct vcpu_svm *svm)
3756 {
3757         return kvm_emulate_wbinvd(&svm->vcpu);
3758 }
3759
3760 static int xsetbv_interception(struct vcpu_svm *svm)
3761 {
3762         u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
3763         u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3764
3765         if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
3766                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3767                 return kvm_skip_emulated_instruction(&svm->vcpu);
3768         }
3769
3770         return 1;
3771 }
3772
3773 static int task_switch_interception(struct vcpu_svm *svm)
3774 {
3775         u16 tss_selector;
3776         int reason;
3777         int int_type = svm->vmcb->control.exit_int_info &
3778                 SVM_EXITINTINFO_TYPE_MASK;
3779         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
3780         uint32_t type =
3781                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
3782         uint32_t idt_v =
3783                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
3784         bool has_error_code = false;
3785         u32 error_code = 0;
3786
3787         tss_selector = (u16)svm->vmcb->control.exit_info_1;
3788
3789         if (svm->vmcb->control.exit_info_2 &
3790             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
3791                 reason = TASK_SWITCH_IRET;
3792         else if (svm->vmcb->control.exit_info_2 &
3793                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
3794                 reason = TASK_SWITCH_JMP;
3795         else if (idt_v)
3796                 reason = TASK_SWITCH_GATE;
3797         else
3798                 reason = TASK_SWITCH_CALL;
3799
3800         if (reason == TASK_SWITCH_GATE) {
3801                 switch (type) {
3802                 case SVM_EXITINTINFO_TYPE_NMI:
3803                         svm->vcpu.arch.nmi_injected = false;
3804                         break;
3805                 case SVM_EXITINTINFO_TYPE_EXEPT:
3806                         if (svm->vmcb->control.exit_info_2 &
3807                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
3808                                 has_error_code = true;
3809                                 error_code =
3810                                         (u32)svm->vmcb->control.exit_info_2;
3811                         }
3812                         kvm_clear_exception_queue(&svm->vcpu);
3813                         break;
3814                 case SVM_EXITINTINFO_TYPE_INTR:
3815                         kvm_clear_interrupt_queue(&svm->vcpu);
3816                         break;
3817                 default:
3818                         break;
3819                 }
3820         }
3821
3822         if (reason != TASK_SWITCH_GATE ||
3823             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
3824             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
3825              (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
3826                 skip_emulated_instruction(&svm->vcpu);
3827
3828         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
3829                 int_vec = -1;
3830
3831         if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
3832                                 has_error_code, error_code) == EMULATE_FAIL) {
3833                 svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3834                 svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
3835                 svm->vcpu.run->internal.ndata = 0;
3836                 return 0;
3837         }
3838         return 1;
3839 }
3840
3841 static int cpuid_interception(struct vcpu_svm *svm)
3842 {
3843         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3844         return kvm_emulate_cpuid(&svm->vcpu);
3845 }
3846
3847 static int iret_interception(struct vcpu_svm *svm)
3848 {
3849         ++svm->vcpu.stat.nmi_window_exits;
3850         clr_intercept(svm, INTERCEPT_IRET);
3851         svm->vcpu.arch.hflags |= HF_IRET_MASK;
3852         svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
3853         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3854         return 1;
3855 }
3856
3857 static int invlpg_interception(struct vcpu_svm *svm)
3858 {
3859         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3860                 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3861
3862         kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
3863         return kvm_skip_emulated_instruction(&svm->vcpu);
3864 }
3865
3866 static int emulate_on_interception(struct vcpu_svm *svm)
3867 {
3868         return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3869 }
3870
3871 static int rsm_interception(struct vcpu_svm *svm)
3872 {
3873         return x86_emulate_instruction(&svm->vcpu, 0, 0,
3874                                        rsm_ins_bytes, 2) == EMULATE_DONE;
3875 }
3876
3877 static int rdpmc_interception(struct vcpu_svm *svm)
3878 {
3879         int err;
3880
3881         if (!static_cpu_has(X86_FEATURE_NRIPS))
3882                 return emulate_on_interception(svm);
3883
3884         err = kvm_rdpmc(&svm->vcpu);
3885         return kvm_complete_insn_gp(&svm->vcpu, err);
3886 }
3887
3888 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
3889                                             unsigned long val)
3890 {
3891         unsigned long cr0 = svm->vcpu.arch.cr0;
3892         bool ret = false;
3893         u64 intercept;
3894
3895         intercept = svm->nested.intercept;
3896
3897         if (!is_guest_mode(&svm->vcpu) ||
3898             (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
3899                 return false;
3900
3901         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
3902         val &= ~SVM_CR0_SELECTIVE_MASK;
3903
3904         if (cr0 ^ val) {
3905                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
3906                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
3907         }
3908
3909         return ret;
3910 }
3911
3912 #define CR_VALID (1ULL << 63)
3913
3914 static int cr_interception(struct vcpu_svm *svm)
3915 {
3916         int reg, cr;
3917         unsigned long val;
3918         int err;
3919
3920         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3921                 return emulate_on_interception(svm);
3922
3923         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
3924                 return emulate_on_interception(svm);
3925
3926         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3927         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
3928                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
3929         else
3930                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
3931
3932         err = 0;
3933         if (cr >= 16) { /* mov to cr */
3934                 cr -= 16;
3935                 val = kvm_register_read(&svm->vcpu, reg);
3936                 switch (cr) {
3937                 case 0:
3938                         if (!check_selective_cr0_intercepted(svm, val))
3939                                 err = kvm_set_cr0(&svm->vcpu, val);
3940                         else
3941                                 return 1;
3942
3943                         break;
3944                 case 3:
3945                         err = kvm_set_cr3(&svm->vcpu, val);
3946                         break;
3947                 case 4:
3948                         err = kvm_set_cr4(&svm->vcpu, val);
3949                         break;
3950                 case 8:
3951                         err = kvm_set_cr8(&svm->vcpu, val);
3952                         break;
3953                 default:
3954                         WARN(1, "unhandled write to CR%d", cr);
3955                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3956                         return 1;
3957                 }
3958         } else { /* mov from cr */
3959                 switch (cr) {
3960                 case 0:
3961                         val = kvm_read_cr0(&svm->vcpu);
3962                         break;
3963                 case 2:
3964                         val = svm->vcpu.arch.cr2;
3965                         break;
3966                 case 3:
3967                         val = kvm_read_cr3(&svm->vcpu);
3968                         break;
3969                 case 4:
3970                         val = kvm_read_cr4(&svm->vcpu);
3971                         break;
3972                 case 8:
3973                         val = kvm_get_cr8(&svm->vcpu);
3974                         break;
3975                 default:
3976                         WARN(1, "unhandled read from CR%d", cr);
3977                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3978                         return 1;
3979                 }
3980                 kvm_register_write(&svm->vcpu, reg, val);
3981         }
3982         return kvm_complete_insn_gp(&svm->vcpu, err);
3983 }
3984
3985 static int dr_interception(struct vcpu_svm *svm)
3986 {
3987         int reg, dr;
3988         unsigned long val;
3989
3990         if (svm->vcpu.guest_debug == 0) {
3991                 /*
3992                  * No more DR vmexits; force a reload of the debug registers
3993                  * and reenter on this instruction.  The next vmexit will
3994                  * retrieve the full state of the debug registers.
3995                  */
3996                 clr_dr_intercepts(svm);
3997                 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
3998                 return 1;
3999         }
4000
4001         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
4002                 return emulate_on_interception(svm);
4003
4004         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
4005         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
4006
4007         if (dr >= 16) { /* mov to DRn */
4008                 if (!kvm_require_dr(&svm->vcpu, dr - 16))
4009                         return 1;
4010                 val = kvm_register_read(&svm->vcpu, reg);
4011                 kvm_set_dr(&svm->vcpu, dr - 16, val);
4012         } else {
4013                 if (!kvm_require_dr(&svm->vcpu, dr))
4014                         return 1;
4015                 kvm_get_dr(&svm->vcpu, dr, &val);
4016                 kvm_register_write(&svm->vcpu, reg, val);
4017         }
4018
4019         return kvm_skip_emulated_instruction(&svm->vcpu);
4020 }
4021
4022 static int cr8_write_interception(struct vcpu_svm *svm)
4023 {
4024         struct kvm_run *kvm_run = svm->vcpu.run;
4025         int r;
4026
4027         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
4028         /* instruction emulation calls kvm_set_cr8() */
4029         r = cr_interception(svm);
4030         if (lapic_in_kernel(&svm->vcpu))
4031                 return r;
4032         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
4033                 return r;
4034         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
4035         return 0;
4036 }
4037
4038 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
4039 {
4040         msr->data = 0;
4041
4042         switch (msr->index) {
4043         case MSR_F10H_DECFG:
4044                 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
4045                         msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
4046                 break;
4047         default:
4048                 return 1;
4049         }
4050
4051         return 0;
4052 }
4053
4054 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
4055 {
4056         struct vcpu_svm *svm = to_svm(vcpu);
4057
4058         switch (msr_info->index) {
4059         case MSR_STAR:
4060                 msr_info->data = svm->vmcb->save.star;
4061                 break;
4062 #ifdef CONFIG_X86_64
4063         case MSR_LSTAR:
4064                 msr_info->data = svm->vmcb->save.lstar;
4065                 break;
4066         case MSR_CSTAR:
4067                 msr_info->data = svm->vmcb->save.cstar;
4068                 break;
4069         case MSR_KERNEL_GS_BASE:
4070                 msr_info->data = svm->vmcb->save.kernel_gs_base;
4071                 break;
4072         case MSR_SYSCALL_MASK:
4073                 msr_info->data = svm->vmcb->save.sfmask;
4074                 break;
4075 #endif
4076         case MSR_IA32_SYSENTER_CS:
4077                 msr_info->data = svm->vmcb->save.sysenter_cs;
4078                 break;
4079         case MSR_IA32_SYSENTER_EIP:
4080                 msr_info->data = svm->sysenter_eip;
4081                 break;
4082         case MSR_IA32_SYSENTER_ESP:
4083                 msr_info->data = svm->sysenter_esp;
4084                 break;
4085         case MSR_TSC_AUX:
4086                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4087                         return 1;
4088                 msr_info->data = svm->tsc_aux;
4089                 break;
4090         /*
4091          * Nobody will change the following 5 values in the VMCB so we can
4092          * safely return them on rdmsr. They will always be 0 until LBRV is
4093          * implemented.
4094          */
4095         case MSR_IA32_DEBUGCTLMSR:
4096                 msr_info->data = svm->vmcb->save.dbgctl;
4097                 break;
4098         case MSR_IA32_LASTBRANCHFROMIP:
4099                 msr_info->data = svm->vmcb->save.br_from;
4100                 break;
4101         case MSR_IA32_LASTBRANCHTOIP:
4102                 msr_info->data = svm->vmcb->save.br_to;
4103                 break;
4104         case MSR_IA32_LASTINTFROMIP:
4105                 msr_info->data = svm->vmcb->save.last_excp_from;
4106                 break;
4107         case MSR_IA32_LASTINTTOIP:
4108                 msr_info->data = svm->vmcb->save.last_excp_to;
4109                 break;
4110         case MSR_VM_HSAVE_PA:
4111                 msr_info->data = svm->nested.hsave_msr;
4112                 break;
4113         case MSR_VM_CR:
4114                 msr_info->data = svm->nested.vm_cr_msr;
4115                 break;
4116         case MSR_IA32_SPEC_CTRL:
4117                 if (!msr_info->host_initiated &&
4118                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
4119                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
4120                         return 1;
4121
4122                 msr_info->data = svm->spec_ctrl;
4123                 break;
4124         case MSR_AMD64_VIRT_SPEC_CTRL:
4125                 if (!msr_info->host_initiated &&
4126                     !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
4127                         return 1;
4128
4129                 msr_info->data = svm->virt_spec_ctrl;
4130                 break;
4131         case MSR_F15H_IC_CFG: {
4132
4133                 int family, model;
4134
4135                 family = guest_cpuid_family(vcpu);
4136                 model  = guest_cpuid_model(vcpu);
4137
4138                 if (family < 0 || model < 0)
4139                         return kvm_get_msr_common(vcpu, msr_info);
4140
4141                 msr_info->data = 0;
4142
4143                 if (family == 0x15 &&
4144                     (model >= 0x2 && model < 0x20))
4145                         msr_info->data = 0x1E;
4146                 }
4147                 break;
4148         case MSR_F10H_DECFG:
4149                 msr_info->data = svm->msr_decfg;
4150                 break;
4151         default:
4152                 return kvm_get_msr_common(vcpu, msr_info);
4153         }
4154         return 0;
4155 }
4156
4157 static int rdmsr_interception(struct vcpu_svm *svm)
4158 {
4159         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
4160         struct msr_data msr_info;
4161
4162         msr_info.index = ecx;
4163         msr_info.host_initiated = false;
4164         if (svm_get_msr(&svm->vcpu, &msr_info)) {
4165                 trace_kvm_msr_read_ex(ecx);
4166                 kvm_inject_gp(&svm->vcpu, 0);
4167                 return 1;
4168         } else {
4169                 trace_kvm_msr_read(ecx, msr_info.data);
4170
4171                 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX,
4172                                    msr_info.data & 0xffffffff);
4173                 kvm_register_write(&svm->vcpu, VCPU_REGS_RDX,
4174                                    msr_info.data >> 32);
4175                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
4176                 return kvm_skip_emulated_instruction(&svm->vcpu);
4177         }
4178 }
4179
4180 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
4181 {
4182         struct vcpu_svm *svm = to_svm(vcpu);
4183         int svm_dis, chg_mask;
4184
4185         if (data & ~SVM_VM_CR_VALID_MASK)
4186                 return 1;
4187
4188         chg_mask = SVM_VM_CR_VALID_MASK;
4189
4190         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
4191                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
4192
4193         svm->nested.vm_cr_msr &= ~chg_mask;
4194         svm->nested.vm_cr_msr |= (data & chg_mask);
4195
4196         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
4197
4198         /* check for svm_disable while efer.svme is set */
4199         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
4200                 return 1;
4201
4202         return 0;
4203 }
4204
4205 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
4206 {
4207         struct vcpu_svm *svm = to_svm(vcpu);
4208
4209         u32 ecx = msr->index;
4210         u64 data = msr->data;
4211         switch (ecx) {
4212         case MSR_IA32_CR_PAT:
4213                 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
4214                         return 1;
4215                 vcpu->arch.pat = data;
4216                 svm->vmcb->save.g_pat = data;
4217                 mark_dirty(svm->vmcb, VMCB_NPT);
4218                 break;
4219         case MSR_IA32_SPEC_CTRL:
4220                 if (!msr->host_initiated &&
4221                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
4222                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
4223                         return 1;
4224
4225                 /* The STIBP bit doesn't fault even if it's not advertised */
4226                 if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD))
4227                         return 1;
4228
4229                 svm->spec_ctrl = data;
4230
4231                 if (!data)
4232                         break;
4233
4234                 /*
4235                  * For non-nested:
4236                  * When it's written (to non-zero) for the first time, pass
4237                  * it through.
4238                  *
4239                  * For nested:
4240                  * The handling of the MSR bitmap for L2 guests is done in
4241                  * nested_svm_vmrun_msrpm.
4242                  * We update the L1 MSR bit as well since it will end up
4243                  * touching the MSR anyway now.
4244                  */
4245                 set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
4246                 break;
4247         case MSR_IA32_PRED_CMD:
4248                 if (!msr->host_initiated &&
4249                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))
4250                         return 1;
4251
4252                 if (data & ~PRED_CMD_IBPB)
4253                         return 1;
4254
4255                 if (!data)
4256                         break;
4257
4258                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
4259                 if (is_guest_mode(vcpu))
4260                         break;
4261                 set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
4262                 break;
4263         case MSR_AMD64_VIRT_SPEC_CTRL:
4264                 if (!msr->host_initiated &&
4265                     !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
4266                         return 1;
4267
4268                 if (data & ~SPEC_CTRL_SSBD)
4269                         return 1;
4270
4271                 svm->virt_spec_ctrl = data;
4272                 break;
4273         case MSR_STAR:
4274                 svm->vmcb->save.star = data;
4275                 break;
4276 #ifdef CONFIG_X86_64
4277         case MSR_LSTAR:
4278                 svm->vmcb->save.lstar = data;
4279                 break;
4280         case MSR_CSTAR:
4281                 svm->vmcb->save.cstar = data;
4282                 break;
4283         case MSR_KERNEL_GS_BASE:
4284                 svm->vmcb->save.kernel_gs_base = data;
4285                 break;
4286         case MSR_SYSCALL_MASK:
4287                 svm->vmcb->save.sfmask = data;
4288                 break;
4289 #endif
4290         case MSR_IA32_SYSENTER_CS:
4291                 svm->vmcb->save.sysenter_cs = data;
4292                 break;
4293         case MSR_IA32_SYSENTER_EIP:
4294                 svm->sysenter_eip = data;
4295                 svm->vmcb->save.sysenter_eip = data;
4296                 break;
4297         case MSR_IA32_SYSENTER_ESP:
4298                 svm->sysenter_esp = data;
4299                 svm->vmcb->save.sysenter_esp = data;
4300                 break;
4301         case MSR_TSC_AUX:
4302                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4303                         return 1;
4304
4305                 /*
4306                  * This is rare, so we update the MSR here instead of using
4307                  * direct_access_msrs.  Doing that would require a rdmsr in
4308                  * svm_vcpu_put.
4309                  */
4310                 svm->tsc_aux = data;
4311                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
4312                 break;
4313         case MSR_IA32_DEBUGCTLMSR:
4314                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
4315                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
4316                                     __func__, data);
4317                         break;
4318                 }
4319                 if (data & DEBUGCTL_RESERVED_BITS)
4320                         return 1;
4321
4322                 svm->vmcb->save.dbgctl = data;
4323                 mark_dirty(svm->vmcb, VMCB_LBR);
4324                 if (data & (1ULL<<0))
4325                         svm_enable_lbrv(svm);
4326                 else
4327                         svm_disable_lbrv(svm);
4328                 break;
4329         case MSR_VM_HSAVE_PA:
4330                 svm->nested.hsave_msr = data;
4331                 break;
4332         case MSR_VM_CR:
4333                 return svm_set_vm_cr(vcpu, data);
4334         case MSR_VM_IGNNE:
4335                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
4336                 break;
4337         case MSR_F10H_DECFG: {
4338                 struct kvm_msr_entry msr_entry;
4339
4340                 msr_entry.index = msr->index;
4341                 if (svm_get_msr_feature(&msr_entry))
4342                         return 1;
4343
4344                 /* Check the supported bits */
4345                 if (data & ~msr_entry.data)
4346                         return 1;
4347
4348                 /* Don't allow the guest to change a bit, #GP */
4349                 if (!msr->host_initiated && (data ^ msr_entry.data))
4350                         return 1;
4351
4352                 svm->msr_decfg = data;
4353                 break;
4354         }
4355         case MSR_IA32_APICBASE:
4356                 if (kvm_vcpu_apicv_active(vcpu))
4357                         avic_update_vapic_bar(to_svm(vcpu), data);
4358                 /* Follow through */
4359         default:
4360                 return kvm_set_msr_common(vcpu, msr);
4361         }
4362         return 0;
4363 }
4364
4365 static int wrmsr_interception(struct vcpu_svm *svm)
4366 {
4367         struct msr_data msr;
4368         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
4369         u64 data = kvm_read_edx_eax(&svm->vcpu);
4370
4371         msr.data = data;
4372         msr.index = ecx;
4373         msr.host_initiated = false;
4374
4375         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
4376         if (kvm_set_msr(&svm->vcpu, &msr)) {
4377                 trace_kvm_msr_write_ex(ecx, data);
4378                 kvm_inject_gp(&svm->vcpu, 0);
4379                 return 1;
4380         } else {
4381                 trace_kvm_msr_write(ecx, data);
4382                 return kvm_skip_emulated_instruction(&svm->vcpu);
4383         }
4384 }
4385
4386 static int msr_interception(struct vcpu_svm *svm)
4387 {
4388         if (svm->vmcb->control.exit_info_1)
4389                 return wrmsr_interception(svm);
4390         else
4391                 return rdmsr_interception(svm);
4392 }
4393
4394 static int interrupt_window_interception(struct vcpu_svm *svm)
4395 {
4396         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
4397         svm_clear_vintr(svm);
4398         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
4399         mark_dirty(svm->vmcb, VMCB_INTR);
4400         ++svm->vcpu.stat.irq_window_exits;
4401         return 1;
4402 }
4403
4404 static int pause_interception(struct vcpu_svm *svm)
4405 {
4406         struct kvm_vcpu *vcpu = &svm->vcpu;
4407         bool in_kernel = (svm_get_cpl(vcpu) == 0);
4408
4409         if (pause_filter_thresh)
4410                 grow_ple_window(vcpu);
4411
4412         kvm_vcpu_on_spin(vcpu, in_kernel);
4413         return 1;
4414 }
4415
4416 static int nop_interception(struct vcpu_svm *svm)
4417 {
4418         return kvm_skip_emulated_instruction(&(svm->vcpu));
4419 }
4420
4421 static int monitor_interception(struct vcpu_svm *svm)
4422 {
4423         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
4424         return nop_interception(svm);
4425 }
4426
4427 static int mwait_interception(struct vcpu_svm *svm)
4428 {
4429         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
4430         return nop_interception(svm);
4431 }
4432
4433 enum avic_ipi_failure_cause {
4434         AVIC_IPI_FAILURE_INVALID_INT_TYPE,
4435         AVIC_IPI_FAILURE_TARGET_NOT_RUNNING,
4436         AVIC_IPI_FAILURE_INVALID_TARGET,
4437         AVIC_IPI_FAILURE_INVALID_BACKING_PAGE,
4438 };
4439
4440 static int avic_incomplete_ipi_interception(struct vcpu_svm *svm)
4441 {
4442         u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
4443         u32 icrl = svm->vmcb->control.exit_info_1;
4444         u32 id = svm->vmcb->control.exit_info_2 >> 32;
4445         u32 index = svm->vmcb->control.exit_info_2 & 0xFF;
4446         struct kvm_lapic *apic = svm->vcpu.arch.apic;
4447
4448         trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index);
4449
4450         switch (id) {
4451         case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
4452                 /*
4453                  * AVIC hardware handles the generation of
4454                  * IPIs when the specified Message Type is Fixed
4455                  * (also known as fixed delivery mode) and
4456                  * the Trigger Mode is edge-triggered. The hardware
4457                  * also supports self and broadcast delivery modes
4458                  * specified via the Destination Shorthand(DSH)
4459                  * field of the ICRL. Logical and physical APIC ID
4460                  * formats are supported. All other IPI types cause
4461                  * a #VMEXIT, which needs to emulated.
4462                  */
4463                 kvm_lapic_reg_write(apic, APIC_ICR2, icrh);
4464                 kvm_lapic_reg_write(apic, APIC_ICR, icrl);
4465                 break;
4466         case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: {
4467                 int i;
4468                 struct kvm_vcpu *vcpu;
4469                 struct kvm *kvm = svm->vcpu.kvm;
4470                 struct kvm_lapic *apic = svm->vcpu.arch.apic;
4471
4472                 /*
4473                  * At this point, we expect that the AVIC HW has already
4474                  * set the appropriate IRR bits on the valid target
4475                  * vcpus. So, we just need to kick the appropriate vcpu.
4476                  */
4477                 kvm_for_each_vcpu(i, vcpu, kvm) {
4478                         bool m = kvm_apic_match_dest(vcpu, apic,
4479                                                      icrl & KVM_APIC_SHORT_MASK,
4480                                                      GET_APIC_DEST_FIELD(icrh),
4481                                                      icrl & KVM_APIC_DEST_MASK);
4482
4483                         if (m && !avic_vcpu_is_running(vcpu))
4484                                 kvm_vcpu_wake_up(vcpu);
4485                 }
4486                 break;
4487         }
4488         case AVIC_IPI_FAILURE_INVALID_TARGET:
4489                 break;
4490         case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
4491                 WARN_ONCE(1, "Invalid backing page\n");
4492                 break;
4493         default:
4494                 pr_err("Unknown IPI interception\n");
4495         }
4496
4497         return 1;
4498 }
4499
4500 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
4501 {
4502         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
4503         int index;
4504         u32 *logical_apic_id_table;
4505         int dlid = GET_APIC_LOGICAL_ID(ldr);
4506
4507         if (!dlid)
4508                 return NULL;
4509
4510         if (flat) { /* flat */
4511                 index = ffs(dlid) - 1;
4512                 if (index > 7)
4513                         return NULL;
4514         } else { /* cluster */
4515                 int cluster = (dlid & 0xf0) >> 4;
4516                 int apic = ffs(dlid & 0x0f) - 1;
4517
4518                 if ((apic < 0) || (apic > 7) ||
4519                     (cluster >= 0xf))
4520                         return NULL;
4521                 index = (cluster << 2) + apic;
4522         }
4523
4524         logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page);
4525
4526         return &logical_apic_id_table[index];
4527 }
4528
4529 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr,
4530                           bool valid)
4531 {
4532         bool flat;
4533         u32 *entry, new_entry;
4534
4535         flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
4536         entry = avic_get_logical_id_entry(vcpu, ldr, flat);
4537         if (!entry)
4538                 return -EINVAL;
4539
4540         new_entry = READ_ONCE(*entry);
4541         new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
4542         new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
4543         if (valid)
4544                 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
4545         else
4546                 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
4547         WRITE_ONCE(*entry, new_entry);
4548
4549         return 0;
4550 }
4551
4552 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
4553 {
4554         int ret;
4555         struct vcpu_svm *svm = to_svm(vcpu);
4556         u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
4557
4558         if (!ldr)
4559                 return 1;
4560
4561         ret = avic_ldr_write(vcpu, vcpu->vcpu_id, ldr, true);
4562         if (ret && svm->ldr_reg) {
4563                 avic_ldr_write(vcpu, 0, svm->ldr_reg, false);
4564                 svm->ldr_reg = 0;
4565         } else {
4566                 svm->ldr_reg = ldr;
4567         }
4568         return ret;
4569 }
4570
4571 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu)
4572 {
4573         u64 *old, *new;
4574         struct vcpu_svm *svm = to_svm(vcpu);
4575         u32 apic_id_reg = kvm_lapic_get_reg(vcpu->arch.apic, APIC_ID);
4576         u32 id = (apic_id_reg >> 24) & 0xff;
4577
4578         if (vcpu->vcpu_id == id)
4579                 return 0;
4580
4581         old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id);
4582         new = avic_get_physical_id_entry(vcpu, id);
4583         if (!new || !old)
4584                 return 1;
4585
4586         /* We need to move physical_id_entry to new offset */
4587         *new = *old;
4588         *old = 0ULL;
4589         to_svm(vcpu)->avic_physical_id_cache = new;
4590
4591         /*
4592          * Also update the guest physical APIC ID in the logical
4593          * APIC ID table entry if already setup the LDR.
4594          */
4595         if (svm->ldr_reg)
4596                 avic_handle_ldr_update(vcpu);
4597
4598         return 0;
4599 }
4600
4601 static int avic_handle_dfr_update(struct kvm_vcpu *vcpu)
4602 {
4603         struct vcpu_svm *svm = to_svm(vcpu);
4604         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
4605         u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
4606         u32 mod = (dfr >> 28) & 0xf;
4607
4608         /*
4609          * We assume that all local APICs are using the same type.
4610          * If this changes, we need to flush the AVIC logical
4611          * APID id table.
4612          */
4613         if (kvm_svm->ldr_mode == mod)
4614                 return 0;
4615
4616         clear_page(page_address(kvm_svm->avic_logical_id_table_page));
4617         kvm_svm->ldr_mode = mod;
4618
4619         if (svm->ldr_reg)
4620                 avic_handle_ldr_update(vcpu);
4621         return 0;
4622 }
4623
4624 static int avic_unaccel_trap_write(struct vcpu_svm *svm)
4625 {
4626         struct kvm_lapic *apic = svm->vcpu.arch.apic;
4627         u32 offset = svm->vmcb->control.exit_info_1 &
4628                                 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4629
4630         switch (offset) {
4631         case APIC_ID:
4632                 if (avic_handle_apic_id_update(&svm->vcpu))
4633                         return 0;
4634                 break;
4635         case APIC_LDR:
4636                 if (avic_handle_ldr_update(&svm->vcpu))
4637                         return 0;
4638                 break;
4639         case APIC_DFR:
4640                 avic_handle_dfr_update(&svm->vcpu);
4641                 break;
4642         default:
4643                 break;
4644         }
4645
4646         kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset));
4647
4648         return 1;
4649 }
4650
4651 static bool is_avic_unaccelerated_access_trap(u32 offset)
4652 {
4653         bool ret = false;
4654
4655         switch (offset) {
4656         case APIC_ID:
4657         case APIC_EOI:
4658         case APIC_RRR:
4659         case APIC_LDR:
4660         case APIC_DFR:
4661         case APIC_SPIV:
4662         case APIC_ESR:
4663         case APIC_ICR:
4664         case APIC_LVTT:
4665         case APIC_LVTTHMR:
4666         case APIC_LVTPC:
4667         case APIC_LVT0:
4668         case APIC_LVT1:
4669         case APIC_LVTERR:
4670         case APIC_TMICT:
4671         case APIC_TDCR:
4672                 ret = true;
4673                 break;
4674         default:
4675                 break;
4676         }
4677         return ret;
4678 }
4679
4680 static int avic_unaccelerated_access_interception(struct vcpu_svm *svm)
4681 {
4682         int ret = 0;
4683         u32 offset = svm->vmcb->control.exit_info_1 &
4684                      AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4685         u32 vector = svm->vmcb->control.exit_info_2 &
4686                      AVIC_UNACCEL_ACCESS_VECTOR_MASK;
4687         bool write = (svm->vmcb->control.exit_info_1 >> 32) &
4688                      AVIC_UNACCEL_ACCESS_WRITE_MASK;
4689         bool trap = is_avic_unaccelerated_access_trap(offset);
4690
4691         trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset,
4692                                             trap, write, vector);
4693         if (trap) {
4694                 /* Handling Trap */
4695                 WARN_ONCE(!write, "svm: Handling trap read.\n");
4696                 ret = avic_unaccel_trap_write(svm);
4697         } else {
4698                 /* Handling Fault */
4699                 ret = (emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE);
4700         }
4701
4702         return ret;
4703 }
4704
4705 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
4706         [SVM_EXIT_READ_CR0]                     = cr_interception,
4707         [SVM_EXIT_READ_CR3]                     = cr_interception,
4708         [SVM_EXIT_READ_CR4]                     = cr_interception,
4709         [SVM_EXIT_READ_CR8]                     = cr_interception,
4710         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
4711         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
4712         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
4713         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
4714         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
4715         [SVM_EXIT_READ_DR0]                     = dr_interception,
4716         [SVM_EXIT_READ_DR1]                     = dr_interception,
4717         [SVM_EXIT_READ_DR2]                     = dr_interception,
4718         [SVM_EXIT_READ_DR3]                     = dr_interception,
4719         [SVM_EXIT_READ_DR4]                     = dr_interception,
4720         [SVM_EXIT_READ_DR5]                     = dr_interception,
4721         [SVM_EXIT_READ_DR6]                     = dr_interception,
4722         [SVM_EXIT_READ_DR7]                     = dr_interception,
4723         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
4724         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
4725         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
4726         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
4727         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
4728         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
4729         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
4730         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
4731         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
4732         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
4733         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
4734         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
4735         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
4736         [SVM_EXIT_EXCP_BASE + AC_VECTOR]        = ac_interception,
4737         [SVM_EXIT_EXCP_BASE + GP_VECTOR]        = gp_interception,
4738         [SVM_EXIT_INTR]                         = intr_interception,
4739         [SVM_EXIT_NMI]                          = nmi_interception,
4740         [SVM_EXIT_SMI]                          = nop_on_interception,
4741         [SVM_EXIT_INIT]                         = nop_on_interception,
4742         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
4743         [SVM_EXIT_RDPMC]                        = rdpmc_interception,
4744         [SVM_EXIT_CPUID]                        = cpuid_interception,
4745         [SVM_EXIT_IRET]                         = iret_interception,
4746         [SVM_EXIT_INVD]                         = emulate_on_interception,
4747         [SVM_EXIT_PAUSE]                        = pause_interception,
4748         [SVM_EXIT_HLT]                          = halt_interception,
4749         [SVM_EXIT_INVLPG]                       = invlpg_interception,
4750         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
4751         [SVM_EXIT_IOIO]                         = io_interception,
4752         [SVM_EXIT_MSR]                          = msr_interception,
4753         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
4754         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
4755         [SVM_EXIT_VMRUN]                        = vmrun_interception,
4756         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
4757         [SVM_EXIT_VMLOAD]                       = vmload_interception,
4758         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
4759         [SVM_EXIT_STGI]                         = stgi_interception,
4760         [SVM_EXIT_CLGI]                         = clgi_interception,
4761         [SVM_EXIT_SKINIT]                       = skinit_interception,
4762         [SVM_EXIT_WBINVD]                       = wbinvd_interception,
4763         [SVM_EXIT_MONITOR]                      = monitor_interception,
4764         [SVM_EXIT_MWAIT]                        = mwait_interception,
4765         [SVM_EXIT_XSETBV]                       = xsetbv_interception,
4766         [SVM_EXIT_NPF]                          = npf_interception,
4767         [SVM_EXIT_RSM]                          = rsm_interception,
4768         [SVM_EXIT_AVIC_INCOMPLETE_IPI]          = avic_incomplete_ipi_interception,
4769         [SVM_EXIT_AVIC_UNACCELERATED_ACCESS]    = avic_unaccelerated_access_interception,
4770 };
4771
4772 static void dump_vmcb(struct kvm_vcpu *vcpu)
4773 {
4774         struct vcpu_svm *svm = to_svm(vcpu);
4775         struct vmcb_control_area *control = &svm->vmcb->control;
4776         struct vmcb_save_area *save = &svm->vmcb->save;
4777
4778         pr_err("VMCB Control Area:\n");
4779         pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
4780         pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
4781         pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
4782         pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
4783         pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
4784         pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
4785         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
4786         pr_err("%-20s%d\n", "pause filter threshold:",
4787                control->pause_filter_thresh);
4788         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
4789         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
4790         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
4791         pr_err("%-20s%d\n", "asid:", control->asid);
4792         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
4793         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
4794         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
4795         pr_err("%-20s%08x\n", "int_state:", control->int_state);
4796         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
4797         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
4798         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
4799         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
4800         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
4801         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
4802         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
4803         pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
4804         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
4805         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
4806         pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
4807         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
4808         pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
4809         pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
4810         pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
4811         pr_err("VMCB State Save Area:\n");
4812         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4813                "es:",
4814                save->es.selector, save->es.attrib,
4815                save->es.limit, save->es.base);
4816         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4817                "cs:",
4818                save->cs.selector, save->cs.attrib,
4819                save->cs.limit, save->cs.base);
4820         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4821                "ss:",
4822                save->ss.selector, save->ss.attrib,
4823                save->ss.limit, save->ss.base);
4824         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4825                "ds:",
4826                save->ds.selector, save->ds.attrib,
4827                save->ds.limit, save->ds.base);
4828         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4829                "fs:",
4830                save->fs.selector, save->fs.attrib,
4831                save->fs.limit, save->fs.base);
4832         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4833                "gs:",
4834                save->gs.selector, save->gs.attrib,
4835                save->gs.limit, save->gs.base);
4836         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4837                "gdtr:",
4838                save->gdtr.selector, save->gdtr.attrib,
4839                save->gdtr.limit, save->gdtr.base);
4840         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4841                "ldtr:",
4842                save->ldtr.selector, save->ldtr.attrib,
4843                save->ldtr.limit, save->ldtr.base);
4844         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4845                "idtr:",
4846                save->idtr.selector, save->idtr.attrib,
4847                save->idtr.limit, save->idtr.base);
4848         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4849                "tr:",
4850                save->tr.selector, save->tr.attrib,
4851                save->tr.limit, save->tr.base);
4852         pr_err("cpl:            %d                efer:         %016llx\n",
4853                 save->cpl, save->efer);
4854         pr_err("%-15s %016llx %-13s %016llx\n",
4855                "cr0:", save->cr0, "cr2:", save->cr2);
4856         pr_err("%-15s %016llx %-13s %016llx\n",
4857                "cr3:", save->cr3, "cr4:", save->cr4);
4858         pr_err("%-15s %016llx %-13s %016llx\n",
4859                "dr6:", save->dr6, "dr7:", save->dr7);
4860         pr_err("%-15s %016llx %-13s %016llx\n",
4861                "rip:", save->rip, "rflags:", save->rflags);
4862         pr_err("%-15s %016llx %-13s %016llx\n",
4863                "rsp:", save->rsp, "rax:", save->rax);
4864         pr_err("%-15s %016llx %-13s %016llx\n",
4865                "star:", save->star, "lstar:", save->lstar);
4866         pr_err("%-15s %016llx %-13s %016llx\n",
4867                "cstar:", save->cstar, "sfmask:", save->sfmask);
4868         pr_err("%-15s %016llx %-13s %016llx\n",
4869                "kernel_gs_base:", save->kernel_gs_base,
4870                "sysenter_cs:", save->sysenter_cs);
4871         pr_err("%-15s %016llx %-13s %016llx\n",
4872                "sysenter_esp:", save->sysenter_esp,
4873                "sysenter_eip:", save->sysenter_eip);
4874         pr_err("%-15s %016llx %-13s %016llx\n",
4875                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
4876         pr_err("%-15s %016llx %-13s %016llx\n",
4877                "br_from:", save->br_from, "br_to:", save->br_to);
4878         pr_err("%-15s %016llx %-13s %016llx\n",
4879                "excp_from:", save->last_excp_from,
4880                "excp_to:", save->last_excp_to);
4881 }
4882
4883 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
4884 {
4885         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
4886
4887         *info1 = control->exit_info_1;
4888         *info2 = control->exit_info_2;
4889 }
4890
4891 static int handle_exit(struct kvm_vcpu *vcpu)
4892 {
4893         struct vcpu_svm *svm = to_svm(vcpu);
4894         struct kvm_run *kvm_run = vcpu->run;
4895         u32 exit_code = svm->vmcb->control.exit_code;
4896
4897         trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
4898
4899         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
4900                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
4901         if (npt_enabled)
4902                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
4903
4904         if (unlikely(svm->nested.exit_required)) {
4905                 nested_svm_vmexit(svm);
4906                 svm->nested.exit_required = false;
4907
4908                 return 1;
4909         }
4910
4911         if (is_guest_mode(vcpu)) {
4912                 int vmexit;
4913
4914                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
4915                                         svm->vmcb->control.exit_info_1,
4916                                         svm->vmcb->control.exit_info_2,
4917                                         svm->vmcb->control.exit_int_info,
4918                                         svm->vmcb->control.exit_int_info_err,
4919                                         KVM_ISA_SVM);
4920
4921                 vmexit = nested_svm_exit_special(svm);
4922
4923                 if (vmexit == NESTED_EXIT_CONTINUE)
4924                         vmexit = nested_svm_exit_handled(svm);
4925
4926                 if (vmexit == NESTED_EXIT_DONE)
4927                         return 1;
4928         }
4929
4930         svm_complete_interrupts(svm);
4931
4932         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
4933                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4934                 kvm_run->fail_entry.hardware_entry_failure_reason
4935                         = svm->vmcb->control.exit_code;
4936                 pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
4937                 dump_vmcb(vcpu);
4938                 return 0;
4939         }
4940
4941         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
4942             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
4943             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
4944             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
4945                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
4946                        "exit_code 0x%x\n",
4947                        __func__, svm->vmcb->control.exit_int_info,
4948                        exit_code);
4949
4950         if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
4951             || !svm_exit_handlers[exit_code]) {
4952                 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
4953                 kvm_queue_exception(vcpu, UD_VECTOR);
4954                 return 1;
4955         }
4956
4957         return svm_exit_handlers[exit_code](svm);
4958 }
4959
4960 static void reload_tss(struct kvm_vcpu *vcpu)
4961 {
4962         int cpu = raw_smp_processor_id();
4963
4964         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
4965         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
4966         load_TR_desc();
4967 }
4968
4969 static void pre_sev_run(struct vcpu_svm *svm, int cpu)
4970 {
4971         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
4972         int asid = sev_get_asid(svm->vcpu.kvm);
4973
4974         /* Assign the asid allocated with this SEV guest */
4975         svm->vmcb->control.asid = asid;
4976
4977         /*
4978          * Flush guest TLB:
4979          *
4980          * 1) when different VMCB for the same ASID is to be run on the same host CPU.
4981          * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
4982          */
4983         if (sd->sev_vmcbs[asid] == svm->vmcb &&
4984             svm->last_cpu == cpu)
4985                 return;
4986
4987         svm->last_cpu = cpu;
4988         sd->sev_vmcbs[asid] = svm->vmcb;
4989         svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
4990         mark_dirty(svm->vmcb, VMCB_ASID);
4991 }
4992
4993 static void pre_svm_run(struct vcpu_svm *svm)
4994 {
4995         int cpu = raw_smp_processor_id();
4996
4997         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
4998
4999         if (sev_guest(svm->vcpu.kvm))
5000                 return pre_sev_run(svm, cpu);
5001
5002         /* FIXME: handle wraparound of asid_generation */
5003         if (svm->asid_generation != sd->asid_generation)
5004                 new_asid(svm, sd);
5005 }
5006
5007 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
5008 {
5009         struct vcpu_svm *svm = to_svm(vcpu);
5010
5011         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
5012         vcpu->arch.hflags |= HF_NMI_MASK;
5013         set_intercept(svm, INTERCEPT_IRET);
5014         ++vcpu->stat.nmi_injections;
5015 }
5016
5017 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
5018 {
5019         struct vmcb_control_area *control;
5020
5021         /* The following fields are ignored when AVIC is enabled */
5022         control = &svm->vmcb->control;
5023         control->int_vector = irq;
5024         control->int_ctl &= ~V_INTR_PRIO_MASK;
5025         control->int_ctl |= V_IRQ_MASK |
5026                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
5027         mark_dirty(svm->vmcb, VMCB_INTR);
5028 }
5029
5030 static void svm_set_irq(struct kvm_vcpu *vcpu)
5031 {
5032         struct vcpu_svm *svm = to_svm(vcpu);
5033
5034         BUG_ON(!(gif_set(svm)));
5035
5036         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
5037         ++vcpu->stat.irq_injections;
5038
5039         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
5040                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
5041 }
5042
5043 static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu)
5044 {
5045         return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK);
5046 }
5047
5048 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
5049 {
5050         struct vcpu_svm *svm = to_svm(vcpu);
5051
5052         if (svm_nested_virtualize_tpr(vcpu) ||
5053             kvm_vcpu_apicv_active(vcpu))
5054                 return;
5055
5056         clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
5057
5058         if (irr == -1)
5059                 return;
5060
5061         if (tpr >= irr)
5062                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
5063 }
5064
5065 static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
5066 {
5067         return;
5068 }
5069
5070 static bool svm_get_enable_apicv(struct kvm_vcpu *vcpu)
5071 {
5072         return avic && irqchip_split(vcpu->kvm);
5073 }
5074
5075 static void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
5076 {
5077 }
5078
5079 static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
5080 {
5081 }
5082
5083 /* Note: Currently only used by Hyper-V. */
5084 static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
5085 {
5086         struct vcpu_svm *svm = to_svm(vcpu);
5087         struct vmcb *vmcb = svm->vmcb;
5088
5089         if (!kvm_vcpu_apicv_active(&svm->vcpu))
5090                 return;
5091
5092         vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
5093         mark_dirty(vmcb, VMCB_INTR);
5094 }
5095
5096 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
5097 {
5098         return;
5099 }
5100
5101 static void svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec)
5102 {
5103         kvm_lapic_set_irr(vec, vcpu->arch.apic);
5104         smp_mb__after_atomic();
5105
5106         if (avic_vcpu_is_running(vcpu))
5107                 wrmsrl(SVM_AVIC_DOORBELL,
5108                        kvm_cpu_get_apicid(vcpu->cpu));
5109         else
5110                 kvm_vcpu_wake_up(vcpu);
5111 }
5112
5113 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
5114 {
5115         unsigned long flags;
5116         struct amd_svm_iommu_ir *cur;
5117
5118         spin_lock_irqsave(&svm->ir_list_lock, flags);
5119         list_for_each_entry(cur, &svm->ir_list, node) {
5120                 if (cur->data != pi->ir_data)
5121                         continue;
5122                 list_del(&cur->node);
5123                 kfree(cur);
5124                 break;
5125         }
5126         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5127 }
5128
5129 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
5130 {
5131         int ret = 0;
5132         unsigned long flags;
5133         struct amd_svm_iommu_ir *ir;
5134
5135         /**
5136          * In some cases, the existing irte is updaed and re-set,
5137          * so we need to check here if it's already been * added
5138          * to the ir_list.
5139          */
5140         if (pi->ir_data && (pi->prev_ga_tag != 0)) {
5141                 struct kvm *kvm = svm->vcpu.kvm;
5142                 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
5143                 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
5144                 struct vcpu_svm *prev_svm;
5145
5146                 if (!prev_vcpu) {
5147                         ret = -EINVAL;
5148                         goto out;
5149                 }
5150
5151                 prev_svm = to_svm(prev_vcpu);
5152                 svm_ir_list_del(prev_svm, pi);
5153         }
5154
5155         /**
5156          * Allocating new amd_iommu_pi_data, which will get
5157          * add to the per-vcpu ir_list.
5158          */
5159         ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL);
5160         if (!ir) {
5161                 ret = -ENOMEM;
5162                 goto out;
5163         }
5164         ir->data = pi->ir_data;
5165
5166         spin_lock_irqsave(&svm->ir_list_lock, flags);
5167         list_add(&ir->node, &svm->ir_list);
5168         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5169 out:
5170         return ret;
5171 }
5172
5173 /**
5174  * Note:
5175  * The HW cannot support posting multicast/broadcast
5176  * interrupts to a vCPU. So, we still use legacy interrupt
5177  * remapping for these kind of interrupts.
5178  *
5179  * For lowest-priority interrupts, we only support
5180  * those with single CPU as the destination, e.g. user
5181  * configures the interrupts via /proc/irq or uses
5182  * irqbalance to make the interrupts single-CPU.
5183  */
5184 static int
5185 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
5186                  struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
5187 {
5188         struct kvm_lapic_irq irq;
5189         struct kvm_vcpu *vcpu = NULL;
5190
5191         kvm_set_msi_irq(kvm, e, &irq);
5192
5193         if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
5194                 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
5195                          __func__, irq.vector);
5196                 return -1;
5197         }
5198
5199         pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
5200                  irq.vector);
5201         *svm = to_svm(vcpu);
5202         vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
5203         vcpu_info->vector = irq.vector;
5204
5205         return 0;
5206 }
5207
5208 /*
5209  * svm_update_pi_irte - set IRTE for Posted-Interrupts
5210  *
5211  * @kvm: kvm
5212  * @host_irq: host irq of the interrupt
5213  * @guest_irq: gsi of the interrupt
5214  * @set: set or unset PI
5215  * returns 0 on success, < 0 on failure
5216  */
5217 static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
5218                               uint32_t guest_irq, bool set)
5219 {
5220         struct kvm_kernel_irq_routing_entry *e;
5221         struct kvm_irq_routing_table *irq_rt;
5222         int idx, ret = -EINVAL;
5223
5224         if (!kvm_arch_has_assigned_device(kvm) ||
5225             !irq_remapping_cap(IRQ_POSTING_CAP))
5226                 return 0;
5227
5228         pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
5229                  __func__, host_irq, guest_irq, set);
5230
5231         idx = srcu_read_lock(&kvm->irq_srcu);
5232         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
5233         WARN_ON(guest_irq >= irq_rt->nr_rt_entries);
5234
5235         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
5236                 struct vcpu_data vcpu_info;
5237                 struct vcpu_svm *svm = NULL;
5238
5239                 if (e->type != KVM_IRQ_ROUTING_MSI)
5240                         continue;
5241
5242                 /**
5243                  * Here, we setup with legacy mode in the following cases:
5244                  * 1. When cannot target interrupt to a specific vcpu.
5245                  * 2. Unsetting posted interrupt.
5246                  * 3. APIC virtialization is disabled for the vcpu.
5247                  */
5248                 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
5249                     kvm_vcpu_apicv_active(&svm->vcpu)) {
5250                         struct amd_iommu_pi_data pi;
5251
5252                         /* Try to enable guest_mode in IRTE */
5253                         pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
5254                                             AVIC_HPA_MASK);
5255                         pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
5256                                                      svm->vcpu.vcpu_id);
5257                         pi.is_guest_mode = true;
5258                         pi.vcpu_data = &vcpu_info;
5259                         ret = irq_set_vcpu_affinity(host_irq, &pi);
5260
5261                         /**
5262                          * Here, we successfully setting up vcpu affinity in
5263                          * IOMMU guest mode. Now, we need to store the posted
5264                          * interrupt information in a per-vcpu ir_list so that
5265                          * we can reference to them directly when we update vcpu
5266                          * scheduling information in IOMMU irte.
5267                          */
5268                         if (!ret && pi.is_guest_mode)
5269                                 svm_ir_list_add(svm, &pi);
5270                 } else {
5271                         /* Use legacy mode in IRTE */
5272                         struct amd_iommu_pi_data pi;
5273
5274                         /**
5275                          * Here, pi is used to:
5276                          * - Tell IOMMU to use legacy mode for this interrupt.
5277                          * - Retrieve ga_tag of prior interrupt remapping data.
5278                          */
5279                         pi.is_guest_mode = false;
5280                         ret = irq_set_vcpu_affinity(host_irq, &pi);
5281
5282                         /**
5283                          * Check if the posted interrupt was previously
5284                          * setup with the guest_mode by checking if the ga_tag
5285                          * was cached. If so, we need to clean up the per-vcpu
5286                          * ir_list.
5287                          */
5288                         if (!ret && pi.prev_ga_tag) {
5289                                 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
5290                                 struct kvm_vcpu *vcpu;
5291
5292                                 vcpu = kvm_get_vcpu_by_id(kvm, id);
5293                                 if (vcpu)
5294                                         svm_ir_list_del(to_svm(vcpu), &pi);
5295                         }
5296                 }
5297
5298                 if (!ret && svm) {
5299                         trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id,
5300                                                  e->gsi, vcpu_info.vector,
5301                                                  vcpu_info.pi_desc_addr, set);
5302                 }
5303
5304                 if (ret < 0) {
5305                         pr_err("%s: failed to update PI IRTE\n", __func__);
5306                         goto out;
5307                 }
5308         }
5309
5310         ret = 0;
5311 out:
5312         srcu_read_unlock(&kvm->irq_srcu, idx);
5313         return ret;
5314 }
5315
5316 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
5317 {
5318         struct vcpu_svm *svm = to_svm(vcpu);
5319         struct vmcb *vmcb = svm->vmcb;
5320         int ret;
5321         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
5322               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
5323         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
5324
5325         return ret;
5326 }
5327
5328 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
5329 {
5330         struct vcpu_svm *svm = to_svm(vcpu);
5331
5332         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
5333 }
5334
5335 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
5336 {
5337         struct vcpu_svm *svm = to_svm(vcpu);
5338
5339         if (masked) {
5340                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
5341                 set_intercept(svm, INTERCEPT_IRET);
5342         } else {
5343                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
5344                 clr_intercept(svm, INTERCEPT_IRET);
5345         }
5346 }
5347
5348 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
5349 {
5350         struct vcpu_svm *svm = to_svm(vcpu);
5351         struct vmcb *vmcb = svm->vmcb;
5352         int ret;
5353
5354         if (!gif_set(svm) ||
5355              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
5356                 return 0;
5357
5358         ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
5359
5360         if (is_guest_mode(vcpu))
5361                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
5362
5363         return ret;
5364 }
5365
5366 static void enable_irq_window(struct kvm_vcpu *vcpu)
5367 {
5368         struct vcpu_svm *svm = to_svm(vcpu);
5369
5370         if (kvm_vcpu_apicv_active(vcpu))
5371                 return;
5372
5373         /*
5374          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
5375          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
5376          * get that intercept, this function will be called again though and
5377          * we'll get the vintr intercept. However, if the vGIF feature is
5378          * enabled, the STGI interception will not occur. Enable the irq
5379          * window under the assumption that the hardware will set the GIF.
5380          */
5381         if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) {
5382                 svm_set_vintr(svm);
5383                 svm_inject_irq(svm, 0x0);
5384         }
5385 }
5386
5387 static void enable_nmi_window(struct kvm_vcpu *vcpu)
5388 {
5389         struct vcpu_svm *svm = to_svm(vcpu);
5390
5391         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
5392             == HF_NMI_MASK)
5393                 return; /* IRET will cause a vm exit */
5394
5395         if (!gif_set(svm)) {
5396                 if (vgif_enabled(svm))
5397                         set_intercept(svm, INTERCEPT_STGI);
5398                 return; /* STGI will cause a vm exit */
5399         }
5400
5401         if (svm->nested.exit_required)
5402                 return; /* we're not going to run the guest yet */
5403
5404         /*
5405          * Something prevents NMI from been injected. Single step over possible
5406          * problem (IRET or exception injection or interrupt shadow)
5407          */
5408         svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
5409         svm->nmi_singlestep = true;
5410         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
5411 }
5412
5413 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
5414 {
5415         return 0;
5416 }
5417
5418 static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
5419 {
5420         return 0;
5421 }
5422
5423 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
5424 {
5425         struct vcpu_svm *svm = to_svm(vcpu);
5426
5427         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
5428                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
5429         else
5430                 svm->asid_generation--;
5431 }
5432
5433 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
5434 {
5435 }
5436
5437 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
5438 {
5439         struct vcpu_svm *svm = to_svm(vcpu);
5440
5441         if (svm_nested_virtualize_tpr(vcpu))
5442                 return;
5443
5444         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
5445                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
5446                 kvm_set_cr8(vcpu, cr8);
5447         }
5448 }
5449
5450 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
5451 {
5452         struct vcpu_svm *svm = to_svm(vcpu);
5453         u64 cr8;
5454
5455         if (svm_nested_virtualize_tpr(vcpu) ||
5456             kvm_vcpu_apicv_active(vcpu))
5457                 return;
5458
5459         cr8 = kvm_get_cr8(vcpu);
5460         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
5461         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
5462 }
5463
5464 static void svm_complete_interrupts(struct vcpu_svm *svm)
5465 {
5466         u8 vector;
5467         int type;
5468         u32 exitintinfo = svm->vmcb->control.exit_int_info;
5469         unsigned int3_injected = svm->int3_injected;
5470
5471         svm->int3_injected = 0;
5472
5473         /*
5474          * If we've made progress since setting HF_IRET_MASK, we've
5475          * executed an IRET and can allow NMI injection.
5476          */
5477         if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
5478             && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
5479                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
5480                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5481         }
5482
5483         svm->vcpu.arch.nmi_injected = false;
5484         kvm_clear_exception_queue(&svm->vcpu);
5485         kvm_clear_interrupt_queue(&svm->vcpu);
5486
5487         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
5488                 return;
5489
5490         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5491
5492         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
5493         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
5494
5495         switch (type) {
5496         case SVM_EXITINTINFO_TYPE_NMI:
5497                 svm->vcpu.arch.nmi_injected = true;
5498                 break;
5499         case SVM_EXITINTINFO_TYPE_EXEPT:
5500                 /*
5501                  * In case of software exceptions, do not reinject the vector,
5502                  * but re-execute the instruction instead. Rewind RIP first
5503                  * if we emulated INT3 before.
5504                  */
5505                 if (kvm_exception_is_soft(vector)) {
5506                         if (vector == BP_VECTOR && int3_injected &&
5507                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
5508                                 kvm_rip_write(&svm->vcpu,
5509                                               kvm_rip_read(&svm->vcpu) -
5510                                               int3_injected);
5511                         break;
5512                 }
5513                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
5514                         u32 err = svm->vmcb->control.exit_int_info_err;
5515                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
5516
5517                 } else
5518                         kvm_requeue_exception(&svm->vcpu, vector);
5519                 break;
5520         case SVM_EXITINTINFO_TYPE_INTR:
5521                 kvm_queue_interrupt(&svm->vcpu, vector, false);
5522                 break;
5523         default:
5524                 break;
5525         }
5526 }
5527
5528 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
5529 {
5530         struct vcpu_svm *svm = to_svm(vcpu);
5531         struct vmcb_control_area *control = &svm->vmcb->control;
5532
5533         control->exit_int_info = control->event_inj;
5534         control->exit_int_info_err = control->event_inj_err;
5535         control->event_inj = 0;
5536         svm_complete_interrupts(svm);
5537 }
5538
5539 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
5540 {
5541         struct vcpu_svm *svm = to_svm(vcpu);
5542
5543         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
5544         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
5545         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
5546
5547         /*
5548          * A vmexit emulation is required before the vcpu can be executed
5549          * again.
5550          */
5551         if (unlikely(svm->nested.exit_required))
5552                 return;
5553
5554         /*
5555          * Disable singlestep if we're injecting an interrupt/exception.
5556          * We don't want our modified rflags to be pushed on the stack where
5557          * we might not be able to easily reset them if we disabled NMI
5558          * singlestep later.
5559          */
5560         if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
5561                 /*
5562                  * Event injection happens before external interrupts cause a
5563                  * vmexit and interrupts are disabled here, so smp_send_reschedule
5564                  * is enough to force an immediate vmexit.
5565                  */
5566                 disable_nmi_singlestep(svm);
5567                 smp_send_reschedule(vcpu->cpu);
5568         }
5569
5570         pre_svm_run(svm);
5571
5572         sync_lapic_to_cr8(vcpu);
5573
5574         svm->vmcb->save.cr2 = vcpu->arch.cr2;
5575
5576         clgi();
5577
5578         local_irq_enable();
5579
5580         /*
5581          * If this vCPU has touched SPEC_CTRL, restore the guest's value if
5582          * it's non-zero. Since vmentry is serialising on affected CPUs, there
5583          * is no need to worry about the conditional branch over the wrmsr
5584          * being speculatively taken.
5585          */
5586         x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
5587
5588         asm volatile (
5589                 "push %%" _ASM_BP "; \n\t"
5590                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
5591                 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
5592                 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
5593                 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
5594                 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
5595                 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
5596 #ifdef CONFIG_X86_64
5597                 "mov %c[r8](%[svm]),  %%r8  \n\t"
5598                 "mov %c[r9](%[svm]),  %%r9  \n\t"
5599                 "mov %c[r10](%[svm]), %%r10 \n\t"
5600                 "mov %c[r11](%[svm]), %%r11 \n\t"
5601                 "mov %c[r12](%[svm]), %%r12 \n\t"
5602                 "mov %c[r13](%[svm]), %%r13 \n\t"
5603                 "mov %c[r14](%[svm]), %%r14 \n\t"
5604                 "mov %c[r15](%[svm]), %%r15 \n\t"
5605 #endif
5606
5607                 /* Enter guest mode */
5608                 "push %%" _ASM_AX " \n\t"
5609                 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
5610                 __ex(SVM_VMLOAD) "\n\t"
5611                 __ex(SVM_VMRUN) "\n\t"
5612                 __ex(SVM_VMSAVE) "\n\t"
5613                 "pop %%" _ASM_AX " \n\t"
5614
5615                 /* Save guest registers, load host registers */
5616                 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
5617                 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
5618                 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
5619                 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
5620                 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
5621                 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
5622 #ifdef CONFIG_X86_64
5623                 "mov %%r8,  %c[r8](%[svm]) \n\t"
5624                 "mov %%r9,  %c[r9](%[svm]) \n\t"
5625                 "mov %%r10, %c[r10](%[svm]) \n\t"
5626                 "mov %%r11, %c[r11](%[svm]) \n\t"
5627                 "mov %%r12, %c[r12](%[svm]) \n\t"
5628                 "mov %%r13, %c[r13](%[svm]) \n\t"
5629                 "mov %%r14, %c[r14](%[svm]) \n\t"
5630                 "mov %%r15, %c[r15](%[svm]) \n\t"
5631 #endif
5632                 /*
5633                 * Clear host registers marked as clobbered to prevent
5634                 * speculative use.
5635                 */
5636                 "xor %%" _ASM_BX ", %%" _ASM_BX " \n\t"
5637                 "xor %%" _ASM_CX ", %%" _ASM_CX " \n\t"
5638                 "xor %%" _ASM_DX ", %%" _ASM_DX " \n\t"
5639                 "xor %%" _ASM_SI ", %%" _ASM_SI " \n\t"
5640                 "xor %%" _ASM_DI ", %%" _ASM_DI " \n\t"
5641 #ifdef CONFIG_X86_64
5642                 "xor %%r8, %%r8 \n\t"
5643                 "xor %%r9, %%r9 \n\t"
5644                 "xor %%r10, %%r10 \n\t"
5645                 "xor %%r11, %%r11 \n\t"
5646                 "xor %%r12, %%r12 \n\t"
5647                 "xor %%r13, %%r13 \n\t"
5648                 "xor %%r14, %%r14 \n\t"
5649                 "xor %%r15, %%r15 \n\t"
5650 #endif
5651                 "pop %%" _ASM_BP
5652                 :
5653                 : [svm]"a"(svm),
5654                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
5655                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
5656                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
5657                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
5658                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
5659                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
5660                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
5661 #ifdef CONFIG_X86_64
5662                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
5663                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
5664                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
5665                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
5666                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
5667                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
5668                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
5669                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
5670 #endif
5671                 : "cc", "memory"
5672 #ifdef CONFIG_X86_64
5673                 , "rbx", "rcx", "rdx", "rsi", "rdi"
5674                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
5675 #else
5676                 , "ebx", "ecx", "edx", "esi", "edi"
5677 #endif
5678                 );
5679
5680         /* Eliminate branch target predictions from guest mode */
5681         vmexit_fill_RSB();
5682
5683 #ifdef CONFIG_X86_64
5684         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
5685 #else
5686         loadsegment(fs, svm->host.fs);
5687 #ifndef CONFIG_X86_32_LAZY_GS
5688         loadsegment(gs, svm->host.gs);
5689 #endif
5690 #endif
5691
5692         /*
5693          * We do not use IBRS in the kernel. If this vCPU has used the
5694          * SPEC_CTRL MSR it may have left it on; save the value and
5695          * turn it off. This is much more efficient than blindly adding
5696          * it to the atomic save/restore list. Especially as the former
5697          * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
5698          *
5699          * For non-nested case:
5700          * If the L01 MSR bitmap does not intercept the MSR, then we need to
5701          * save it.
5702          *
5703          * For nested case:
5704          * If the L02 MSR bitmap does not intercept the MSR, then we need to
5705          * save it.
5706          */
5707         if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
5708                 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
5709
5710         x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
5711
5712         reload_tss(vcpu);
5713
5714         local_irq_disable();
5715
5716         vcpu->arch.cr2 = svm->vmcb->save.cr2;
5717         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
5718         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
5719         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
5720
5721         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5722                 kvm_before_interrupt(&svm->vcpu);
5723
5724         stgi();
5725
5726         /* Any pending NMI will happen here */
5727
5728         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5729                 kvm_after_interrupt(&svm->vcpu);
5730
5731         sync_cr8_to_lapic(vcpu);
5732
5733         svm->next_rip = 0;
5734
5735         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
5736
5737         /* if exit due to PF check for async PF */
5738         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
5739                 svm->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
5740
5741         if (npt_enabled) {
5742                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
5743                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
5744         }
5745
5746         /*
5747          * We need to handle MC intercepts here before the vcpu has a chance to
5748          * change the physical cpu
5749          */
5750         if (unlikely(svm->vmcb->control.exit_code ==
5751                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
5752                 svm_handle_mce(svm);
5753
5754         mark_all_clean(svm->vmcb);
5755 }
5756 STACK_FRAME_NON_STANDARD(svm_vcpu_run);
5757
5758 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5759 {
5760         struct vcpu_svm *svm = to_svm(vcpu);
5761
5762         svm->vmcb->save.cr3 = __sme_set(root);
5763         mark_dirty(svm->vmcb, VMCB_CR);
5764         svm_flush_tlb(vcpu, true);
5765 }
5766
5767 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5768 {
5769         struct vcpu_svm *svm = to_svm(vcpu);
5770
5771         svm->vmcb->control.nested_cr3 = __sme_set(root);
5772         mark_dirty(svm->vmcb, VMCB_NPT);
5773
5774         /* Also sync guest cr3 here in case we live migrate */
5775         svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
5776         mark_dirty(svm->vmcb, VMCB_CR);
5777
5778         svm_flush_tlb(vcpu, true);
5779 }
5780
5781 static int is_disabled(void)
5782 {
5783         u64 vm_cr;
5784
5785         rdmsrl(MSR_VM_CR, vm_cr);
5786         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
5787                 return 1;
5788
5789         return 0;
5790 }
5791
5792 static void
5793 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
5794 {
5795         /*
5796          * Patch in the VMMCALL instruction:
5797          */
5798         hypercall[0] = 0x0f;
5799         hypercall[1] = 0x01;
5800         hypercall[2] = 0xd9;
5801 }
5802
5803 static void svm_check_processor_compat(void *rtn)
5804 {
5805         *(int *)rtn = 0;
5806 }
5807
5808 static bool svm_cpu_has_accelerated_tpr(void)
5809 {
5810         return false;
5811 }
5812
5813 static bool svm_has_emulated_msr(int index)
5814 {
5815         return true;
5816 }
5817
5818 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
5819 {
5820         return 0;
5821 }
5822
5823 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
5824 {
5825         struct vcpu_svm *svm = to_svm(vcpu);
5826
5827         /* Update nrips enabled cache */
5828         svm->nrips_enabled = !!guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS);
5829
5830         if (!kvm_vcpu_apicv_active(vcpu))
5831                 return;
5832
5833         guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
5834 }
5835
5836 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
5837 {
5838         switch (func) {
5839         case 0x1:
5840                 if (avic)
5841                         entry->ecx &= ~bit(X86_FEATURE_X2APIC);
5842                 break;
5843         case 0x80000001:
5844                 if (nested)
5845                         entry->ecx |= (1 << 2); /* Set SVM bit */
5846                 break;
5847         case 0x8000000A:
5848                 entry->eax = 1; /* SVM revision 1 */
5849                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
5850                                    ASID emulation to nested SVM */
5851                 entry->ecx = 0; /* Reserved */
5852                 entry->edx = 0; /* Per default do not support any
5853                                    additional features */
5854
5855                 /* Support next_rip if host supports it */
5856                 if (boot_cpu_has(X86_FEATURE_NRIPS))
5857                         entry->edx |= SVM_FEATURE_NRIP;
5858
5859                 /* Support NPT for the guest if enabled */
5860                 if (npt_enabled)
5861                         entry->edx |= SVM_FEATURE_NPT;
5862
5863                 break;
5864         case 0x8000001F:
5865                 /* Support memory encryption cpuid if host supports it */
5866                 if (boot_cpu_has(X86_FEATURE_SEV))
5867                         cpuid(0x8000001f, &entry->eax, &entry->ebx,
5868                                 &entry->ecx, &entry->edx);
5869
5870         }
5871 }
5872
5873 static int svm_get_lpage_level(void)
5874 {
5875         return PT_PDPE_LEVEL;
5876 }
5877
5878 static bool svm_rdtscp_supported(void)
5879 {
5880         return boot_cpu_has(X86_FEATURE_RDTSCP);
5881 }
5882
5883 static bool svm_invpcid_supported(void)
5884 {
5885         return false;
5886 }
5887
5888 static bool svm_mpx_supported(void)
5889 {
5890         return false;
5891 }
5892
5893 static bool svm_xsaves_supported(void)
5894 {
5895         return false;
5896 }
5897
5898 static bool svm_umip_emulated(void)
5899 {
5900         return false;
5901 }
5902
5903 static bool svm_has_wbinvd_exit(void)
5904 {
5905         return true;
5906 }
5907
5908 #define PRE_EX(exit)  { .exit_code = (exit), \
5909                         .stage = X86_ICPT_PRE_EXCEPT, }
5910 #define POST_EX(exit) { .exit_code = (exit), \
5911                         .stage = X86_ICPT_POST_EXCEPT, }
5912 #define POST_MEM(exit) { .exit_code = (exit), \
5913                         .stage = X86_ICPT_POST_MEMACCESS, }
5914
5915 static const struct __x86_intercept {
5916         u32 exit_code;
5917         enum x86_intercept_stage stage;
5918 } x86_intercept_map[] = {
5919         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
5920         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
5921         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
5922         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
5923         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
5924         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
5925         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
5926         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
5927         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
5928         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
5929         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
5930         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
5931         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
5932         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
5933         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
5934         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
5935         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
5936         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
5937         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
5938         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
5939         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
5940         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
5941         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
5942         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
5943         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
5944         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
5945         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
5946         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
5947         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
5948         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
5949         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
5950         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
5951         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
5952         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
5953         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
5954         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
5955         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
5956         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
5957         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
5958         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
5959         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
5960         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
5961         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
5962         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
5963         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
5964         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
5965 };
5966
5967 #undef PRE_EX
5968 #undef POST_EX
5969 #undef POST_MEM
5970
5971 static int svm_check_intercept(struct kvm_vcpu *vcpu,
5972                                struct x86_instruction_info *info,
5973                                enum x86_intercept_stage stage)
5974 {
5975         struct vcpu_svm *svm = to_svm(vcpu);
5976         int vmexit, ret = X86EMUL_CONTINUE;
5977         struct __x86_intercept icpt_info;
5978         struct vmcb *vmcb = svm->vmcb;
5979
5980         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
5981                 goto out;
5982
5983         icpt_info = x86_intercept_map[info->intercept];
5984
5985         if (stage != icpt_info.stage)
5986                 goto out;
5987
5988         switch (icpt_info.exit_code) {
5989         case SVM_EXIT_READ_CR0:
5990                 if (info->intercept == x86_intercept_cr_read)
5991                         icpt_info.exit_code += info->modrm_reg;
5992                 break;
5993         case SVM_EXIT_WRITE_CR0: {
5994                 unsigned long cr0, val;
5995                 u64 intercept;
5996
5997                 if (info->intercept == x86_intercept_cr_write)
5998                         icpt_info.exit_code += info->modrm_reg;
5999
6000                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
6001                     info->intercept == x86_intercept_clts)
6002                         break;
6003
6004                 intercept = svm->nested.intercept;
6005
6006                 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
6007                         break;
6008
6009                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
6010                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
6011
6012                 if (info->intercept == x86_intercept_lmsw) {
6013                         cr0 &= 0xfUL;
6014                         val &= 0xfUL;
6015                         /* lmsw can't clear PE - catch this here */
6016                         if (cr0 & X86_CR0_PE)
6017                                 val |= X86_CR0_PE;
6018                 }
6019
6020                 if (cr0 ^ val)
6021                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
6022
6023                 break;
6024         }
6025         case SVM_EXIT_READ_DR0:
6026         case SVM_EXIT_WRITE_DR0:
6027                 icpt_info.exit_code += info->modrm_reg;
6028                 break;
6029         case SVM_EXIT_MSR:
6030                 if (info->intercept == x86_intercept_wrmsr)
6031                         vmcb->control.exit_info_1 = 1;
6032                 else
6033                         vmcb->control.exit_info_1 = 0;
6034                 break;
6035         case SVM_EXIT_PAUSE:
6036                 /*
6037                  * We get this for NOP only, but pause
6038                  * is rep not, check this here
6039                  */
6040                 if (info->rep_prefix != REPE_PREFIX)
6041                         goto out;
6042                 break;
6043         case SVM_EXIT_IOIO: {
6044                 u64 exit_info;
6045                 u32 bytes;
6046
6047                 if (info->intercept == x86_intercept_in ||
6048                     info->intercept == x86_intercept_ins) {
6049                         exit_info = ((info->src_val & 0xffff) << 16) |
6050                                 SVM_IOIO_TYPE_MASK;
6051                         bytes = info->dst_bytes;
6052                 } else {
6053                         exit_info = (info->dst_val & 0xffff) << 16;
6054                         bytes = info->src_bytes;
6055                 }
6056
6057                 if (info->intercept == x86_intercept_outs ||
6058                     info->intercept == x86_intercept_ins)
6059                         exit_info |= SVM_IOIO_STR_MASK;
6060
6061                 if (info->rep_prefix)
6062                         exit_info |= SVM_IOIO_REP_MASK;
6063
6064                 bytes = min(bytes, 4u);
6065
6066                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
6067
6068                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
6069
6070                 vmcb->control.exit_info_1 = exit_info;
6071                 vmcb->control.exit_info_2 = info->next_rip;
6072
6073                 break;
6074         }
6075         default:
6076                 break;
6077         }
6078
6079         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
6080         if (static_cpu_has(X86_FEATURE_NRIPS))
6081                 vmcb->control.next_rip  = info->next_rip;
6082         vmcb->control.exit_code = icpt_info.exit_code;
6083         vmexit = nested_svm_exit_handled(svm);
6084
6085         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
6086                                            : X86EMUL_CONTINUE;
6087
6088 out:
6089         return ret;
6090 }
6091
6092 static void svm_handle_external_intr(struct kvm_vcpu *vcpu)
6093 {
6094         local_irq_enable();
6095         /*
6096          * We must have an instruction with interrupts enabled, so
6097          * the timer interrupt isn't delayed by the interrupt shadow.
6098          */
6099         asm("nop");
6100         local_irq_disable();
6101 }
6102
6103 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
6104 {
6105         if (pause_filter_thresh)
6106                 shrink_ple_window(vcpu);
6107 }
6108
6109 static inline void avic_post_state_restore(struct kvm_vcpu *vcpu)
6110 {
6111         if (avic_handle_apic_id_update(vcpu) != 0)
6112                 return;
6113         if (avic_handle_dfr_update(vcpu) != 0)
6114                 return;
6115         avic_handle_ldr_update(vcpu);
6116 }
6117
6118 static void svm_setup_mce(struct kvm_vcpu *vcpu)
6119 {
6120         /* [63:9] are reserved. */
6121         vcpu->arch.mcg_cap &= 0x1ff;
6122 }
6123
6124 static int svm_smi_allowed(struct kvm_vcpu *vcpu)
6125 {
6126         struct vcpu_svm *svm = to_svm(vcpu);
6127
6128         /* Per APM Vol.2 15.22.2 "Response to SMI" */
6129         if (!gif_set(svm))
6130                 return 0;
6131
6132         if (is_guest_mode(&svm->vcpu) &&
6133             svm->nested.intercept & (1ULL << INTERCEPT_SMI)) {
6134                 /* TODO: Might need to set exit_info_1 and exit_info_2 here */
6135                 svm->vmcb->control.exit_code = SVM_EXIT_SMI;
6136                 svm->nested.exit_required = true;
6137                 return 0;
6138         }
6139
6140         return 1;
6141 }
6142
6143 static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
6144 {
6145         struct vcpu_svm *svm = to_svm(vcpu);
6146         int ret;
6147
6148         if (is_guest_mode(vcpu)) {
6149                 /* FED8h - SVM Guest */
6150                 put_smstate(u64, smstate, 0x7ed8, 1);
6151                 /* FEE0h - SVM Guest VMCB Physical Address */
6152                 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb);
6153
6154                 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
6155                 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
6156                 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
6157
6158                 ret = nested_svm_vmexit(svm);
6159                 if (ret)
6160                         return ret;
6161         }
6162         return 0;
6163 }
6164
6165 static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, u64 smbase)
6166 {
6167         struct vcpu_svm *svm = to_svm(vcpu);
6168         struct vmcb *nested_vmcb;
6169         struct page *page;
6170         struct {
6171                 u64 guest;
6172                 u64 vmcb;
6173         } svm_state_save;
6174         int ret;
6175
6176         ret = kvm_vcpu_read_guest(vcpu, smbase + 0xfed8, &svm_state_save,
6177                                   sizeof(svm_state_save));
6178         if (ret)
6179                 return ret;
6180
6181         if (svm_state_save.guest) {
6182                 vcpu->arch.hflags &= ~HF_SMM_MASK;
6183                 nested_vmcb = nested_svm_map(svm, svm_state_save.vmcb, &page);
6184                 if (nested_vmcb)
6185                         enter_svm_guest_mode(svm, svm_state_save.vmcb, nested_vmcb, page);
6186                 else
6187                         ret = 1;
6188                 vcpu->arch.hflags |= HF_SMM_MASK;
6189         }
6190         return ret;
6191 }
6192
6193 static int enable_smi_window(struct kvm_vcpu *vcpu)
6194 {
6195         struct vcpu_svm *svm = to_svm(vcpu);
6196
6197         if (!gif_set(svm)) {
6198                 if (vgif_enabled(svm))
6199                         set_intercept(svm, INTERCEPT_STGI);
6200                 /* STGI will cause a vm exit */
6201                 return 1;
6202         }
6203         return 0;
6204 }
6205
6206 static int sev_asid_new(void)
6207 {
6208         int pos;
6209
6210         /*
6211          * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid.
6212          */
6213         pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1);
6214         if (pos >= max_sev_asid)
6215                 return -EBUSY;
6216
6217         set_bit(pos, sev_asid_bitmap);
6218         return pos + 1;
6219 }
6220
6221 static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
6222 {
6223         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6224         int asid, ret;
6225
6226         ret = -EBUSY;
6227         asid = sev_asid_new();
6228         if (asid < 0)
6229                 return ret;
6230
6231         ret = sev_platform_init(&argp->error);
6232         if (ret)
6233                 goto e_free;
6234
6235         sev->active = true;
6236         sev->asid = asid;
6237         INIT_LIST_HEAD(&sev->regions_list);
6238
6239         return 0;
6240
6241 e_free:
6242         __sev_asid_free(asid);
6243         return ret;
6244 }
6245
6246 static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
6247 {
6248         struct sev_data_activate *data;
6249         int asid = sev_get_asid(kvm);
6250         int ret;
6251
6252         wbinvd_on_all_cpus();
6253
6254         ret = sev_guest_df_flush(error);
6255         if (ret)
6256                 return ret;
6257
6258         data = kzalloc(sizeof(*data), GFP_KERNEL);
6259         if (!data)
6260                 return -ENOMEM;
6261
6262         /* activate ASID on the given handle */
6263         data->handle = handle;
6264         data->asid   = asid;
6265         ret = sev_guest_activate(data, error);
6266         kfree(data);
6267
6268         return ret;
6269 }
6270
6271 static int __sev_issue_cmd(int fd, int id, void *data, int *error)
6272 {
6273         struct fd f;
6274         int ret;
6275
6276         f = fdget(fd);
6277         if (!f.file)
6278                 return -EBADF;
6279
6280         ret = sev_issue_cmd_external_user(f.file, id, data, error);
6281
6282         fdput(f);
6283         return ret;
6284 }
6285
6286 static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
6287 {
6288         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6289
6290         return __sev_issue_cmd(sev->fd, id, data, error);
6291 }
6292
6293 static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
6294 {
6295         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6296         struct sev_data_launch_start *start;
6297         struct kvm_sev_launch_start params;
6298         void *dh_blob, *session_blob;
6299         int *error = &argp->error;
6300         int ret;
6301
6302         if (!sev_guest(kvm))
6303                 return -ENOTTY;
6304
6305         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6306                 return -EFAULT;
6307
6308         start = kzalloc(sizeof(*start), GFP_KERNEL);
6309         if (!start)
6310                 return -ENOMEM;
6311
6312         dh_blob = NULL;
6313         if (params.dh_uaddr) {
6314                 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
6315                 if (IS_ERR(dh_blob)) {
6316                         ret = PTR_ERR(dh_blob);
6317                         goto e_free;
6318                 }
6319
6320                 start->dh_cert_address = __sme_set(__pa(dh_blob));
6321                 start->dh_cert_len = params.dh_len;
6322         }
6323
6324         session_blob = NULL;
6325         if (params.session_uaddr) {
6326                 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
6327                 if (IS_ERR(session_blob)) {
6328                         ret = PTR_ERR(session_blob);
6329                         goto e_free_dh;
6330                 }
6331
6332                 start->session_address = __sme_set(__pa(session_blob));
6333                 start->session_len = params.session_len;
6334         }
6335
6336         start->handle = params.handle;
6337         start->policy = params.policy;
6338
6339         /* create memory encryption context */
6340         ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error);
6341         if (ret)
6342                 goto e_free_session;
6343
6344         /* Bind ASID to this guest */
6345         ret = sev_bind_asid(kvm, start->handle, error);
6346         if (ret)
6347                 goto e_free_session;
6348
6349         /* return handle to userspace */
6350         params.handle = start->handle;
6351         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params))) {
6352                 sev_unbind_asid(kvm, start->handle);
6353                 ret = -EFAULT;
6354                 goto e_free_session;
6355         }
6356
6357         sev->handle = start->handle;
6358         sev->fd = argp->sev_fd;
6359
6360 e_free_session:
6361         kfree(session_blob);
6362 e_free_dh:
6363         kfree(dh_blob);
6364 e_free:
6365         kfree(start);
6366         return ret;
6367 }
6368
6369 static int get_num_contig_pages(int idx, struct page **inpages,
6370                                 unsigned long npages)
6371 {
6372         unsigned long paddr, next_paddr;
6373         int i = idx + 1, pages = 1;
6374
6375         /* find the number of contiguous pages starting from idx */
6376         paddr = __sme_page_pa(inpages[idx]);
6377         while (i < npages) {
6378                 next_paddr = __sme_page_pa(inpages[i++]);
6379                 if ((paddr + PAGE_SIZE) == next_paddr) {
6380                         pages++;
6381                         paddr = next_paddr;
6382                         continue;
6383                 }
6384                 break;
6385         }
6386
6387         return pages;
6388 }
6389
6390 static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
6391 {
6392         unsigned long vaddr, vaddr_end, next_vaddr, npages, size;
6393         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6394         struct kvm_sev_launch_update_data params;
6395         struct sev_data_launch_update_data *data;
6396         struct page **inpages;
6397         int i, ret, pages;
6398
6399         if (!sev_guest(kvm))
6400                 return -ENOTTY;
6401
6402         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6403                 return -EFAULT;
6404
6405         data = kzalloc(sizeof(*data), GFP_KERNEL);
6406         if (!data)
6407                 return -ENOMEM;
6408
6409         vaddr = params.uaddr;
6410         size = params.len;
6411         vaddr_end = vaddr + size;
6412
6413         /* Lock the user memory. */
6414         inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
6415         if (!inpages) {
6416                 ret = -ENOMEM;
6417                 goto e_free;
6418         }
6419
6420         /*
6421          * The LAUNCH_UPDATE command will perform in-place encryption of the
6422          * memory content (i.e it will write the same memory region with C=1).
6423          * It's possible that the cache may contain the data with C=0, i.e.,
6424          * unencrypted so invalidate it first.
6425          */
6426         sev_clflush_pages(inpages, npages);
6427
6428         for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
6429                 int offset, len;
6430
6431                 /*
6432                  * If the user buffer is not page-aligned, calculate the offset
6433                  * within the page.
6434                  */
6435                 offset = vaddr & (PAGE_SIZE - 1);
6436
6437                 /* Calculate the number of pages that can be encrypted in one go. */
6438                 pages = get_num_contig_pages(i, inpages, npages);
6439
6440                 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
6441
6442                 data->handle = sev->handle;
6443                 data->len = len;
6444                 data->address = __sme_page_pa(inpages[i]) + offset;
6445                 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error);
6446                 if (ret)
6447                         goto e_unpin;
6448
6449                 size -= len;
6450                 next_vaddr = vaddr + len;
6451         }
6452
6453 e_unpin:
6454         /* content of memory is updated, mark pages dirty */
6455         for (i = 0; i < npages; i++) {
6456                 set_page_dirty_lock(inpages[i]);
6457                 mark_page_accessed(inpages[i]);
6458         }
6459         /* unlock the user pages */
6460         sev_unpin_memory(kvm, inpages, npages);
6461 e_free:
6462         kfree(data);
6463         return ret;
6464 }
6465
6466 static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
6467 {
6468         void __user *measure = (void __user *)(uintptr_t)argp->data;
6469         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6470         struct sev_data_launch_measure *data;
6471         struct kvm_sev_launch_measure params;
6472         void __user *p = NULL;
6473         void *blob = NULL;
6474         int ret;
6475
6476         if (!sev_guest(kvm))
6477                 return -ENOTTY;
6478
6479         if (copy_from_user(&params, measure, sizeof(params)))
6480                 return -EFAULT;
6481
6482         data = kzalloc(sizeof(*data), GFP_KERNEL);
6483         if (!data)
6484                 return -ENOMEM;
6485
6486         /* User wants to query the blob length */
6487         if (!params.len)
6488                 goto cmd;
6489
6490         p = (void __user *)(uintptr_t)params.uaddr;
6491         if (p) {
6492                 if (params.len > SEV_FW_BLOB_MAX_SIZE) {
6493                         ret = -EINVAL;
6494                         goto e_free;
6495                 }
6496
6497                 ret = -ENOMEM;
6498                 blob = kmalloc(params.len, GFP_KERNEL);
6499                 if (!blob)
6500                         goto e_free;
6501
6502                 data->address = __psp_pa(blob);
6503                 data->len = params.len;
6504         }
6505
6506 cmd:
6507         data->handle = sev->handle;
6508         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error);
6509
6510         /*
6511          * If we query the session length, FW responded with expected data.
6512          */
6513         if (!params.len)
6514                 goto done;
6515
6516         if (ret)
6517                 goto e_free_blob;
6518
6519         if (blob) {
6520                 if (copy_to_user(p, blob, params.len))
6521                         ret = -EFAULT;
6522         }
6523
6524 done:
6525         params.len = data->len;
6526         if (copy_to_user(measure, &params, sizeof(params)))
6527                 ret = -EFAULT;
6528 e_free_blob:
6529         kfree(blob);
6530 e_free:
6531         kfree(data);
6532         return ret;
6533 }
6534
6535 static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
6536 {
6537         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6538         struct sev_data_launch_finish *data;
6539         int ret;
6540
6541         if (!sev_guest(kvm))
6542                 return -ENOTTY;
6543
6544         data = kzalloc(sizeof(*data), GFP_KERNEL);
6545         if (!data)
6546                 return -ENOMEM;
6547
6548         data->handle = sev->handle;
6549         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error);
6550
6551         kfree(data);
6552         return ret;
6553 }
6554
6555 static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
6556 {
6557         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6558         struct kvm_sev_guest_status params;
6559         struct sev_data_guest_status *data;
6560         int ret;
6561
6562         if (!sev_guest(kvm))
6563                 return -ENOTTY;
6564
6565         data = kzalloc(sizeof(*data), GFP_KERNEL);
6566         if (!data)
6567                 return -ENOMEM;
6568
6569         data->handle = sev->handle;
6570         ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error);
6571         if (ret)
6572                 goto e_free;
6573
6574         params.policy = data->policy;
6575         params.state = data->state;
6576         params.handle = data->handle;
6577
6578         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params)))
6579                 ret = -EFAULT;
6580 e_free:
6581         kfree(data);
6582         return ret;
6583 }
6584
6585 static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
6586                                unsigned long dst, int size,
6587                                int *error, bool enc)
6588 {
6589         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6590         struct sev_data_dbg *data;
6591         int ret;
6592
6593         data = kzalloc(sizeof(*data), GFP_KERNEL);
6594         if (!data)
6595                 return -ENOMEM;
6596
6597         data->handle = sev->handle;
6598         data->dst_addr = dst;
6599         data->src_addr = src;
6600         data->len = size;
6601
6602         ret = sev_issue_cmd(kvm,
6603                             enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
6604                             data, error);
6605         kfree(data);
6606         return ret;
6607 }
6608
6609 static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
6610                              unsigned long dst_paddr, int sz, int *err)
6611 {
6612         int offset;
6613
6614         /*
6615          * Its safe to read more than we are asked, caller should ensure that
6616          * destination has enough space.
6617          */
6618         src_paddr = round_down(src_paddr, 16);
6619         offset = src_paddr & 15;
6620         sz = round_up(sz + offset, 16);
6621
6622         return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
6623 }
6624
6625 static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
6626                                   unsigned long __user dst_uaddr,
6627                                   unsigned long dst_paddr,
6628                                   int size, int *err)
6629 {
6630         struct page *tpage = NULL;
6631         int ret, offset;
6632
6633         /* if inputs are not 16-byte then use intermediate buffer */
6634         if (!IS_ALIGNED(dst_paddr, 16) ||
6635             !IS_ALIGNED(paddr,     16) ||
6636             !IS_ALIGNED(size,      16)) {
6637                 tpage = (void *)alloc_page(GFP_KERNEL);
6638                 if (!tpage)
6639                         return -ENOMEM;
6640
6641                 dst_paddr = __sme_page_pa(tpage);
6642         }
6643
6644         ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
6645         if (ret)
6646                 goto e_free;
6647
6648         if (tpage) {
6649                 offset = paddr & 15;
6650                 if (copy_to_user((void __user *)(uintptr_t)dst_uaddr,
6651                                  page_address(tpage) + offset, size))
6652                         ret = -EFAULT;
6653         }
6654
6655 e_free:
6656         if (tpage)
6657                 __free_page(tpage);
6658
6659         return ret;
6660 }
6661
6662 static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
6663                                   unsigned long __user vaddr,
6664                                   unsigned long dst_paddr,
6665                                   unsigned long __user dst_vaddr,
6666                                   int size, int *error)
6667 {
6668         struct page *src_tpage = NULL;
6669         struct page *dst_tpage = NULL;
6670         int ret, len = size;
6671
6672         /* If source buffer is not aligned then use an intermediate buffer */
6673         if (!IS_ALIGNED(vaddr, 16)) {
6674                 src_tpage = alloc_page(GFP_KERNEL);
6675                 if (!src_tpage)
6676                         return -ENOMEM;
6677
6678                 if (copy_from_user(page_address(src_tpage),
6679                                 (void __user *)(uintptr_t)vaddr, size)) {
6680                         __free_page(src_tpage);
6681                         return -EFAULT;
6682                 }
6683
6684                 paddr = __sme_page_pa(src_tpage);
6685         }
6686
6687         /*
6688          *  If destination buffer or length is not aligned then do read-modify-write:
6689          *   - decrypt destination in an intermediate buffer
6690          *   - copy the source buffer in an intermediate buffer
6691          *   - use the intermediate buffer as source buffer
6692          */
6693         if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
6694                 int dst_offset;
6695
6696                 dst_tpage = alloc_page(GFP_KERNEL);
6697                 if (!dst_tpage) {
6698                         ret = -ENOMEM;
6699                         goto e_free;
6700                 }
6701
6702                 ret = __sev_dbg_decrypt(kvm, dst_paddr,
6703                                         __sme_page_pa(dst_tpage), size, error);
6704                 if (ret)
6705                         goto e_free;
6706
6707                 /*
6708                  *  If source is kernel buffer then use memcpy() otherwise
6709                  *  copy_from_user().
6710                  */
6711                 dst_offset = dst_paddr & 15;
6712
6713                 if (src_tpage)
6714                         memcpy(page_address(dst_tpage) + dst_offset,
6715                                page_address(src_tpage), size);
6716                 else {
6717                         if (copy_from_user(page_address(dst_tpage) + dst_offset,
6718                                            (void __user *)(uintptr_t)vaddr, size)) {
6719                                 ret = -EFAULT;
6720                                 goto e_free;
6721                         }
6722                 }
6723
6724                 paddr = __sme_page_pa(dst_tpage);
6725                 dst_paddr = round_down(dst_paddr, 16);
6726                 len = round_up(size, 16);
6727         }
6728
6729         ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
6730
6731 e_free:
6732         if (src_tpage)
6733                 __free_page(src_tpage);
6734         if (dst_tpage)
6735                 __free_page(dst_tpage);
6736         return ret;
6737 }
6738
6739 static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
6740 {
6741         unsigned long vaddr, vaddr_end, next_vaddr;
6742         unsigned long dst_vaddr, dst_vaddr_end;
6743         struct page **src_p, **dst_p;
6744         struct kvm_sev_dbg debug;
6745         unsigned long n;
6746         int ret, size;
6747
6748         if (!sev_guest(kvm))
6749                 return -ENOTTY;
6750
6751         if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
6752                 return -EFAULT;
6753
6754         vaddr = debug.src_uaddr;
6755         size = debug.len;
6756         vaddr_end = vaddr + size;
6757         dst_vaddr = debug.dst_uaddr;
6758         dst_vaddr_end = dst_vaddr + size;
6759
6760         for (; vaddr < vaddr_end; vaddr = next_vaddr) {
6761                 int len, s_off, d_off;
6762
6763                 /* lock userspace source and destination page */
6764                 src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
6765                 if (!src_p)
6766                         return -EFAULT;
6767
6768                 dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
6769                 if (!dst_p) {
6770                         sev_unpin_memory(kvm, src_p, n);
6771                         return -EFAULT;
6772                 }
6773
6774                 /*
6775                  * The DBG_{DE,EN}CRYPT commands will perform {dec,en}cryption of the
6776                  * memory content (i.e it will write the same memory region with C=1).
6777                  * It's possible that the cache may contain the data with C=0, i.e.,
6778                  * unencrypted so invalidate it first.
6779                  */
6780                 sev_clflush_pages(src_p, 1);
6781                 sev_clflush_pages(dst_p, 1);
6782
6783                 /*
6784                  * Since user buffer may not be page aligned, calculate the
6785                  * offset within the page.
6786                  */
6787                 s_off = vaddr & ~PAGE_MASK;
6788                 d_off = dst_vaddr & ~PAGE_MASK;
6789                 len = min_t(size_t, (PAGE_SIZE - s_off), size);
6790
6791                 if (dec)
6792                         ret = __sev_dbg_decrypt_user(kvm,
6793                                                      __sme_page_pa(src_p[0]) + s_off,
6794                                                      dst_vaddr,
6795                                                      __sme_page_pa(dst_p[0]) + d_off,
6796                                                      len, &argp->error);
6797                 else
6798                         ret = __sev_dbg_encrypt_user(kvm,
6799                                                      __sme_page_pa(src_p[0]) + s_off,
6800                                                      vaddr,
6801                                                      __sme_page_pa(dst_p[0]) + d_off,
6802                                                      dst_vaddr,
6803                                                      len, &argp->error);
6804
6805                 sev_unpin_memory(kvm, src_p, 1);
6806                 sev_unpin_memory(kvm, dst_p, 1);
6807
6808                 if (ret)
6809                         goto err;
6810
6811                 next_vaddr = vaddr + len;
6812                 dst_vaddr = dst_vaddr + len;
6813                 size -= len;
6814         }
6815 err:
6816         return ret;
6817 }
6818
6819 static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
6820 {
6821         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6822         struct sev_data_launch_secret *data;
6823         struct kvm_sev_launch_secret params;
6824         struct page **pages;
6825         void *blob, *hdr;
6826         unsigned long n;
6827         int ret, offset;
6828
6829         if (!sev_guest(kvm))
6830                 return -ENOTTY;
6831
6832         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6833                 return -EFAULT;
6834
6835         pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
6836         if (!pages)
6837                 return -ENOMEM;
6838
6839         /*
6840          * The secret must be copied into contiguous memory region, lets verify
6841          * that userspace memory pages are contiguous before we issue command.
6842          */
6843         if (get_num_contig_pages(0, pages, n) != n) {
6844                 ret = -EINVAL;
6845                 goto e_unpin_memory;
6846         }
6847
6848         ret = -ENOMEM;
6849         data = kzalloc(sizeof(*data), GFP_KERNEL);
6850         if (!data)
6851                 goto e_unpin_memory;
6852
6853         offset = params.guest_uaddr & (PAGE_SIZE - 1);
6854         data->guest_address = __sme_page_pa(pages[0]) + offset;
6855         data->guest_len = params.guest_len;
6856
6857         blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
6858         if (IS_ERR(blob)) {
6859                 ret = PTR_ERR(blob);
6860                 goto e_free;
6861         }
6862
6863         data->trans_address = __psp_pa(blob);
6864         data->trans_len = params.trans_len;
6865
6866         hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
6867         if (IS_ERR(hdr)) {
6868                 ret = PTR_ERR(hdr);
6869                 goto e_free_blob;
6870         }
6871         data->hdr_address = __psp_pa(hdr);
6872         data->hdr_len = params.hdr_len;
6873
6874         data->handle = sev->handle;
6875         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error);
6876
6877         kfree(hdr);
6878
6879 e_free_blob:
6880         kfree(blob);
6881 e_free:
6882         kfree(data);
6883 e_unpin_memory:
6884         sev_unpin_memory(kvm, pages, n);
6885         return ret;
6886 }
6887
6888 static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
6889 {
6890         struct kvm_sev_cmd sev_cmd;
6891         int r;
6892
6893         if (!svm_sev_enabled())
6894                 return -ENOTTY;
6895
6896         if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
6897                 return -EFAULT;
6898
6899         mutex_lock(&kvm->lock);
6900
6901         switch (sev_cmd.id) {
6902         case KVM_SEV_INIT:
6903                 r = sev_guest_init(kvm, &sev_cmd);
6904                 break;
6905         case KVM_SEV_LAUNCH_START:
6906                 r = sev_launch_start(kvm, &sev_cmd);
6907                 break;
6908         case KVM_SEV_LAUNCH_UPDATE_DATA:
6909                 r = sev_launch_update_data(kvm, &sev_cmd);
6910                 break;
6911         case KVM_SEV_LAUNCH_MEASURE:
6912                 r = sev_launch_measure(kvm, &sev_cmd);
6913                 break;
6914         case KVM_SEV_LAUNCH_FINISH:
6915                 r = sev_launch_finish(kvm, &sev_cmd);
6916                 break;
6917         case KVM_SEV_GUEST_STATUS:
6918                 r = sev_guest_status(kvm, &sev_cmd);
6919                 break;
6920         case KVM_SEV_DBG_DECRYPT:
6921                 r = sev_dbg_crypt(kvm, &sev_cmd, true);
6922                 break;
6923         case KVM_SEV_DBG_ENCRYPT:
6924                 r = sev_dbg_crypt(kvm, &sev_cmd, false);
6925                 break;
6926         case KVM_SEV_LAUNCH_SECRET:
6927                 r = sev_launch_secret(kvm, &sev_cmd);
6928                 break;
6929         default:
6930                 r = -EINVAL;
6931                 goto out;
6932         }
6933
6934         if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
6935                 r = -EFAULT;
6936
6937 out:
6938         mutex_unlock(&kvm->lock);
6939         return r;
6940 }
6941
6942 static int svm_register_enc_region(struct kvm *kvm,
6943                                    struct kvm_enc_region *range)
6944 {
6945         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6946         struct enc_region *region;
6947         int ret = 0;
6948
6949         if (!sev_guest(kvm))
6950                 return -ENOTTY;
6951
6952         region = kzalloc(sizeof(*region), GFP_KERNEL);
6953         if (!region)
6954                 return -ENOMEM;
6955
6956         region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
6957         if (!region->pages) {
6958                 ret = -ENOMEM;
6959                 goto e_free;
6960         }
6961
6962         /*
6963          * The guest may change the memory encryption attribute from C=0 -> C=1
6964          * or vice versa for this memory range. Lets make sure caches are
6965          * flushed to ensure that guest data gets written into memory with
6966          * correct C-bit.
6967          */
6968         sev_clflush_pages(region->pages, region->npages);
6969
6970         region->uaddr = range->addr;
6971         region->size = range->size;
6972
6973         mutex_lock(&kvm->lock);
6974         list_add_tail(&region->list, &sev->regions_list);
6975         mutex_unlock(&kvm->lock);
6976
6977         return ret;
6978
6979 e_free:
6980         kfree(region);
6981         return ret;
6982 }
6983
6984 static struct enc_region *
6985 find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
6986 {
6987         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6988         struct list_head *head = &sev->regions_list;
6989         struct enc_region *i;
6990
6991         list_for_each_entry(i, head, list) {
6992                 if (i->uaddr == range->addr &&
6993                     i->size == range->size)
6994                         return i;
6995         }
6996
6997         return NULL;
6998 }
6999
7000
7001 static int svm_unregister_enc_region(struct kvm *kvm,
7002                                      struct kvm_enc_region *range)
7003 {
7004         struct enc_region *region;
7005         int ret;
7006
7007         mutex_lock(&kvm->lock);
7008
7009         if (!sev_guest(kvm)) {
7010                 ret = -ENOTTY;
7011                 goto failed;
7012         }
7013
7014         region = find_enc_region(kvm, range);
7015         if (!region) {
7016                 ret = -EINVAL;
7017                 goto failed;
7018         }
7019
7020         __unregister_enc_region_locked(kvm, region);
7021
7022         mutex_unlock(&kvm->lock);
7023         return 0;
7024
7025 failed:
7026         mutex_unlock(&kvm->lock);
7027         return ret;
7028 }
7029
7030 static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
7031         .cpu_has_kvm_support = has_svm,
7032         .disabled_by_bios = is_disabled,
7033         .hardware_setup = svm_hardware_setup,
7034         .hardware_unsetup = svm_hardware_unsetup,
7035         .check_processor_compatibility = svm_check_processor_compat,
7036         .hardware_enable = svm_hardware_enable,
7037         .hardware_disable = svm_hardware_disable,
7038         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
7039         .has_emulated_msr = svm_has_emulated_msr,
7040
7041         .vcpu_create = svm_create_vcpu,
7042         .vcpu_free = svm_free_vcpu,
7043         .vcpu_reset = svm_vcpu_reset,
7044
7045         .vm_alloc = svm_vm_alloc,
7046         .vm_free = svm_vm_free,
7047         .vm_init = avic_vm_init,
7048         .vm_destroy = svm_vm_destroy,
7049
7050         .prepare_guest_switch = svm_prepare_guest_switch,
7051         .vcpu_load = svm_vcpu_load,
7052         .vcpu_put = svm_vcpu_put,
7053         .vcpu_blocking = svm_vcpu_blocking,
7054         .vcpu_unblocking = svm_vcpu_unblocking,
7055
7056         .update_bp_intercept = update_bp_intercept,
7057         .get_msr_feature = svm_get_msr_feature,
7058         .get_msr = svm_get_msr,
7059         .set_msr = svm_set_msr,
7060         .get_segment_base = svm_get_segment_base,
7061         .get_segment = svm_get_segment,
7062         .set_segment = svm_set_segment,
7063         .get_cpl = svm_get_cpl,
7064         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
7065         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
7066         .decache_cr3 = svm_decache_cr3,
7067         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
7068         .set_cr0 = svm_set_cr0,
7069         .set_cr3 = svm_set_cr3,
7070         .set_cr4 = svm_set_cr4,
7071         .set_efer = svm_set_efer,
7072         .get_idt = svm_get_idt,
7073         .set_idt = svm_set_idt,
7074         .get_gdt = svm_get_gdt,
7075         .set_gdt = svm_set_gdt,
7076         .get_dr6 = svm_get_dr6,
7077         .set_dr6 = svm_set_dr6,
7078         .set_dr7 = svm_set_dr7,
7079         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
7080         .cache_reg = svm_cache_reg,
7081         .get_rflags = svm_get_rflags,
7082         .set_rflags = svm_set_rflags,
7083
7084         .tlb_flush = svm_flush_tlb,
7085
7086         .run = svm_vcpu_run,
7087         .handle_exit = handle_exit,
7088         .skip_emulated_instruction = skip_emulated_instruction,
7089         .set_interrupt_shadow = svm_set_interrupt_shadow,
7090         .get_interrupt_shadow = svm_get_interrupt_shadow,
7091         .patch_hypercall = svm_patch_hypercall,
7092         .set_irq = svm_set_irq,
7093         .set_nmi = svm_inject_nmi,
7094         .queue_exception = svm_queue_exception,
7095         .cancel_injection = svm_cancel_injection,
7096         .interrupt_allowed = svm_interrupt_allowed,
7097         .nmi_allowed = svm_nmi_allowed,
7098         .get_nmi_mask = svm_get_nmi_mask,
7099         .set_nmi_mask = svm_set_nmi_mask,
7100         .enable_nmi_window = enable_nmi_window,
7101         .enable_irq_window = enable_irq_window,
7102         .update_cr8_intercept = update_cr8_intercept,
7103         .set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
7104         .get_enable_apicv = svm_get_enable_apicv,
7105         .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
7106         .load_eoi_exitmap = svm_load_eoi_exitmap,
7107         .hwapic_irr_update = svm_hwapic_irr_update,
7108         .hwapic_isr_update = svm_hwapic_isr_update,
7109         .sync_pir_to_irr = kvm_lapic_find_highest_irr,
7110         .apicv_post_state_restore = avic_post_state_restore,
7111
7112         .set_tss_addr = svm_set_tss_addr,
7113         .set_identity_map_addr = svm_set_identity_map_addr,
7114         .get_tdp_level = get_npt_level,
7115         .get_mt_mask = svm_get_mt_mask,
7116
7117         .get_exit_info = svm_get_exit_info,
7118
7119         .get_lpage_level = svm_get_lpage_level,
7120
7121         .cpuid_update = svm_cpuid_update,
7122
7123         .rdtscp_supported = svm_rdtscp_supported,
7124         .invpcid_supported = svm_invpcid_supported,
7125         .mpx_supported = svm_mpx_supported,
7126         .xsaves_supported = svm_xsaves_supported,
7127         .umip_emulated = svm_umip_emulated,
7128
7129         .set_supported_cpuid = svm_set_supported_cpuid,
7130
7131         .has_wbinvd_exit = svm_has_wbinvd_exit,
7132
7133         .read_l1_tsc_offset = svm_read_l1_tsc_offset,
7134         .write_tsc_offset = svm_write_tsc_offset,
7135
7136         .set_tdp_cr3 = set_tdp_cr3,
7137
7138         .check_intercept = svm_check_intercept,
7139         .handle_external_intr = svm_handle_external_intr,
7140
7141         .sched_in = svm_sched_in,
7142
7143         .pmu_ops = &amd_pmu_ops,
7144         .deliver_posted_interrupt = svm_deliver_avic_intr,
7145         .update_pi_irte = svm_update_pi_irte,
7146         .setup_mce = svm_setup_mce,
7147
7148         .smi_allowed = svm_smi_allowed,
7149         .pre_enter_smm = svm_pre_enter_smm,
7150         .pre_leave_smm = svm_pre_leave_smm,
7151         .enable_smi_window = enable_smi_window,
7152
7153         .mem_enc_op = svm_mem_enc_op,
7154         .mem_enc_reg_region = svm_register_enc_region,
7155         .mem_enc_unreg_region = svm_unregister_enc_region,
7156 };
7157
7158 static int __init svm_init(void)
7159 {
7160         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
7161                         __alignof__(struct vcpu_svm), THIS_MODULE);
7162 }
7163
7164 static void __exit svm_exit(void)
7165 {
7166         kvm_exit();
7167 }
7168
7169 module_init(svm_init)
7170 module_exit(svm_exit)