arch/x86/kvm/vmx/vmx.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * This module enables machines with Intel VT-x extensions to run virtual
   6  * machines without emulation or binary translation.
   7  *
   8  * Copyright (C) 2006 Qumranet, Inc.
   9  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  10  *
  11  * Authors:
  12  *   Avi Kivity   <avi@qumranet.com>
  13  *   Yaniv Kamay  <yaniv@qumranet.com>
  14  */
  15
  16 #include <linux/highmem.h>
  17 #include <linux/hrtimer.h>
  18 #include <linux/kernel.h>
  19 #include <linux/kvm_host.h>
  20 #include <linux/module.h>
  21 #include <linux/moduleparam.h>
  22 #include <linux/mod_devicetable.h>
  23 #include <linux/mm.h>
  24 #include <linux/objtool.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/smt.h>
  27 #include <linux/slab.h>
  28 #include <linux/tboot.h>
  29 #include <linux/trace_events.h>
  30 #include <linux/entry-kvm.h>
  31
  32 #include <asm/apic.h>
  33 #include <asm/asm.h>
  34 #include <asm/cpu.h>
  35 #include <asm/cpu_device_id.h>
  36 #include <asm/debugreg.h>
  37 #include <asm/desc.h>
  38 #include <asm/fpu/api.h>
  39 #include <asm/fpu/xstate.h>
  40 #include <asm/idtentry.h>
  41 #include <asm/io.h>
  42 #include <asm/irq_remapping.h>
  43 #include <asm/kexec.h>
  44 #include <asm/perf_event.h>
  45 #include <asm/mmu_context.h>
  46 #include <asm/mshyperv.h>
  47 #include <asm/mwait.h>
  48 #include <asm/spec-ctrl.h>
  49 #include <asm/virtext.h>
  50 #include <asm/vmx.h>
  51
  52 #include "capabilities.h"
  53 #include "cpuid.h"
  54 #include "evmcs.h"
  55 #include "hyperv.h"
  56 #include "kvm_onhyperv.h"
  57 #include "irq.h"
  58 #include "kvm_cache_regs.h"
  59 #include "lapic.h"
  60 #include "mmu.h"
  61 #include "nested.h"
  62 #include "pmu.h"
  63 #include "sgx.h"
  64 #include "trace.h"
  65 #include "vmcs.h"
  66 #include "vmcs12.h"
  67 #include "vmx.h"
  68 #include "x86.h"
  69
  70 MODULE_AUTHOR("Qumranet");
  71 MODULE_LICENSE("GPL");
  72
  73 #ifdef MODULE
  74 static const struct x86_cpu_id vmx_cpu_id[] = {
  75         X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL),
  76         {}
  77 };
  78 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
  79 #endif
  80
  81 bool __read_mostly enable_vpid = 1;
  82 module_param_named(vpid, enable_vpid, bool, 0444);
  83
  84 static bool __read_mostly enable_vnmi = 1;
  85 module_param_named(vnmi, enable_vnmi, bool, S_IRUGO);
  86
  87 bool __read_mostly flexpriority_enabled = 1;
  88 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
  89
  90 bool __read_mostly enable_ept = 1;
  91 module_param_named(ept, enable_ept, bool, S_IRUGO);
  92
  93 bool __read_mostly enable_unrestricted_guest = 1;
  94 module_param_named(unrestricted_guest,
  95                         enable_unrestricted_guest, bool, S_IRUGO);
  96
  97 bool __read_mostly enable_ept_ad_bits = 1;
  98 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
  99
 100 static bool __read_mostly emulate_invalid_guest_state = true;
 101 module_param(emulate_invalid_guest_state, bool, S_IRUGO);
 102
 103 static bool __read_mostly fasteoi = 1;
 104 module_param(fasteoi, bool, S_IRUGO);
 105
 106 module_param(enable_apicv, bool, S_IRUGO);
 107
 108 /*
 109  * If nested=1, nested virtualization is supported, i.e., guests may use
 110  * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 111  * use VMX instructions.
 112  */
 113 static bool __read_mostly nested = 1;
 114 module_param(nested, bool, S_IRUGO);
 115
 116 bool __read_mostly enable_pml = 1;
 117 module_param_named(pml, enable_pml, bool, S_IRUGO);
 118
 119 static bool __read_mostly dump_invalid_vmcs = 0;
 120 module_param(dump_invalid_vmcs, bool, 0644);
 121
 122 #define MSR_BITMAP_MODE_X2APIC          1
 123 #define MSR_BITMAP_MODE_X2APIC_APICV    2
 124
 125 #define KVM_VMX_TSC_MULTIPLIER_MAX     0xffffffffffffffffULL
 126
 127 /* Guest_tsc -> host_tsc conversion requires 64-bit division.  */
 128 static int __read_mostly cpu_preemption_timer_multi;
 129 static bool __read_mostly enable_preemption_timer = 1;
 130 #ifdef CONFIG_X86_64
 131 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
 132 #endif
 133
 134 extern bool __read_mostly allow_smaller_maxphyaddr;
 135 module_param(allow_smaller_maxphyaddr, bool, S_IRUGO);
 136
 137 #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD)
 138 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE
 139 #define KVM_VM_CR0_ALWAYS_ON                            \
 140         (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
 141
 142 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE
 143 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
 144 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
 145
 146 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
 147
 148 #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \
 149         RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \
 150         RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \
 151         RTIT_STATUS_BYTECNT))
 152
 153 /*
 154  * List of MSRs that can be directly passed to the guest.
 155  * In addition to these x2apic and PT MSRs are handled specially.
 156  */
 157 static u32 vmx_possible_passthrough_msrs[MAX_POSSIBLE_PASSTHROUGH_MSRS] = {
 158         MSR_IA32_SPEC_CTRL,
 159         MSR_IA32_PRED_CMD,
 160         MSR_IA32_TSC,
 161 #ifdef CONFIG_X86_64
 162         MSR_FS_BASE,
 163         MSR_GS_BASE,
 164         MSR_KERNEL_GS_BASE,
 165         MSR_IA32_XFD,
 166         MSR_IA32_XFD_ERR,
 167 #endif
 168         MSR_IA32_SYSENTER_CS,
 169         MSR_IA32_SYSENTER_ESP,
 170         MSR_IA32_SYSENTER_EIP,
 171         MSR_CORE_C1_RES,
 172         MSR_CORE_C3_RESIDENCY,
 173         MSR_CORE_C6_RESIDENCY,
 174         MSR_CORE_C7_RESIDENCY,
 175 };
 176
 177 /*
 178  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 179  * ple_gap:    upper bound on the amount of time between two successive
 180  *             executions of PAUSE in a loop. Also indicate if ple enabled.
 181  *             According to test, this time is usually smaller than 128 cycles.
 182  * ple_window: upper bound on the amount of time a guest is allowed to execute
 183  *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 184  *             less than 2^12 cycles
 185  * Time is measured based on a counter that runs at the same rate as the TSC,
 186  * refer SDM volume 3b section 21.6.13 & 22.1.3.
 187  */
 188 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP;
 189 module_param(ple_gap, uint, 0444);
 190
 191 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
 192 module_param(ple_window, uint, 0444);
 193
 194 /* Default doubles per-vcpu window every exit. */
 195 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 196 module_param(ple_window_grow, uint, 0444);
 197
 198 /* Default resets per-vcpu window every exit to ple_window. */
 199 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 200 module_param(ple_window_shrink, uint, 0444);
 201
 202 /* Default is to compute the maximum so we can never overflow. */
 203 static unsigned int ple_window_max        = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
 204 module_param(ple_window_max, uint, 0444);
 205
 206 /* Default is SYSTEM mode, 1 for host-guest mode */
 207 int __read_mostly pt_mode = PT_MODE_SYSTEM;
 208 module_param(pt_mode, int, S_IRUGO);
 209
 210 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush);
 211 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond);
 212 static DEFINE_MUTEX(vmx_l1d_flush_mutex);
 213
 214 /* Storage for pre module init parameter parsing */
 215 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO;
 216
 217 static const struct {
 218         const char *option;
 219         bool for_parse;
 220 } vmentry_l1d_param[] = {
 221         [VMENTER_L1D_FLUSH_AUTO]         = {"auto", true},
 222         [VMENTER_L1D_FLUSH_NEVER]        = {"never", true},
 223         [VMENTER_L1D_FLUSH_COND]         = {"cond", true},
 224         [VMENTER_L1D_FLUSH_ALWAYS]       = {"always", true},
 225         [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false},
 226         [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false},
 227 };
 228
 229 #define L1D_CACHE_ORDER 4
 230 static void *vmx_l1d_flush_pages;
 231
 232 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)
 233 {
 234         struct page *page;
 235         unsigned int i;
 236
 237         if (!boot_cpu_has_bug(X86_BUG_L1TF)) {
 238                 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
 239                 return 0;
 240         }
 241
 242         if (!enable_ept) {
 243                 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED;
 244                 return 0;
 245         }
 246
 247         if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) {
 248                 u64 msr;
 249
 250                 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr);
 251                 if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) {
 252                         l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
 253                         return 0;
 254                 }
 255         }
 256
 257         /* If set to auto use the default l1tf mitigation method */
 258         if (l1tf == VMENTER_L1D_FLUSH_AUTO) {
 259                 switch (l1tf_mitigation) {
 260                 case L1TF_MITIGATION_OFF:
 261                         l1tf = VMENTER_L1D_FLUSH_NEVER;
 262                         break;
 263                 case L1TF_MITIGATION_FLUSH_NOWARN:
 264                 case L1TF_MITIGATION_FLUSH:
 265                 case L1TF_MITIGATION_FLUSH_NOSMT:
 266                         l1tf = VMENTER_L1D_FLUSH_COND;
 267                         break;
 268                 case L1TF_MITIGATION_FULL:
 269                 case L1TF_MITIGATION_FULL_FORCE:
 270                         l1tf = VMENTER_L1D_FLUSH_ALWAYS;
 271                         break;
 272                 }
 273         } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) {
 274                 l1tf = VMENTER_L1D_FLUSH_ALWAYS;
 275         }
 276
 277         if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages &&
 278             !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) {
 279                 /*
 280                  * This allocation for vmx_l1d_flush_pages is not tied to a VM
 281                  * lifetime and so should not be charged to a memcg.
 282                  */
 283                 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER);
 284                 if (!page)
 285                         return -ENOMEM;
 286                 vmx_l1d_flush_pages = page_address(page);
 287
 288                 /*
 289                  * Initialize each page with a different pattern in
 290                  * order to protect against KSM in the nested
 291                  * virtualization case.
 292                  */
 293                 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) {
 294                         memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1,
 295                                PAGE_SIZE);
 296                 }
 297         }
 298
 299         l1tf_vmx_mitigation = l1tf;
 300
 301         if (l1tf != VMENTER_L1D_FLUSH_NEVER)
 302                 static_branch_enable(&vmx_l1d_should_flush);
 303         else
 304                 static_branch_disable(&vmx_l1d_should_flush);
 305
 306         if (l1tf == VMENTER_L1D_FLUSH_COND)
 307                 static_branch_enable(&vmx_l1d_flush_cond);
 308         else
 309                 static_branch_disable(&vmx_l1d_flush_cond);
 310         return 0;
 311 }
 312
 313 static int vmentry_l1d_flush_parse(const char *s)
 314 {
 315         unsigned int i;
 316
 317         if (s) {
 318                 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) {
 319                         if (vmentry_l1d_param[i].for_parse &&
 320                             sysfs_streq(s, vmentry_l1d_param[i].option))
 321                                 return i;
 322                 }
 323         }
 324         return -EINVAL;
 325 }
 326
 327 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp)
 328 {
 329         int l1tf, ret;
 330
 331         l1tf = vmentry_l1d_flush_parse(s);
 332         if (l1tf < 0)
 333                 return l1tf;
 334
 335         if (!boot_cpu_has(X86_BUG_L1TF))
 336                 return 0;
 337
 338         /*
 339          * Has vmx_init() run already? If not then this is the pre init
 340          * parameter parsing. In that case just store the value and let
 341          * vmx_init() do the proper setup after enable_ept has been
 342          * established.
 343          */
 344         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) {
 345                 vmentry_l1d_flush_param = l1tf;
 346                 return 0;
 347         }
 348
 349         mutex_lock(&vmx_l1d_flush_mutex);
 350         ret = vmx_setup_l1d_flush(l1tf);
 351         mutex_unlock(&vmx_l1d_flush_mutex);
 352         return ret;
 353 }
 354
 355 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp)
 356 {
 357         if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param)))
 358                 return sprintf(s, "???\n");
 359
 360         return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option);
 361 }
 362
 363 static const struct kernel_param_ops vmentry_l1d_flush_ops = {
 364         .set = vmentry_l1d_flush_set,
 365         .get = vmentry_l1d_flush_get,
 366 };
 367 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644);
 368
 369 static u32 vmx_segment_access_rights(struct kvm_segment *var);
 370
 371 void vmx_vmexit(void);
 372
 373 #define vmx_insn_failed(fmt...)         \
 374 do {                                    \
 375         WARN_ONCE(1, fmt);              \
 376         pr_warn_ratelimited(fmt);       \
 377 } while (0)
 378
 379 asmlinkage void vmread_error(unsigned long field, bool fault)
 380 {
 381         if (fault)
 382                 kvm_spurious_fault();
 383         else
 384                 vmx_insn_failed("kvm: vmread failed: field=%lx\n", field);
 385 }
 386
 387 noinline void vmwrite_error(unsigned long field, unsigned long value)
 388 {
 389         vmx_insn_failed("kvm: vmwrite failed: field=%lx val=%lx err=%d\n",
 390                         field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
 391 }
 392
 393 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr)
 394 {
 395         vmx_insn_failed("kvm: vmclear failed: %p/%llx\n", vmcs, phys_addr);
 396 }
 397
 398 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr)
 399 {
 400         vmx_insn_failed("kvm: vmptrld failed: %p/%llx\n", vmcs, phys_addr);
 401 }
 402
 403 noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva)
 404 {
 405         vmx_insn_failed("kvm: invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n",
 406                         ext, vpid, gva);
 407 }
 408
 409 noinline void invept_error(unsigned long ext, u64 eptp, gpa_t gpa)
 410 {
 411         vmx_insn_failed("kvm: invept failed: ext=0x%lx eptp=%llx gpa=0x%llx\n",
 412                         ext, eptp, gpa);
 413 }
 414
 415 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 416 DEFINE_PER_CPU(struct vmcs *, current_vmcs);
 417 /*
 418  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
 419  * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
 420  */
 421 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
 422
 423 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
 424 static DEFINE_SPINLOCK(vmx_vpid_lock);
 425
 426 struct vmcs_config vmcs_config;
 427 struct vmx_capability vmx_capability;
 428
 429 #define VMX_SEGMENT_FIELD(seg)                                  \
 430         [VCPU_SREG_##seg] = {                                   \
 431                 .selector = GUEST_##seg##_SELECTOR,             \
 432                 .base = GUEST_##seg##_BASE,                     \
 433                 .limit = GUEST_##seg##_LIMIT,                   \
 434                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
 435         }
 436
 437 static const struct kvm_vmx_segment_field {
 438         unsigned selector;
 439         unsigned base;
 440         unsigned limit;
 441         unsigned ar_bytes;
 442 } kvm_vmx_segment_fields[] = {
 443         VMX_SEGMENT_FIELD(CS),
 444         VMX_SEGMENT_FIELD(DS),
 445         VMX_SEGMENT_FIELD(ES),
 446         VMX_SEGMENT_FIELD(FS),
 447         VMX_SEGMENT_FIELD(GS),
 448         VMX_SEGMENT_FIELD(SS),
 449         VMX_SEGMENT_FIELD(TR),
 450         VMX_SEGMENT_FIELD(LDTR),
 451 };
 452
 453 static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
 454 {
 455         vmx->segment_cache.bitmask = 0;
 456 }
 457
 458 static unsigned long host_idt_base;
 459
 460 #if IS_ENABLED(CONFIG_HYPERV)
 461 static bool __read_mostly enlightened_vmcs = true;
 462 module_param(enlightened_vmcs, bool, 0444);
 463
 464 static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu)
 465 {
 466         struct hv_enlightened_vmcs *evmcs;
 467         struct hv_partition_assist_pg **p_hv_pa_pg =
 468                         &to_kvm_hv(vcpu->kvm)->hv_pa_pg;
 469         /*
 470          * Synthetic VM-Exit is not enabled in current code and so All
 471          * evmcs in singe VM shares same assist page.
 472          */
 473         if (!*p_hv_pa_pg)
 474                 *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT);
 475
 476         if (!*p_hv_pa_pg)
 477                 return -ENOMEM;
 478
 479         evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs;
 480
 481         evmcs->partition_assist_page =
 482                 __pa(*p_hv_pa_pg);
 483         evmcs->hv_vm_id = (unsigned long)vcpu->kvm;
 484         evmcs->hv_enlightenments_control.nested_flush_hypercall = 1;
 485
 486         return 0;
 487 }
 488
 489 #endif /* IS_ENABLED(CONFIG_HYPERV) */
 490
 491 /*
 492  * Comment's format: document - errata name - stepping - processor name.
 493  * Refer from
 494  * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp
 495  */
 496 static u32 vmx_preemption_cpu_tfms[] = {
 497 /* 323344.pdf - BA86   - D0 - Xeon 7500 Series */
 498 0x000206E6,
 499 /* 323056.pdf - AAX65  - C2 - Xeon L3406 */
 500 /* 322814.pdf - AAT59  - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
 501 /* 322911.pdf - AAU65  - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
 502 0x00020652,
 503 /* 322911.pdf - AAU65  - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
 504 0x00020655,
 505 /* 322373.pdf - AAO95  - B1 - Xeon 3400 Series */
 506 /* 322166.pdf - AAN92  - B1 - i7-800 and i5-700 Desktop */
 507 /*
 508  * 320767.pdf - AAP86  - B1 -
 509  * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
 510  */
 511 0x000106E5,
 512 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */
 513 0x000106A0,
 514 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */
 515 0x000106A1,
 516 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
 517 0x000106A4,
 518  /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
 519  /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
 520  /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
 521 0x000106A5,
 522  /* Xeon E3-1220 V2 */
 523 0x000306A8,
 524 };
 525
 526 static inline bool cpu_has_broken_vmx_preemption_timer(void)
 527 {
 528         u32 eax = cpuid_eax(0x00000001), i;
 529
 530         /* Clear the reserved bits */
 531         eax &= ~(0x3U << 14 | 0xfU << 28);
 532         for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++)
 533                 if (eax == vmx_preemption_cpu_tfms[i])
 534                         return true;
 535
 536         return false;
 537 }
 538
 539 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
 540 {
 541         return flexpriority_enabled && lapic_in_kernel(vcpu);
 542 }
 543
 544 static int possible_passthrough_msr_slot(u32 msr)
 545 {
 546         u32 i;
 547
 548         for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++)
 549                 if (vmx_possible_passthrough_msrs[i] == msr)
 550                         return i;
 551
 552         return -ENOENT;
 553 }
 554
 555 static bool is_valid_passthrough_msr(u32 msr)
 556 {
 557         bool r;
 558
 559         switch (msr) {
 560         case 0x800 ... 0x8ff:
 561                 /* x2APIC MSRs. These are handled in vmx_update_msr_bitmap_x2apic() */
 562                 return true;
 563         case MSR_IA32_RTIT_STATUS:
 564         case MSR_IA32_RTIT_OUTPUT_BASE:
 565         case MSR_IA32_RTIT_OUTPUT_MASK:
 566         case MSR_IA32_RTIT_CR3_MATCH:
 567         case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
 568                 /* PT MSRs. These are handled in pt_update_intercept_for_msr() */
 569         case MSR_LBR_SELECT:
 570         case MSR_LBR_TOS:
 571         case MSR_LBR_INFO_0 ... MSR_LBR_INFO_0 + 31:
 572         case MSR_LBR_NHM_FROM ... MSR_LBR_NHM_FROM + 31:
 573         case MSR_LBR_NHM_TO ... MSR_LBR_NHM_TO + 31:
 574         case MSR_LBR_CORE_FROM ... MSR_LBR_CORE_FROM + 8:
 575         case MSR_LBR_CORE_TO ... MSR_LBR_CORE_TO + 8:
 576                 /* LBR MSRs. These are handled in vmx_update_intercept_for_lbr_msrs() */
 577                 return true;
 578         }
 579
 580         r = possible_passthrough_msr_slot(msr) != -ENOENT;
 581
 582         WARN(!r, "Invalid MSR %x, please adapt vmx_possible_passthrough_msrs[]", msr);
 583
 584         return r;
 585 }
 586
 587 struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr)
 588 {
 589         int i;
 590
 591         i = kvm_find_user_return_msr(msr);
 592         if (i >= 0)
 593                 return &vmx->guest_uret_msrs[i];
 594         return NULL;
 595 }
 596
 597 static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx,
 598                                   struct vmx_uret_msr *msr, u64 data)
 599 {
 600         unsigned int slot = msr - vmx->guest_uret_msrs;
 601         int ret = 0;
 602
 603         if (msr->load_into_hardware) {
 604                 preempt_disable();
 605                 ret = kvm_set_user_return_msr(slot, data, msr->mask);
 606                 preempt_enable();
 607         }
 608         if (!ret)
 609                 msr->data = data;
 610         return ret;
 611 }
 612
 613 #ifdef CONFIG_KEXEC_CORE
 614 static void crash_vmclear_local_loaded_vmcss(void)
 615 {
 616         int cpu = raw_smp_processor_id();
 617         struct loaded_vmcs *v;
 618
 619         list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
 620                             loaded_vmcss_on_cpu_link)
 621                 vmcs_clear(v->vmcs);
 622 }
 623 #endif /* CONFIG_KEXEC_CORE */
 624
 625 static void __loaded_vmcs_clear(void *arg)
 626 {
 627         struct loaded_vmcs *loaded_vmcs = arg;
 628         int cpu = raw_smp_processor_id();
 629
 630         if (loaded_vmcs->cpu != cpu)
 631                 return; /* vcpu migration can race with cpu offline */
 632         if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
 633                 per_cpu(current_vmcs, cpu) = NULL;
 634
 635         vmcs_clear(loaded_vmcs->vmcs);
 636         if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
 637                 vmcs_clear(loaded_vmcs->shadow_vmcs);
 638
 639         list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
 640
 641         /*
 642          * Ensure all writes to loaded_vmcs, including deleting it from its
 643          * current percpu list, complete before setting loaded_vmcs->cpu to
 644          * -1, otherwise a different cpu can see loaded_vmcs->cpu == -1 first
 645          * and add loaded_vmcs to its percpu list before it's deleted from this
 646          * cpu's list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs().
 647          */
 648         smp_wmb();
 649
 650         loaded_vmcs->cpu = -1;
 651         loaded_vmcs->launched = 0;
 652 }
 653
 654 void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
 655 {
 656         int cpu = loaded_vmcs->cpu;
 657
 658         if (cpu != -1)
 659                 smp_call_function_single(cpu,
 660                          __loaded_vmcs_clear, loaded_vmcs, 1);
 661 }
 662
 663 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
 664                                        unsigned field)
 665 {
 666         bool ret;
 667         u32 mask = 1 << (seg * SEG_FIELD_NR + field);
 668
 669         if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) {
 670                 kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS);
 671                 vmx->segment_cache.bitmask = 0;
 672         }
 673         ret = vmx->segment_cache.bitmask & mask;
 674         vmx->segment_cache.bitmask |= mask;
 675         return ret;
 676 }
 677
 678 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
 679 {
 680         u16 *p = &vmx->segment_cache.seg[seg].selector;
 681
 682         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
 683                 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
 684         return *p;
 685 }
 686
 687 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
 688 {
 689         ulong *p = &vmx->segment_cache.seg[seg].base;
 690
 691         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
 692                 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
 693         return *p;
 694 }
 695
 696 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
 697 {
 698         u32 *p = &vmx->segment_cache.seg[seg].limit;
 699
 700         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
 701                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
 702         return *p;
 703 }
 704
 705 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
 706 {
 707         u32 *p = &vmx->segment_cache.seg[seg].ar;
 708
 709         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
 710                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
 711         return *p;
 712 }
 713
 714 void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu)
 715 {
 716         u32 eb;
 717
 718         eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
 719              (1u << DB_VECTOR) | (1u << AC_VECTOR);
 720         /*
 721          * Guest access to VMware backdoor ports could legitimately
 722          * trigger #GP because of TSS I/O permission bitmap.
 723          * We intercept those #GP and allow access to them anyway
 724          * as VMware does.
 725          */
 726         if (enable_vmware_backdoor)
 727                 eb |= (1u << GP_VECTOR);
 728         if ((vcpu->guest_debug &
 729              (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
 730             (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
 731                 eb |= 1u << BP_VECTOR;
 732         if (to_vmx(vcpu)->rmode.vm86_active)
 733                 eb = ~0;
 734         if (!vmx_need_pf_intercept(vcpu))
 735                 eb &= ~(1u << PF_VECTOR);
 736
 737         /* When we are running a nested L2 guest and L1 specified for it a
 738          * certain exception bitmap, we must trap the same exceptions and pass
 739          * them to L1. When running L2, we will only handle the exceptions
 740          * specified above if L1 did not want them.
 741          */
 742         if (is_guest_mode(vcpu))
 743                 eb |= get_vmcs12(vcpu)->exception_bitmap;
 744         else {
 745                 int mask = 0, match = 0;
 746
 747                 if (enable_ept && (eb & (1u << PF_VECTOR))) {
 748                         /*
 749                          * If EPT is enabled, #PF is currently only intercepted
 750                          * if MAXPHYADDR is smaller on the guest than on the
 751                          * host.  In that case we only care about present,
 752                          * non-reserved faults.  For vmcs02, however, PFEC_MASK
 753                          * and PFEC_MATCH are set in prepare_vmcs02_rare.
 754                          */
 755                         mask = PFERR_PRESENT_MASK | PFERR_RSVD_MASK;
 756                         match = PFERR_PRESENT_MASK;
 757                 }
 758                 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask);
 759                 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, match);
 760         }
 761
 762         /*
 763          * Disabling xfd interception indicates that dynamic xfeatures
 764          * might be used in the guest. Always trap #NM in this case
 765          * to save guest xfd_err timely.
 766          */
 767         if (vcpu->arch.xfd_no_write_intercept)
 768                 eb |= (1u << NM_VECTOR);
 769
 770         vmcs_write32(EXCEPTION_BITMAP, eb);
 771 }
 772
 773 /*
 774  * Check if MSR is intercepted for currently loaded MSR bitmap.
 775  */
 776 static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr)
 777 {
 778         if (!(exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS))
 779                 return true;
 780
 781         return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap,
 782                                          MSR_IA32_SPEC_CTRL);
 783 }
 784
 785 static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
 786                 unsigned long entry, unsigned long exit)
 787 {
 788         vm_entry_controls_clearbit(vmx, entry);
 789         vm_exit_controls_clearbit(vmx, exit);
 790 }
 791
 792 int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr)
 793 {
 794         unsigned int i;
 795
 796         for (i = 0; i < m->nr; ++i) {
 797                 if (m->val[i].index == msr)
 798                         return i;
 799         }
 800         return -ENOENT;
 801 }
 802
 803 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
 804 {
 805         int i;
 806         struct msr_autoload *m = &vmx->msr_autoload;
 807
 808         switch (msr) {
 809         case MSR_EFER:
 810                 if (cpu_has_load_ia32_efer()) {
 811                         clear_atomic_switch_msr_special(vmx,
 812                                         VM_ENTRY_LOAD_IA32_EFER,
 813                                         VM_EXIT_LOAD_IA32_EFER);
 814                         return;
 815                 }
 816                 break;
 817         case MSR_CORE_PERF_GLOBAL_CTRL:
 818                 if (cpu_has_load_perf_global_ctrl()) {
 819                         clear_atomic_switch_msr_special(vmx,
 820                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
 821                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
 822                         return;
 823                 }
 824                 break;
 825         }
 826         i = vmx_find_loadstore_msr_slot(&m->guest, msr);
 827         if (i < 0)
 828                 goto skip_guest;
 829         --m->guest.nr;
 830         m->guest.val[i] = m->guest.val[m->guest.nr];
 831         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
 832
 833 skip_guest:
 834         i = vmx_find_loadstore_msr_slot(&m->host, msr);
 835         if (i < 0)
 836                 return;
 837
 838         --m->host.nr;
 839         m->host.val[i] = m->host.val[m->host.nr];
 840         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
 841 }
 842
 843 static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
 844                 unsigned long entry, unsigned long exit,
 845                 unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
 846                 u64 guest_val, u64 host_val)
 847 {
 848         vmcs_write64(guest_val_vmcs, guest_val);
 849         if (host_val_vmcs != HOST_IA32_EFER)
 850                 vmcs_write64(host_val_vmcs, host_val);
 851         vm_entry_controls_setbit(vmx, entry);
 852         vm_exit_controls_setbit(vmx, exit);
 853 }
 854
 855 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
 856                                   u64 guest_val, u64 host_val, bool entry_only)
 857 {
 858         int i, j = 0;
 859         struct msr_autoload *m = &vmx->msr_autoload;
 860
 861         switch (msr) {
 862         case MSR_EFER:
 863                 if (cpu_has_load_ia32_efer()) {
 864                         add_atomic_switch_msr_special(vmx,
 865                                         VM_ENTRY_LOAD_IA32_EFER,
 866                                         VM_EXIT_LOAD_IA32_EFER,
 867                                         GUEST_IA32_EFER,
 868                                         HOST_IA32_EFER,
 869                                         guest_val, host_val);
 870                         return;
 871                 }
 872                 break;
 873         case MSR_CORE_PERF_GLOBAL_CTRL:
 874                 if (cpu_has_load_perf_global_ctrl()) {
 875                         add_atomic_switch_msr_special(vmx,
 876                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
 877                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
 878                                         GUEST_IA32_PERF_GLOBAL_CTRL,
 879                                         HOST_IA32_PERF_GLOBAL_CTRL,
 880                                         guest_val, host_val);
 881                         return;
 882                 }
 883                 break;
 884         case MSR_IA32_PEBS_ENABLE:
 885                 /* PEBS needs a quiescent period after being disabled (to write
 886                  * a record).  Disabling PEBS through VMX MSR swapping doesn't
 887                  * provide that period, so a CPU could write host's record into
 888                  * guest's memory.
 889                  */
 890                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
 891         }
 892
 893         i = vmx_find_loadstore_msr_slot(&m->guest, msr);
 894         if (!entry_only)
 895                 j = vmx_find_loadstore_msr_slot(&m->host, msr);
 896
 897         if ((i < 0 && m->guest.nr == MAX_NR_LOADSTORE_MSRS) ||
 898             (j < 0 &&  m->host.nr == MAX_NR_LOADSTORE_MSRS)) {
 899                 printk_once(KERN_WARNING "Not enough msr switch entries. "
 900                                 "Can't add msr %x\n", msr);
 901                 return;
 902         }
 903         if (i < 0) {
 904                 i = m->guest.nr++;
 905                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
 906         }
 907         m->guest.val[i].index = msr;
 908         m->guest.val[i].value = guest_val;
 909
 910         if (entry_only)
 911                 return;
 912
 913         if (j < 0) {
 914                 j = m->host.nr++;
 915                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
 916         }
 917         m->host.val[j].index = msr;
 918         m->host.val[j].value = host_val;
 919 }
 920
 921 static bool update_transition_efer(struct vcpu_vmx *vmx)
 922 {
 923         u64 guest_efer = vmx->vcpu.arch.efer;
 924         u64 ignore_bits = 0;
 925         int i;
 926
 927         /* Shadow paging assumes NX to be available.  */
 928         if (!enable_ept)
 929                 guest_efer |= EFER_NX;
 930
 931         /*
 932          * LMA and LME handled by hardware; SCE meaningless outside long mode.
 933          */
 934         ignore_bits |= EFER_SCE;
 935 #ifdef CONFIG_X86_64
 936         ignore_bits |= EFER_LMA | EFER_LME;
 937         /* SCE is meaningful only in long mode on Intel */
 938         if (guest_efer & EFER_LMA)
 939                 ignore_bits &= ~(u64)EFER_SCE;
 940 #endif
 941
 942         /*
 943          * On EPT, we can't emulate NX, so we must switch EFER atomically.
 944          * On CPUs that support "load IA32_EFER", always switch EFER
 945          * atomically, since it's faster than switching it manually.
 946          */
 947         if (cpu_has_load_ia32_efer() ||
 948             (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
 949                 if (!(guest_efer & EFER_LMA))
 950                         guest_efer &= ~EFER_LME;
 951                 if (guest_efer != host_efer)
 952                         add_atomic_switch_msr(vmx, MSR_EFER,
 953                                               guest_efer, host_efer, false);
 954                 else
 955                         clear_atomic_switch_msr(vmx, MSR_EFER);
 956                 return false;
 957         }
 958
 959         i = kvm_find_user_return_msr(MSR_EFER);
 960         if (i < 0)
 961                 return false;
 962
 963         clear_atomic_switch_msr(vmx, MSR_EFER);
 964
 965         guest_efer &= ~ignore_bits;
 966         guest_efer |= host_efer & ignore_bits;
 967
 968         vmx->guest_uret_msrs[i].data = guest_efer;
 969         vmx->guest_uret_msrs[i].mask = ~ignore_bits;
 970
 971         return true;
 972 }
 973
 974 #ifdef CONFIG_X86_32
 975 /*
 976  * On 32-bit kernels, VM exits still load the FS and GS bases from the
 977  * VMCS rather than the segment table.  KVM uses this helper to figure
 978  * out the current bases to poke them into the VMCS before entry.
 979  */
 980 static unsigned long segment_base(u16 selector)
 981 {
 982         struct desc_struct *table;
 983         unsigned long v;
 984
 985         if (!(selector & ~SEGMENT_RPL_MASK))
 986                 return 0;
 987
 988         table = get_current_gdt_ro();
 989
 990         if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) {
 991                 u16 ldt_selector = kvm_read_ldt();
 992
 993                 if (!(ldt_selector & ~SEGMENT_RPL_MASK))
 994                         return 0;
 995
 996                 table = (struct desc_struct *)segment_base(ldt_selector);
 997         }
 998         v = get_desc_base(&table[selector >> 3]);
 999         return v;
1000 }
1001 #endif
1002
1003 static inline bool pt_can_write_msr(struct vcpu_vmx *vmx)
1004 {
1005         return vmx_pt_mode_is_host_guest() &&
1006                !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN);
1007 }
1008
1009 static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base)
1010 {
1011         /* The base must be 128-byte aligned and a legal physical address. */
1012         return kvm_vcpu_is_legal_aligned_gpa(vcpu, base, 128);
1013 }
1014
1015 static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range)
1016 {
1017         u32 i;
1018
1019         wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status);
1020         wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base);
1021         wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask);
1022         wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match);
1023         for (i = 0; i < addr_range; i++) {
1024                 wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]);
1025                 wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]);
1026         }
1027 }
1028
1029 static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range)
1030 {
1031         u32 i;
1032
1033         rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status);
1034         rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base);
1035         rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask);
1036         rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match);
1037         for (i = 0; i < addr_range; i++) {
1038                 rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]);
1039                 rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]);
1040         }
1041 }
1042
1043 static void pt_guest_enter(struct vcpu_vmx *vmx)
1044 {
1045         if (vmx_pt_mode_is_system())
1046                 return;
1047
1048         /*
1049          * GUEST_IA32_RTIT_CTL is already set in the VMCS.
1050          * Save host state before VM entry.
1051          */
1052         rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
1053         if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) {
1054                 wrmsrl(MSR_IA32_RTIT_CTL, 0);
1055                 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges);
1056                 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges);
1057         }
1058 }
1059
1060 static void pt_guest_exit(struct vcpu_vmx *vmx)
1061 {
1062         if (vmx_pt_mode_is_system())
1063                 return;
1064
1065         if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) {
1066                 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges);
1067                 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges);
1068         }
1069
1070         /*
1071          * KVM requires VM_EXIT_CLEAR_IA32_RTIT_CTL to expose PT to the guest,
1072          * i.e. RTIT_CTL is always cleared on VM-Exit.  Restore it if necessary.
1073          */
1074         if (vmx->pt_desc.host.ctl)
1075                 wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
1076 }
1077
1078 void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel,
1079                         unsigned long fs_base, unsigned long gs_base)
1080 {
1081         if (unlikely(fs_sel != host->fs_sel)) {
1082                 if (!(fs_sel & 7))
1083                         vmcs_write16(HOST_FS_SELECTOR, fs_sel);
1084                 else
1085                         vmcs_write16(HOST_FS_SELECTOR, 0);
1086                 host->fs_sel = fs_sel;
1087         }
1088         if (unlikely(gs_sel != host->gs_sel)) {
1089                 if (!(gs_sel & 7))
1090                         vmcs_write16(HOST_GS_SELECTOR, gs_sel);
1091                 else
1092                         vmcs_write16(HOST_GS_SELECTOR, 0);
1093                 host->gs_sel = gs_sel;
1094         }
1095         if (unlikely(fs_base != host->fs_base)) {
1096                 vmcs_writel(HOST_FS_BASE, fs_base);
1097                 host->fs_base = fs_base;
1098         }
1099         if (unlikely(gs_base != host->gs_base)) {
1100                 vmcs_writel(HOST_GS_BASE, gs_base);
1101                 host->gs_base = gs_base;
1102         }
1103 }
1104
1105 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1106 {
1107         struct vcpu_vmx *vmx = to_vmx(vcpu);
1108         struct vmcs_host_state *host_state;
1109 #ifdef CONFIG_X86_64
1110         int cpu = raw_smp_processor_id();
1111 #endif
1112         unsigned long fs_base, gs_base;
1113         u16 fs_sel, gs_sel;
1114         int i;
1115
1116         vmx->req_immediate_exit = false;
1117
1118         /*
1119          * Note that guest MSRs to be saved/restored can also be changed
1120          * when guest state is loaded. This happens when guest transitions
1121          * to/from long-mode by setting MSR_EFER.LMA.
1122          */
1123         if (!vmx->guest_uret_msrs_loaded) {
1124                 vmx->guest_uret_msrs_loaded = true;
1125                 for (i = 0; i < kvm_nr_uret_msrs; ++i) {
1126                         if (!vmx->guest_uret_msrs[i].load_into_hardware)
1127                                 continue;
1128
1129                         kvm_set_user_return_msr(i,
1130                                                 vmx->guest_uret_msrs[i].data,
1131                                                 vmx->guest_uret_msrs[i].mask);
1132                 }
1133         }
1134
1135         if (vmx->nested.need_vmcs12_to_shadow_sync)
1136                 nested_sync_vmcs12_to_shadow(vcpu);
1137
1138         if (vmx->guest_state_loaded)
1139                 return;
1140
1141         host_state = &vmx->loaded_vmcs->host_state;
1142
1143         /*
1144          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
1145          * allow segment selectors with cpl > 0 or ti == 1.
1146          */
1147         host_state->ldt_sel = kvm_read_ldt();
1148
1149 #ifdef CONFIG_X86_64
1150         savesegment(ds, host_state->ds_sel);
1151         savesegment(es, host_state->es_sel);
1152
1153         gs_base = cpu_kernelmode_gs_base(cpu);
1154         if (likely(is_64bit_mm(current->mm))) {
1155                 current_save_fsgs();
1156                 fs_sel = current->thread.fsindex;
1157                 gs_sel = current->thread.gsindex;
1158                 fs_base = current->thread.fsbase;
1159                 vmx->msr_host_kernel_gs_base = current->thread.gsbase;
1160         } else {
1161                 savesegment(fs, fs_sel);
1162                 savesegment(gs, gs_sel);
1163                 fs_base = read_msr(MSR_FS_BASE);
1164                 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
1165         }
1166
1167         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1168 #else
1169         savesegment(fs, fs_sel);
1170         savesegment(gs, gs_sel);
1171         fs_base = segment_base(fs_sel);
1172         gs_base = segment_base(gs_sel);
1173 #endif
1174
1175         vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base);
1176         vmx->guest_state_loaded = true;
1177 }
1178
1179 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx)
1180 {
1181         struct vmcs_host_state *host_state;
1182
1183         if (!vmx->guest_state_loaded)
1184                 return;
1185
1186         host_state = &vmx->loaded_vmcs->host_state;
1187
1188         ++vmx->vcpu.stat.host_state_reload;
1189
1190 #ifdef CONFIG_X86_64
1191         rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1192 #endif
1193         if (host_state->ldt_sel || (host_state->gs_sel & 7)) {
1194                 kvm_load_ldt(host_state->ldt_sel);
1195 #ifdef CONFIG_X86_64
1196                 load_gs_index(host_state->gs_sel);
1197 #else
1198                 loadsegment(gs, host_state->gs_sel);
1199 #endif
1200         }
1201         if (host_state->fs_sel & 7)
1202                 loadsegment(fs, host_state->fs_sel);
1203 #ifdef CONFIG_X86_64
1204         if (unlikely(host_state->ds_sel | host_state->es_sel)) {
1205                 loadsegment(ds, host_state->ds_sel);
1206                 loadsegment(es, host_state->es_sel);
1207         }
1208 #endif
1209         invalidate_tss_limit();
1210 #ifdef CONFIG_X86_64
1211         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1212 #endif
1213         load_fixmap_gdt(raw_smp_processor_id());
1214         vmx->guest_state_loaded = false;
1215         vmx->guest_uret_msrs_loaded = false;
1216 }
1217
1218 #ifdef CONFIG_X86_64
1219 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx)
1220 {
1221         preempt_disable();
1222         if (vmx->guest_state_loaded)
1223                 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1224         preempt_enable();
1225         return vmx->msr_guest_kernel_gs_base;
1226 }
1227
1228 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data)
1229 {
1230         preempt_disable();
1231         if (vmx->guest_state_loaded)
1232                 wrmsrl(MSR_KERNEL_GS_BASE, data);
1233         preempt_enable();
1234         vmx->msr_guest_kernel_gs_base = data;
1235 }
1236 #endif
1237
1238 void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu,
1239                         struct loaded_vmcs *buddy)
1240 {
1241         struct vcpu_vmx *vmx = to_vmx(vcpu);
1242         bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
1243         struct vmcs *prev;
1244
1245         if (!already_loaded) {
1246                 loaded_vmcs_clear(vmx->loaded_vmcs);
1247                 local_irq_disable();
1248
1249                 /*
1250                  * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to
1251                  * this cpu's percpu list, otherwise it may not yet be deleted
1252                  * from its previous cpu's percpu list.  Pairs with the
1253                  * smb_wmb() in __loaded_vmcs_clear().
1254                  */
1255                 smp_rmb();
1256
1257                 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
1258                          &per_cpu(loaded_vmcss_on_cpu, cpu));
1259                 local_irq_enable();
1260         }
1261
1262         prev = per_cpu(current_vmcs, cpu);
1263         if (prev != vmx->loaded_vmcs->vmcs) {
1264                 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
1265                 vmcs_load(vmx->loaded_vmcs->vmcs);
1266
1267                 /*
1268                  * No indirect branch prediction barrier needed when switching
1269                  * the active VMCS within a guest, e.g. on nested VM-Enter.
1270                  * The L1 VMM can protect itself with retpolines, IBPB or IBRS.
1271                  */
1272                 if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev))
1273                         indirect_branch_prediction_barrier();
1274         }
1275
1276         if (!already_loaded) {
1277                 void *gdt = get_current_gdt_ro();
1278
1279                 /*
1280                  * Flush all EPTP/VPID contexts, the new pCPU may have stale
1281                  * TLB entries from its previous association with the vCPU.
1282                  */
1283                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1284
1285                 /*
1286                  * Linux uses per-cpu TSS and GDT, so set these when switching
1287                  * processors.  See 22.2.4.
1288                  */
1289                 vmcs_writel(HOST_TR_BASE,
1290                             (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss);
1291                 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt);   /* 22.2.4 */
1292
1293                 if (IS_ENABLED(CONFIG_IA32_EMULATION) || IS_ENABLED(CONFIG_X86_32)) {
1294                         /* 22.2.3 */
1295                         vmcs_writel(HOST_IA32_SYSENTER_ESP,
1296                                     (unsigned long)(cpu_entry_stack(cpu) + 1));
1297                 }
1298
1299                 vmx->loaded_vmcs->cpu = cpu;
1300         }
1301 }
1302
1303 /*
1304  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
1305  * vcpu mutex is already taken.
1306  */
1307 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1308 {
1309         struct vcpu_vmx *vmx = to_vmx(vcpu);
1310
1311         vmx_vcpu_load_vmcs(vcpu, cpu, NULL);
1312
1313         vmx_vcpu_pi_load(vcpu, cpu);
1314
1315         vmx->host_debugctlmsr = get_debugctlmsr();
1316 }
1317
1318 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
1319 {
1320         vmx_vcpu_pi_put(vcpu);
1321
1322         vmx_prepare_switch_to_host(to_vmx(vcpu));
1323 }
1324
1325 bool vmx_emulation_required(struct kvm_vcpu *vcpu)
1326 {
1327         return emulate_invalid_guest_state && !vmx_guest_state_valid(vcpu);
1328 }
1329
1330 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
1331 {
1332         struct vcpu_vmx *vmx = to_vmx(vcpu);
1333         unsigned long rflags, save_rflags;
1334
1335         if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) {
1336                 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS);
1337                 rflags = vmcs_readl(GUEST_RFLAGS);
1338                 if (vmx->rmode.vm86_active) {
1339                         rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
1340                         save_rflags = vmx->rmode.save_rflags;
1341                         rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
1342                 }
1343                 vmx->rflags = rflags;
1344         }
1345         return vmx->rflags;
1346 }
1347
1348 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1349 {
1350         struct vcpu_vmx *vmx = to_vmx(vcpu);
1351         unsigned long old_rflags;
1352
1353         if (is_unrestricted_guest(vcpu)) {
1354                 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS);
1355                 vmx->rflags = rflags;
1356                 vmcs_writel(GUEST_RFLAGS, rflags);
1357                 return;
1358         }
1359
1360         old_rflags = vmx_get_rflags(vcpu);
1361         vmx->rflags = rflags;
1362         if (vmx->rmode.vm86_active) {
1363                 vmx->rmode.save_rflags = rflags;
1364                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
1365         }
1366         vmcs_writel(GUEST_RFLAGS, rflags);
1367
1368         if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM)
1369                 vmx->emulation_required = vmx_emulation_required(vcpu);
1370 }
1371
1372 static bool vmx_get_if_flag(struct kvm_vcpu *vcpu)
1373 {
1374         return vmx_get_rflags(vcpu) & X86_EFLAGS_IF;
1375 }
1376
1377 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
1378 {
1379         u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
1380         int ret = 0;
1381
1382         if (interruptibility & GUEST_INTR_STATE_STI)
1383                 ret |= KVM_X86_SHADOW_INT_STI;
1384         if (interruptibility & GUEST_INTR_STATE_MOV_SS)
1385                 ret |= KVM_X86_SHADOW_INT_MOV_SS;
1386
1387         return ret;
1388 }
1389
1390 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
1391 {
1392         u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
1393         u32 interruptibility = interruptibility_old;
1394
1395         interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
1396
1397         if (mask & KVM_X86_SHADOW_INT_MOV_SS)
1398                 interruptibility |= GUEST_INTR_STATE_MOV_SS;
1399         else if (mask & KVM_X86_SHADOW_INT_STI)
1400                 interruptibility |= GUEST_INTR_STATE_STI;
1401
1402         if ((interruptibility != interruptibility_old))
1403                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
1404 }
1405
1406 static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data)
1407 {
1408         struct vcpu_vmx *vmx = to_vmx(vcpu);
1409         unsigned long value;
1410
1411         /*
1412          * Any MSR write that attempts to change bits marked reserved will
1413          * case a #GP fault.
1414          */
1415         if (data & vmx->pt_desc.ctl_bitmask)
1416                 return 1;
1417
1418         /*
1419          * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will
1420          * result in a #GP unless the same write also clears TraceEn.
1421          */
1422         if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) &&
1423                 ((vmx->pt_desc.guest.ctl ^ data) & ~RTIT_CTL_TRACEEN))
1424                 return 1;
1425
1426         /*
1427          * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit
1428          * and FabricEn would cause #GP, if
1429          * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0
1430          */
1431         if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) &&
1432                 !(data & RTIT_CTL_FABRIC_EN) &&
1433                 !intel_pt_validate_cap(vmx->pt_desc.caps,
1434                                         PT_CAP_single_range_output))
1435                 return 1;
1436
1437         /*
1438          * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that
1439          * utilize encodings marked reserved will cause a #GP fault.
1440          */
1441         value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods);
1442         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) &&
1443                         !test_bit((data & RTIT_CTL_MTC_RANGE) >>
1444                         RTIT_CTL_MTC_RANGE_OFFSET, &value))
1445                 return 1;
1446         value = intel_pt_validate_cap(vmx->pt_desc.caps,
1447                                                 PT_CAP_cycle_thresholds);
1448         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) &&
1449                         !test_bit((data & RTIT_CTL_CYC_THRESH) >>
1450                         RTIT_CTL_CYC_THRESH_OFFSET, &value))
1451                 return 1;
1452         value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods);
1453         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) &&
1454                         !test_bit((data & RTIT_CTL_PSB_FREQ) >>
1455                         RTIT_CTL_PSB_FREQ_OFFSET, &value))
1456                 return 1;
1457
1458         /*
1459          * If ADDRx_CFG is reserved or the encodings is >2 will
1460          * cause a #GP fault.
1461          */
1462         value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET;
1463         if ((value && (vmx->pt_desc.num_address_ranges < 1)) || (value > 2))
1464                 return 1;
1465         value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET;
1466         if ((value && (vmx->pt_desc.num_address_ranges < 2)) || (value > 2))
1467                 return 1;
1468         value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET;
1469         if ((value && (vmx->pt_desc.num_address_ranges < 3)) || (value > 2))
1470                 return 1;
1471         value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET;
1472         if ((value && (vmx->pt_desc.num_address_ranges < 4)) || (value > 2))
1473                 return 1;
1474
1475         return 0;
1476 }
1477
1478 static bool vmx_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
1479                                         void *insn, int insn_len)
1480 {
1481         /*
1482          * Emulation of instructions in SGX enclaves is impossible as RIP does
1483          * not point at the failing instruction, and even if it did, the code
1484          * stream is inaccessible.  Inject #UD instead of exiting to userspace
1485          * so that guest userspace can't DoS the guest simply by triggering
1486          * emulation (enclaves are CPL3 only).
1487          */
1488         if (to_vmx(vcpu)->exit_reason.enclave_mode) {
1489                 kvm_queue_exception(vcpu, UD_VECTOR);
1490                 return false;
1491         }
1492         return true;
1493 }
1494
1495 static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
1496 {
1497         union vmx_exit_reason exit_reason = to_vmx(vcpu)->exit_reason;
1498         unsigned long rip, orig_rip;
1499         u32 instr_len;
1500
1501         /*
1502          * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on
1503          * undefined behavior: Intel's SDM doesn't mandate the VMCS field be
1504          * set when EPT misconfig occurs.  In practice, real hardware updates
1505          * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors
1506          * (namely Hyper-V) don't set it due to it being undefined behavior,
1507          * i.e. we end up advancing IP with some random value.
1508          */
1509         if (!static_cpu_has(X86_FEATURE_HYPERVISOR) ||
1510             exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) {
1511                 instr_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
1512
1513                 /*
1514                  * Emulating an enclave's instructions isn't supported as KVM
1515                  * cannot access the enclave's memory or its true RIP, e.g. the
1516                  * vmcs.GUEST_RIP points at the exit point of the enclave, not
1517                  * the RIP that actually triggered the VM-Exit.  But, because
1518                  * most instructions that cause VM-Exit will #UD in an enclave,
1519                  * most instruction-based VM-Exits simply do not occur.
1520                  *
1521                  * There are a few exceptions, notably the debug instructions
1522                  * INT1ICEBRK and INT3, as they are allowed in debug enclaves
1523                  * and generate #DB/#BP as expected, which KVM might intercept.
1524                  * But again, the CPU does the dirty work and saves an instr
1525                  * length of zero so VMMs don't shoot themselves in the foot.
1526                  * WARN if KVM tries to skip a non-zero length instruction on
1527                  * a VM-Exit from an enclave.
1528                  */
1529                 if (!instr_len)
1530                         goto rip_updated;
1531
1532                 WARN(exit_reason.enclave_mode,
1533                      "KVM: skipping instruction after SGX enclave VM-Exit");
1534
1535                 orig_rip = kvm_rip_read(vcpu);
1536                 rip = orig_rip + instr_len;
1537 #ifdef CONFIG_X86_64
1538                 /*
1539                  * We need to mask out the high 32 bits of RIP if not in 64-bit
1540                  * mode, but just finding out that we are in 64-bit mode is
1541                  * quite expensive.  Only do it if there was a carry.
1542                  */
1543                 if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu))
1544                         rip = (u32)rip;
1545 #endif
1546                 kvm_rip_write(vcpu, rip);
1547         } else {
1548                 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
1549                         return 0;
1550         }
1551
1552 rip_updated:
1553         /* skipping an emulated instruction also counts */
1554         vmx_set_interrupt_shadow(vcpu, 0);
1555
1556         return 1;
1557 }
1558
1559 /*
1560  * Recognizes a pending MTF VM-exit and records the nested state for later
1561  * delivery.
1562  */
1563 static void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu)
1564 {
1565         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1566         struct vcpu_vmx *vmx = to_vmx(vcpu);
1567
1568         if (!is_guest_mode(vcpu))
1569                 return;
1570
1571         /*
1572          * Per the SDM, MTF takes priority over debug-trap exceptions besides
1573          * T-bit traps. As instruction emulation is completed (i.e. at the
1574          * instruction boundary), any #DB exception pending delivery must be a
1575          * debug-trap. Record the pending MTF state to be delivered in
1576          * vmx_check_nested_events().
1577          */
1578         if (nested_cpu_has_mtf(vmcs12) &&
1579             (!vcpu->arch.exception.pending ||
1580              vcpu->arch.exception.nr == DB_VECTOR))
1581                 vmx->nested.mtf_pending = true;
1582         else
1583                 vmx->nested.mtf_pending = false;
1584 }
1585
1586 static int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu)
1587 {
1588         vmx_update_emulated_instruction(vcpu);
1589         return skip_emulated_instruction(vcpu);
1590 }
1591
1592 static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
1593 {
1594         /*
1595          * Ensure that we clear the HLT state in the VMCS.  We don't need to
1596          * explicitly skip the instruction because if the HLT state is set,
1597          * then the instruction is already executing and RIP has already been
1598          * advanced.
1599          */
1600         if (kvm_hlt_in_guest(vcpu->kvm) &&
1601                         vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
1602                 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
1603 }
1604
1605 static void vmx_queue_exception(struct kvm_vcpu *vcpu)
1606 {
1607         struct vcpu_vmx *vmx = to_vmx(vcpu);
1608         unsigned nr = vcpu->arch.exception.nr;
1609         bool has_error_code = vcpu->arch.exception.has_error_code;
1610         u32 error_code = vcpu->arch.exception.error_code;
1611         u32 intr_info = nr | INTR_INFO_VALID_MASK;
1612
1613         kvm_deliver_exception_payload(vcpu);
1614
1615         if (has_error_code) {
1616                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
1617                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
1618         }
1619
1620         if (vmx->rmode.vm86_active) {
1621                 int inc_eip = 0;
1622                 if (kvm_exception_is_soft(nr))
1623                         inc_eip = vcpu->arch.event_exit_inst_len;
1624                 kvm_inject_realmode_interrupt(vcpu, nr, inc_eip);
1625                 return;
1626         }
1627
1628         WARN_ON_ONCE(vmx->emulation_required);
1629
1630         if (kvm_exception_is_soft(nr)) {
1631                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
1632                              vmx->vcpu.arch.event_exit_inst_len);
1633                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
1634         } else
1635                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
1636
1637         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
1638
1639         vmx_clear_hlt(vcpu);
1640 }
1641
1642 static void vmx_setup_uret_msr(struct vcpu_vmx *vmx, unsigned int msr,
1643                                bool load_into_hardware)
1644 {
1645         struct vmx_uret_msr *uret_msr;
1646
1647         uret_msr = vmx_find_uret_msr(vmx, msr);
1648         if (!uret_msr)
1649                 return;
1650
1651         uret_msr->load_into_hardware = load_into_hardware;
1652 }
1653
1654 /*
1655  * Configuring user return MSRs to automatically save, load, and restore MSRs
1656  * that need to be shoved into hardware when running the guest.  Note, omitting
1657  * an MSR here does _NOT_ mean it's not emulated, only that it will not be
1658  * loaded into hardware when running the guest.
1659  */
1660 static void vmx_setup_uret_msrs(struct vcpu_vmx *vmx)
1661 {
1662 #ifdef CONFIG_X86_64
1663         bool load_syscall_msrs;
1664
1665         /*
1666          * The SYSCALL MSRs are only needed on long mode guests, and only
1667          * when EFER.SCE is set.
1668          */
1669         load_syscall_msrs = is_long_mode(&vmx->vcpu) &&
1670                             (vmx->vcpu.arch.efer & EFER_SCE);
1671
1672         vmx_setup_uret_msr(vmx, MSR_STAR, load_syscall_msrs);
1673         vmx_setup_uret_msr(vmx, MSR_LSTAR, load_syscall_msrs);
1674         vmx_setup_uret_msr(vmx, MSR_SYSCALL_MASK, load_syscall_msrs);
1675 #endif
1676         vmx_setup_uret_msr(vmx, MSR_EFER, update_transition_efer(vmx));
1677
1678         vmx_setup_uret_msr(vmx, MSR_TSC_AUX,
1679                            guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP) ||
1680                            guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDPID));
1681
1682         /*
1683          * hle=0, rtm=0, tsx_ctrl=1 can be found with some combinations of new
1684          * kernel and old userspace.  If those guests run on a tsx=off host, do
1685          * allow guests to use TSX_CTRL, but don't change the value in hardware
1686          * so that TSX remains always disabled.
1687          */
1688         vmx_setup_uret_msr(vmx, MSR_IA32_TSX_CTRL, boot_cpu_has(X86_FEATURE_RTM));
1689
1690         /*
1691          * The set of MSRs to load may have changed, reload MSRs before the
1692          * next VM-Enter.
1693          */
1694         vmx->guest_uret_msrs_loaded = false;
1695 }
1696
1697 u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
1698 {
1699         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1700
1701         if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING))
1702                 return vmcs12->tsc_offset;
1703
1704         return 0;
1705 }
1706
1707 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
1708 {
1709         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1710
1711         if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING) &&
1712             nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING))
1713                 return vmcs12->tsc_multiplier;
1714
1715         return kvm_default_tsc_scaling_ratio;
1716 }
1717
1718 static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1719 {
1720         vmcs_write64(TSC_OFFSET, offset);
1721 }
1722
1723 static void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier)
1724 {
1725         vmcs_write64(TSC_MULTIPLIER, multiplier);
1726 }
1727
1728 /*
1729  * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
1730  * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
1731  * all guests if the "nested" module option is off, and can also be disabled
1732  * for a single guest by disabling its VMX cpuid bit.
1733  */
1734 bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
1735 {
1736         return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX);
1737 }
1738
1739 static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
1740                                                  uint64_t val)
1741 {
1742         uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits;
1743
1744         return !(val & ~valid_bits);
1745 }
1746
1747 static int vmx_get_msr_feature(struct kvm_msr_entry *msr)
1748 {
1749         switch (msr->index) {
1750         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
1751                 if (!nested)
1752                         return 1;
1753                 return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data);
1754         case MSR_IA32_PERF_CAPABILITIES:
1755                 msr->data = vmx_get_perf_capabilities();
1756                 return 0;
1757         default:
1758                 return KVM_MSR_RET_INVALID;
1759         }
1760 }
1761
1762 /*
1763  * Reads an msr value (of 'msr_info->index') into 'msr_info->data'.
1764  * Returns 0 on success, non-0 otherwise.
1765  * Assumes vcpu_load() was already called.
1766  */
1767 static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1768 {
1769         struct vcpu_vmx *vmx = to_vmx(vcpu);
1770         struct vmx_uret_msr *msr;
1771         u32 index;
1772
1773         switch (msr_info->index) {
1774 #ifdef CONFIG_X86_64
1775         case MSR_FS_BASE:
1776                 msr_info->data = vmcs_readl(GUEST_FS_BASE);
1777                 break;
1778         case MSR_GS_BASE:
1779                 msr_info->data = vmcs_readl(GUEST_GS_BASE);
1780                 break;
1781         case MSR_KERNEL_GS_BASE:
1782                 msr_info->data = vmx_read_guest_kernel_gs_base(vmx);
1783                 break;
1784 #endif
1785         case MSR_EFER:
1786                 return kvm_get_msr_common(vcpu, msr_info);
1787         case MSR_IA32_TSX_CTRL:
1788                 if (!msr_info->host_initiated &&
1789                     !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR))
1790                         return 1;
1791                 goto find_uret_msr;
1792         case MSR_IA32_UMWAIT_CONTROL:
1793                 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx))
1794                         return 1;
1795
1796                 msr_info->data = vmx->msr_ia32_umwait_control;
1797                 break;
1798         case MSR_IA32_SPEC_CTRL:
1799                 if (!msr_info->host_initiated &&
1800                     !guest_has_spec_ctrl_msr(vcpu))
1801                         return 1;
1802
1803                 msr_info->data = to_vmx(vcpu)->spec_ctrl;
1804                 break;
1805         case MSR_IA32_SYSENTER_CS:
1806                 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
1807                 break;
1808         case MSR_IA32_SYSENTER_EIP:
1809                 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP);
1810                 break;
1811         case MSR_IA32_SYSENTER_ESP:
1812                 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP);
1813                 break;
1814         case MSR_IA32_BNDCFGS:
1815                 if (!kvm_mpx_supported() ||
1816                     (!msr_info->host_initiated &&
1817                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
1818                         return 1;
1819                 msr_info->data = vmcs_read64(GUEST_BNDCFGS);
1820                 break;
1821         case MSR_IA32_MCG_EXT_CTL:
1822                 if (!msr_info->host_initiated &&
1823                     !(vmx->msr_ia32_feature_control &
1824                       FEAT_CTL_LMCE_ENABLED))
1825                         return 1;
1826                 msr_info->data = vcpu->arch.mcg_ext_ctl;
1827                 break;
1828         case MSR_IA32_FEAT_CTL:
1829                 msr_info->data = vmx->msr_ia32_feature_control;
1830                 break;
1831         case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3:
1832                 if (!msr_info->host_initiated &&
1833                     !guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC))
1834                         return 1;
1835                 msr_info->data = to_vmx(vcpu)->msr_ia32_sgxlepubkeyhash
1836                         [msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0];
1837                 break;
1838         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
1839                 if (!nested_vmx_allowed(vcpu))
1840                         return 1;
1841                 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index,
1842                                     &msr_info->data))
1843                         return 1;
1844                 /*
1845                  * Enlightened VMCS v1 doesn't have certain VMCS fields but
1846                  * instead of just ignoring the features, different Hyper-V
1847                  * versions are either trying to use them and fail or do some
1848                  * sanity checking and refuse to boot. Filter all unsupported
1849                  * features out.
1850                  */
1851                 if (!msr_info->host_initiated &&
1852                     vmx->nested.enlightened_vmcs_enabled)
1853                         nested_evmcs_filter_control_msr(msr_info->index,
1854                                                         &msr_info->data);
1855                 break;
1856         case MSR_IA32_RTIT_CTL:
1857                 if (!vmx_pt_mode_is_host_guest())
1858                         return 1;
1859                 msr_info->data = vmx->pt_desc.guest.ctl;
1860                 break;
1861         case MSR_IA32_RTIT_STATUS:
1862                 if (!vmx_pt_mode_is_host_guest())
1863                         return 1;
1864                 msr_info->data = vmx->pt_desc.guest.status;
1865                 break;
1866         case MSR_IA32_RTIT_CR3_MATCH:
1867                 if (!vmx_pt_mode_is_host_guest() ||
1868                         !intel_pt_validate_cap(vmx->pt_desc.caps,
1869                                                 PT_CAP_cr3_filtering))
1870                         return 1;
1871                 msr_info->data = vmx->pt_desc.guest.cr3_match;
1872                 break;
1873         case MSR_IA32_RTIT_OUTPUT_BASE:
1874                 if (!vmx_pt_mode_is_host_guest() ||
1875                         (!intel_pt_validate_cap(vmx->pt_desc.caps,
1876                                         PT_CAP_topa_output) &&
1877                          !intel_pt_validate_cap(vmx->pt_desc.caps,
1878                                         PT_CAP_single_range_output)))
1879                         return 1;
1880                 msr_info->data = vmx->pt_desc.guest.output_base;
1881                 break;
1882         case MSR_IA32_RTIT_OUTPUT_MASK:
1883                 if (!vmx_pt_mode_is_host_guest() ||
1884                         (!intel_pt_validate_cap(vmx->pt_desc.caps,
1885                                         PT_CAP_topa_output) &&
1886                          !intel_pt_validate_cap(vmx->pt_desc.caps,
1887                                         PT_CAP_single_range_output)))
1888                         return 1;
1889                 msr_info->data = vmx->pt_desc.guest.output_mask;
1890                 break;
1891         case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
1892                 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A;
1893                 if (!vmx_pt_mode_is_host_guest() ||
1894                     (index >= 2 * vmx->pt_desc.num_address_ranges))
1895                         return 1;
1896                 if (index % 2)
1897                         msr_info->data = vmx->pt_desc.guest.addr_b[index / 2];
1898                 else
1899                         msr_info->data = vmx->pt_desc.guest.addr_a[index / 2];
1900                 break;
1901         case MSR_IA32_DEBUGCTLMSR:
1902                 msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL);
1903                 break;
1904         default:
1905         find_uret_msr:
1906                 msr = vmx_find_uret_msr(vmx, msr_info->index);
1907                 if (msr) {
1908                         msr_info->data = msr->data;
1909                         break;
1910                 }
1911                 return kvm_get_msr_common(vcpu, msr_info);
1912         }
1913
1914         return 0;
1915 }
1916
1917 static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu,
1918                                                     u64 data)
1919 {
1920 #ifdef CONFIG_X86_64
1921         if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
1922                 return (u32)data;
1923 #endif
1924         return (unsigned long)data;
1925 }
1926
1927 static u64 vcpu_supported_debugctl(struct kvm_vcpu *vcpu)
1928 {
1929         u64 debugctl = vmx_supported_debugctl();
1930
1931         if (!intel_pmu_lbr_is_enabled(vcpu))
1932                 debugctl &= ~DEBUGCTLMSR_LBR_MASK;
1933
1934         if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))
1935                 debugctl &= ~DEBUGCTLMSR_BUS_LOCK_DETECT;
1936
1937         return debugctl;
1938 }
1939
1940 /*
1941  * Writes msr value into the appropriate "register".
1942  * Returns 0 on success, non-0 otherwise.
1943  * Assumes vcpu_load() was already called.
1944  */
1945 static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1946 {
1947         struct vcpu_vmx *vmx = to_vmx(vcpu);
1948         struct vmx_uret_msr *msr;
1949         int ret = 0;
1950         u32 msr_index = msr_info->index;
1951         u64 data = msr_info->data;
1952         u32 index;
1953
1954         switch (msr_index) {
1955         case MSR_EFER:
1956                 ret = kvm_set_msr_common(vcpu, msr_info);
1957                 break;
1958 #ifdef CONFIG_X86_64
1959         case MSR_FS_BASE:
1960                 vmx_segment_cache_clear(vmx);
1961                 vmcs_writel(GUEST_FS_BASE, data);
1962                 break;
1963         case MSR_GS_BASE:
1964                 vmx_segment_cache_clear(vmx);
1965                 vmcs_writel(GUEST_GS_BASE, data);
1966                 break;
1967         case MSR_KERNEL_GS_BASE:
1968                 vmx_write_guest_kernel_gs_base(vmx, data);
1969                 break;
1970         case MSR_IA32_XFD:
1971                 ret = kvm_set_msr_common(vcpu, msr_info);
1972                 /*
1973                  * Always intercepting WRMSR could incur non-negligible
1974                  * overhead given xfd might be changed frequently in
1975                  * guest context switch. Disable write interception
1976                  * upon the first write with a non-zero value (indicating
1977                  * potential usage on dynamic xfeatures). Also update
1978                  * exception bitmap to trap #NM for proper virtualization
1979                  * of guest xfd_err.
1980                  */
1981                 if (!ret && data) {
1982                         vmx_disable_intercept_for_msr(vcpu, MSR_IA32_XFD,
1983                                                       MSR_TYPE_RW);
1984                         vcpu->arch.xfd_no_write_intercept = true;
1985                         vmx_update_exception_bitmap(vcpu);
1986                 }
1987                 break;
1988 #endif
1989         case MSR_IA32_SYSENTER_CS:
1990                 if (is_guest_mode(vcpu))
1991                         get_vmcs12(vcpu)->guest_sysenter_cs = data;
1992                 vmcs_write32(GUEST_SYSENTER_CS, data);
1993                 break;
1994         case MSR_IA32_SYSENTER_EIP:
1995                 if (is_guest_mode(vcpu)) {
1996                         data = nested_vmx_truncate_sysenter_addr(vcpu, data);
1997                         get_vmcs12(vcpu)->guest_sysenter_eip = data;
1998                 }
1999                 vmcs_writel(GUEST_SYSENTER_EIP, data);
2000                 break;
2001         case MSR_IA32_SYSENTER_ESP:
2002                 if (is_guest_mode(vcpu)) {
2003                         data = nested_vmx_truncate_sysenter_addr(vcpu, data);
2004                         get_vmcs12(vcpu)->guest_sysenter_esp = data;
2005                 }
2006                 vmcs_writel(GUEST_SYSENTER_ESP, data);
2007                 break;
2008         case MSR_IA32_DEBUGCTLMSR: {
2009                 u64 invalid = data & ~vcpu_supported_debugctl(vcpu);
2010                 if (invalid & (DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR)) {
2011                         if (report_ignored_msrs)
2012                                 vcpu_unimpl(vcpu, "%s: BTF|LBR in IA32_DEBUGCTLMSR 0x%llx, nop\n",
2013                                             __func__, data);
2014                         data &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR);
2015                         invalid &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR);
2016                 }
2017
2018                 if (invalid)
2019                         return 1;
2020
2021                 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls &
2022                                                 VM_EXIT_SAVE_DEBUG_CONTROLS)
2023                         get_vmcs12(vcpu)->guest_ia32_debugctl = data;
2024
2025                 vmcs_write64(GUEST_IA32_DEBUGCTL, data);
2026                 if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event &&
2027                     (data & DEBUGCTLMSR_LBR))
2028                         intel_pmu_create_guest_lbr_event(vcpu);
2029                 return 0;
2030         }
2031         case MSR_IA32_BNDCFGS:
2032                 if (!kvm_mpx_supported() ||
2033                     (!msr_info->host_initiated &&
2034                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
2035                         return 1;
2036                 if (is_noncanonical_address(data & PAGE_MASK, vcpu) ||
2037                     (data & MSR_IA32_BNDCFGS_RSVD))
2038                         return 1;
2039                 vmcs_write64(GUEST_BNDCFGS, data);
2040                 break;
2041         case MSR_IA32_UMWAIT_CONTROL:
2042                 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx))
2043                         return 1;
2044
2045                 /* The reserved bit 1 and non-32 bit [63:32] should be zero */
2046                 if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32)))
2047                         return 1;
2048
2049                 vmx->msr_ia32_umwait_control = data;
2050                 break;
2051         case MSR_IA32_SPEC_CTRL:
2052                 if (!msr_info->host_initiated &&
2053                     !guest_has_spec_ctrl_msr(vcpu))
2054                         return 1;
2055
2056                 if (kvm_spec_ctrl_test_value(data))
2057                         return 1;
2058
2059                 vmx->spec_ctrl = data;
2060                 if (!data)
2061                         break;
2062
2063                 /*
2064                  * For non-nested:
2065                  * When it's written (to non-zero) for the first time, pass
2066                  * it through.
2067                  *
2068                  * For nested:
2069                  * The handling of the MSR bitmap for L2 guests is done in
2070                  * nested_vmx_prepare_msr_bitmap. We should not touch the
2071                  * vmcs02.msr_bitmap here since it gets completely overwritten
2072                  * in the merging. We update the vmcs01 here for L1 as well
2073                  * since it will end up touching the MSR anyway now.
2074                  */
2075                 vmx_disable_intercept_for_msr(vcpu,
2076                                               MSR_IA32_SPEC_CTRL,
2077                                               MSR_TYPE_RW);
2078                 break;
2079         case MSR_IA32_TSX_CTRL:
2080                 if (!msr_info->host_initiated &&
2081                     !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR))
2082                         return 1;
2083                 if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR))
2084                         return 1;
2085                 goto find_uret_msr;
2086         case MSR_IA32_PRED_CMD:
2087                 if (!msr_info->host_initiated &&
2088                     !guest_has_pred_cmd_msr(vcpu))
2089                         return 1;
2090
2091                 if (data & ~PRED_CMD_IBPB)
2092                         return 1;
2093                 if (!boot_cpu_has(X86_FEATURE_IBPB))
2094                         return 1;
2095                 if (!data)
2096                         break;
2097
2098                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
2099
2100                 /*
2101                  * For non-nested:
2102                  * When it's written (to non-zero) for the first time, pass
2103                  * it through.
2104                  *
2105                  * For nested:
2106                  * The handling of the MSR bitmap for L2 guests is done in
2107                  * nested_vmx_prepare_msr_bitmap. We should not touch the
2108                  * vmcs02.msr_bitmap here since it gets completely overwritten
2109                  * in the merging.
2110                  */
2111                 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_PRED_CMD, MSR_TYPE_W);
2112                 break;
2113         case MSR_IA32_CR_PAT:
2114                 if (!kvm_pat_valid(data))
2115                         return 1;
2116
2117                 if (is_guest_mode(vcpu) &&
2118                     get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT)
2119                         get_vmcs12(vcpu)->guest_ia32_pat = data;
2120
2121                 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2122                         vmcs_write64(GUEST_IA32_PAT, data);
2123                         vcpu->arch.pat = data;
2124                         break;
2125                 }
2126                 ret = kvm_set_msr_common(vcpu, msr_info);
2127                 break;
2128         case MSR_IA32_MCG_EXT_CTL:
2129                 if ((!msr_info->host_initiated &&
2130                      !(to_vmx(vcpu)->msr_ia32_feature_control &
2131                        FEAT_CTL_LMCE_ENABLED)) ||
2132                     (data & ~MCG_EXT_CTL_LMCE_EN))
2133                         return 1;
2134                 vcpu->arch.mcg_ext_ctl = data;
2135                 break;
2136         case MSR_IA32_FEAT_CTL:
2137                 if (!vmx_feature_control_msr_valid(vcpu, data) ||
2138                     (to_vmx(vcpu)->msr_ia32_feature_control &
2139                      FEAT_CTL_LOCKED && !msr_info->host_initiated))
2140                         return 1;
2141                 vmx->msr_ia32_feature_control = data;
2142                 if (msr_info->host_initiated && data == 0)
2143                         vmx_leave_nested(vcpu);
2144
2145                 /* SGX may be enabled/disabled by guest's firmware */
2146                 vmx_write_encls_bitmap(vcpu, NULL);
2147                 break;
2148         case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3:
2149                 /*
2150                  * On real hardware, the LE hash MSRs are writable before
2151                  * the firmware sets bit 0 in MSR 0x7a ("activating" SGX),
2152                  * at which point SGX related bits in IA32_FEATURE_CONTROL
2153                  * become writable.
2154                  *
2155                  * KVM does not emulate SGX activation for simplicity, so
2156                  * allow writes to the LE hash MSRs if IA32_FEATURE_CONTROL
2157                  * is unlocked.  This is technically not architectural
2158                  * behavior, but it's close enough.
2159                  */
2160                 if (!msr_info->host_initiated &&
2161                     (!guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC) ||
2162                     ((vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED) &&
2163                     !(vmx->msr_ia32_feature_control & FEAT_CTL_SGX_LC_ENABLED))))
2164                         return 1;
2165                 vmx->msr_ia32_sgxlepubkeyhash
2166                         [msr_index - MSR_IA32_SGXLEPUBKEYHASH0] = data;
2167                 break;
2168         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
2169                 if (!msr_info->host_initiated)
2170                         return 1; /* they are read-only */
2171                 if (!nested_vmx_allowed(vcpu))
2172                         return 1;
2173                 return vmx_set_vmx_msr(vcpu, msr_index, data);
2174         case MSR_IA32_RTIT_CTL:
2175                 if (!vmx_pt_mode_is_host_guest() ||
2176                         vmx_rtit_ctl_check(vcpu, data) ||
2177                         vmx->nested.vmxon)
2178                         return 1;
2179                 vmcs_write64(GUEST_IA32_RTIT_CTL, data);
2180                 vmx->pt_desc.guest.ctl = data;
2181                 pt_update_intercept_for_msr(vcpu);
2182                 break;
2183         case MSR_IA32_RTIT_STATUS:
2184                 if (!pt_can_write_msr(vmx))
2185                         return 1;
2186                 if (data & MSR_IA32_RTIT_STATUS_MASK)
2187                         return 1;
2188                 vmx->pt_desc.guest.status = data;
2189                 break;
2190         case MSR_IA32_RTIT_CR3_MATCH:
2191                 if (!pt_can_write_msr(vmx))
2192                         return 1;
2193                 if (!intel_pt_validate_cap(vmx->pt_desc.caps,
2194                                            PT_CAP_cr3_filtering))
2195                         return 1;
2196                 vmx->pt_desc.guest.cr3_match = data;
2197                 break;
2198         case MSR_IA32_RTIT_OUTPUT_BASE:
2199                 if (!pt_can_write_msr(vmx))
2200                         return 1;
2201                 if (!intel_pt_validate_cap(vmx->pt_desc.caps,
2202                                            PT_CAP_topa_output) &&
2203                     !intel_pt_validate_cap(vmx->pt_desc.caps,
2204                                            PT_CAP_single_range_output))
2205                         return 1;
2206                 if (!pt_output_base_valid(vcpu, data))
2207                         return 1;
2208                 vmx->pt_desc.guest.output_base = data;
2209                 break;
2210         case MSR_IA32_RTIT_OUTPUT_MASK:
2211                 if (!pt_can_write_msr(vmx))
2212                         return 1;
2213                 if (!intel_pt_validate_cap(vmx->pt_desc.caps,
2214                                            PT_CAP_topa_output) &&
2215                     !intel_pt_validate_cap(vmx->pt_desc.caps,
2216                                            PT_CAP_single_range_output))
2217                         return 1;
2218                 vmx->pt_desc.guest.output_mask = data;
2219                 break;
2220         case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
2221                 if (!pt_can_write_msr(vmx))
2222                         return 1;
2223                 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A;
2224                 if (index >= 2 * vmx->pt_desc.num_address_ranges)
2225                         return 1;
2226                 if (is_noncanonical_address(data, vcpu))
2227                         return 1;
2228                 if (index % 2)
2229                         vmx->pt_desc.guest.addr_b[index / 2] = data;
2230                 else
2231                         vmx->pt_desc.guest.addr_a[index / 2] = data;
2232                 break;
2233         case MSR_IA32_PERF_CAPABILITIES:
2234                 if (data && !vcpu_to_pmu(vcpu)->version)
2235                         return 1;
2236                 if (data & PMU_CAP_LBR_FMT) {
2237                         if ((data & PMU_CAP_LBR_FMT) !=
2238                             (vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT))
2239                                 return 1;
2240                         if (!intel_pmu_lbr_is_compatible(vcpu))
2241                                 return 1;
2242                 }
2243                 ret = kvm_set_msr_common(vcpu, msr_info);
2244                 break;
2245
2246         default:
2247         find_uret_msr:
2248                 msr = vmx_find_uret_msr(vmx, msr_index);
2249                 if (msr)
2250                         ret = vmx_set_guest_uret_msr(vmx, msr, data);
2251                 else
2252                         ret = kvm_set_msr_common(vcpu, msr_info);
2253         }
2254
2255         return ret;
2256 }
2257
2258 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2259 {
2260         unsigned long guest_owned_bits;
2261
2262         kvm_register_mark_available(vcpu, reg);
2263
2264         switch (reg) {
2265         case VCPU_REGS_RSP:
2266                 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
2267                 break;
2268         case VCPU_REGS_RIP:
2269                 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
2270                 break;
2271         case VCPU_EXREG_PDPTR:
2272                 if (enable_ept)
2273                         ept_save_pdptrs(vcpu);
2274                 break;
2275         case VCPU_EXREG_CR0:
2276                 guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
2277
2278                 vcpu->arch.cr0 &= ~guest_owned_bits;
2279                 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits;
2280                 break;
2281         case VCPU_EXREG_CR3:
2282                 /*
2283                  * When intercepting CR3 loads, e.g. for shadowing paging, KVM's
2284                  * CR3 is loaded into hardware, not the guest's CR3.
2285                  */
2286                 if (!(exec_controls_get(to_vmx(vcpu)) & CPU_BASED_CR3_LOAD_EXITING))
2287                         vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
2288                 break;
2289         case VCPU_EXREG_CR4:
2290                 guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
2291
2292                 vcpu->arch.cr4 &= ~guest_owned_bits;
2293                 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits;
2294                 break;
2295         default:
2296                 KVM_BUG_ON(1, vcpu->kvm);
2297                 break;
2298         }
2299 }
2300
2301 static __init int cpu_has_kvm_support(void)
2302 {
2303         return cpu_has_vmx();
2304 }
2305
2306 static __init int vmx_disabled_by_bios(void)
2307 {
2308         return !boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
2309                !boot_cpu_has(X86_FEATURE_VMX);
2310 }
2311
2312 static int kvm_cpu_vmxon(u64 vmxon_pointer)
2313 {
2314         u64 msr;
2315
2316         cr4_set_bits(X86_CR4_VMXE);
2317
2318         asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t"
2319                           _ASM_EXTABLE(1b, %l[fault])
2320                           : : [vmxon_pointer] "m"(vmxon_pointer)
2321                           : : fault);
2322         return 0;
2323
2324 fault:
2325         WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
2326                   rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
2327         cr4_clear_bits(X86_CR4_VMXE);
2328
2329         return -EFAULT;
2330 }
2331
2332 static int vmx_hardware_enable(void)
2333 {
2334         int cpu = raw_smp_processor_id();
2335         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
2336         int r;
2337
2338         if (cr4_read_shadow() & X86_CR4_VMXE)
2339                 return -EBUSY;
2340
2341         /*
2342          * This can happen if we hot-added a CPU but failed to allocate
2343          * VP assist page for it.
2344          */
2345         if (static_branch_unlikely(&enable_evmcs) &&
2346             !hv_get_vp_assist_page(cpu))
2347                 return -EFAULT;
2348
2349         intel_pt_handle_vmx(1);
2350
2351         r = kvm_cpu_vmxon(phys_addr);
2352         if (r) {
2353                 intel_pt_handle_vmx(0);
2354                 return r;
2355         }
2356
2357         if (enable_ept)
2358                 ept_sync_global();
2359
2360         return 0;
2361 }
2362
2363 static void vmclear_local_loaded_vmcss(void)
2364 {
2365         int cpu = raw_smp_processor_id();
2366         struct loaded_vmcs *v, *n;
2367
2368         list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
2369                                  loaded_vmcss_on_cpu_link)
2370                 __loaded_vmcs_clear(v);
2371 }
2372
2373 static void vmx_hardware_disable(void)
2374 {
2375         vmclear_local_loaded_vmcss();
2376
2377         if (cpu_vmxoff())
2378                 kvm_spurious_fault();
2379
2380         intel_pt_handle_vmx(0);
2381 }
2382
2383 /*
2384  * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID
2385  * directly instead of going through cpu_has(), to ensure KVM is trapping
2386  * ENCLS whenever it's supported in hardware.  It does not matter whether
2387  * the host OS supports or has enabled SGX.
2388  */
2389 static bool cpu_has_sgx(void)
2390 {
2391         return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0));
2392 }
2393
2394 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
2395                                       u32 msr, u32 *result)
2396 {
2397         u32 vmx_msr_low, vmx_msr_high;
2398         u32 ctl = ctl_min | ctl_opt;
2399
2400         rdmsr(msr, vmx_msr_low, vmx_msr_high);
2401
2402         ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
2403         ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */
2404
2405         /* Ensure minimum (required) set of control bits are supported. */
2406         if (ctl_min & ~ctl)
2407                 return -EIO;
2408
2409         *result = ctl;
2410         return 0;
2411 }
2412
2413 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf,
2414                                     struct vmx_capability *vmx_cap)
2415 {
2416         u32 vmx_msr_low, vmx_msr_high;
2417         u32 min, opt, min2, opt2;
2418         u32 _pin_based_exec_control = 0;
2419         u32 _cpu_based_exec_control = 0;
2420         u32 _cpu_based_2nd_exec_control = 0;
2421         u32 _vmexit_control = 0;
2422         u32 _vmentry_control = 0;
2423
2424         memset(vmcs_conf, 0, sizeof(*vmcs_conf));
2425         min = CPU_BASED_HLT_EXITING |
2426 #ifdef CONFIG_X86_64
2427               CPU_BASED_CR8_LOAD_EXITING |
2428               CPU_BASED_CR8_STORE_EXITING |
2429 #endif
2430               CPU_BASED_CR3_LOAD_EXITING |
2431               CPU_BASED_CR3_STORE_EXITING |
2432               CPU_BASED_UNCOND_IO_EXITING |
2433               CPU_BASED_MOV_DR_EXITING |
2434               CPU_BASED_USE_TSC_OFFSETTING |
2435               CPU_BASED_MWAIT_EXITING |
2436               CPU_BASED_MONITOR_EXITING |
2437               CPU_BASED_INVLPG_EXITING |
2438               CPU_BASED_RDPMC_EXITING;
2439
2440         opt = CPU_BASED_TPR_SHADOW |
2441               CPU_BASED_USE_MSR_BITMAPS |
2442               CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
2443         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
2444                                 &_cpu_based_exec_control) < 0)
2445                 return -EIO;
2446 #ifdef CONFIG_X86_64
2447         if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
2448                 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
2449                                            ~CPU_BASED_CR8_STORE_EXITING;
2450 #endif
2451         if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
2452                 min2 = 0;
2453                 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2454                         SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2455                         SECONDARY_EXEC_WBINVD_EXITING |
2456                         SECONDARY_EXEC_ENABLE_VPID |
2457                         SECONDARY_EXEC_ENABLE_EPT |
2458                         SECONDARY_EXEC_UNRESTRICTED_GUEST |
2459                         SECONDARY_EXEC_PAUSE_LOOP_EXITING |
2460                         SECONDARY_EXEC_DESC |
2461                         SECONDARY_EXEC_ENABLE_RDTSCP |
2462                         SECONDARY_EXEC_ENABLE_INVPCID |
2463                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
2464                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2465                         SECONDARY_EXEC_SHADOW_VMCS |
2466                         SECONDARY_EXEC_XSAVES |
2467                         SECONDARY_EXEC_RDSEED_EXITING |
2468                         SECONDARY_EXEC_RDRAND_EXITING |
2469                         SECONDARY_EXEC_ENABLE_PML |
2470                         SECONDARY_EXEC_TSC_SCALING |
2471                         SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |
2472                         SECONDARY_EXEC_PT_USE_GPA |
2473                         SECONDARY_EXEC_PT_CONCEAL_VMX |
2474                         SECONDARY_EXEC_ENABLE_VMFUNC |
2475                         SECONDARY_EXEC_BUS_LOCK_DETECTION;
2476                 if (cpu_has_sgx())
2477                         opt2 |= SECONDARY_EXEC_ENCLS_EXITING;
2478                 if (adjust_vmx_controls(min2, opt2,
2479                                         MSR_IA32_VMX_PROCBASED_CTLS2,
2480                                         &_cpu_based_2nd_exec_control) < 0)
2481                         return -EIO;
2482         }
2483 #ifndef CONFIG_X86_64
2484         if (!(_cpu_based_2nd_exec_control &
2485                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
2486                 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
2487 #endif
2488
2489         if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
2490                 _cpu_based_2nd_exec_control &= ~(
2491                                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
2492                                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2493                                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
2494
2495         rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP,
2496                 &vmx_cap->ept, &vmx_cap->vpid);
2497
2498         if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
2499                 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
2500                    enabled */
2501                 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
2502                                              CPU_BASED_CR3_STORE_EXITING |
2503                                              CPU_BASED_INVLPG_EXITING);
2504         } else if (vmx_cap->ept) {
2505                 vmx_cap->ept = 0;
2506                 pr_warn_once("EPT CAP should not exist if not support "
2507                                 "1-setting enable EPT VM-execution control\n");
2508         }
2509         if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) &&
2510                 vmx_cap->vpid) {
2511                 vmx_cap->vpid = 0;
2512                 pr_warn_once("VPID CAP should not exist if not support "
2513                                 "1-setting enable VPID VM-execution control\n");
2514         }
2515
2516         min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT;
2517 #ifdef CONFIG_X86_64
2518         min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
2519 #endif
2520         opt = VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
2521               VM_EXIT_LOAD_IA32_PAT |
2522               VM_EXIT_LOAD_IA32_EFER |
2523               VM_EXIT_CLEAR_BNDCFGS |
2524               VM_EXIT_PT_CONCEAL_PIP |
2525               VM_EXIT_CLEAR_IA32_RTIT_CTL;
2526         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
2527                                 &_vmexit_control) < 0)
2528                 return -EIO;
2529
2530         min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
2531         opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR |
2532                  PIN_BASED_VMX_PREEMPTION_TIMER;
2533         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
2534                                 &_pin_based_exec_control) < 0)
2535                 return -EIO;
2536
2537         if (cpu_has_broken_vmx_preemption_timer())
2538                 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
2539         if (!(_cpu_based_2nd_exec_control &
2540                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY))
2541                 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
2542
2543         min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
2544         opt = VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL |
2545               VM_ENTRY_LOAD_IA32_PAT |
2546               VM_ENTRY_LOAD_IA32_EFER |
2547               VM_ENTRY_LOAD_BNDCFGS |
2548               VM_ENTRY_PT_CONCEAL_PIP |
2549               VM_ENTRY_LOAD_IA32_RTIT_CTL;
2550         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
2551                                 &_vmentry_control) < 0)
2552                 return -EIO;
2553
2554         /*
2555          * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they
2556          * can't be used due to an errata where VM Exit may incorrectly clear
2557          * IA32_PERF_GLOBAL_CTRL[34:32].  Workaround the errata by using the
2558          * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL.
2559          */
2560         if (boot_cpu_data.x86 == 0x6) {
2561                 switch (boot_cpu_data.x86_model) {
2562                 case 26: /* AAK155 */
2563                 case 30: /* AAP115 */
2564                 case 37: /* AAT100 */
2565                 case 44: /* BC86,AAY89,BD102 */
2566                 case 46: /* BA97 */
2567                         _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
2568                         _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
2569                         pr_warn_once("kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
2570                                         "does not work properly. Using workaround\n");
2571                         break;
2572                 default:
2573                         break;
2574                 }
2575         }
2576
2577
2578         rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
2579
2580         /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
2581         if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
2582                 return -EIO;
2583
2584 #ifdef CONFIG_X86_64
2585         /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
2586         if (vmx_msr_high & (1u<<16))
2587                 return -EIO;
2588 #endif
2589
2590         /* Require Write-Back (WB) memory type for VMCS accesses. */
2591         if (((vmx_msr_high >> 18) & 15) != 6)
2592                 return -EIO;
2593
2594         vmcs_conf->size = vmx_msr_high & 0x1fff;
2595         vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff;
2596
2597         vmcs_conf->revision_id = vmx_msr_low;
2598
2599         vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
2600         vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
2601         vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
2602         vmcs_conf->vmexit_ctrl         = _vmexit_control;
2603         vmcs_conf->vmentry_ctrl        = _vmentry_control;
2604
2605 #if IS_ENABLED(CONFIG_HYPERV)
2606         if (enlightened_vmcs)
2607                 evmcs_sanitize_exec_ctrls(vmcs_conf);
2608 #endif
2609
2610         return 0;
2611 }
2612
2613 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags)
2614 {
2615         int node = cpu_to_node(cpu);
2616         struct page *pages;
2617         struct vmcs *vmcs;
2618
2619         pages = __alloc_pages_node(node, flags, 0);
2620         if (!pages)
2621                 return NULL;
2622         vmcs = page_address(pages);
2623         memset(vmcs, 0, vmcs_config.size);
2624
2625         /* KVM supports Enlightened VMCS v1 only */
2626         if (static_branch_unlikely(&enable_evmcs))
2627                 vmcs->hdr.revision_id = KVM_EVMCS_VERSION;
2628         else
2629                 vmcs->hdr.revision_id = vmcs_config.revision_id;
2630
2631         if (shadow)
2632                 vmcs->hdr.shadow_vmcs = 1;
2633         return vmcs;
2634 }
2635
2636 void free_vmcs(struct vmcs *vmcs)
2637 {
2638         free_page((unsigned long)vmcs);
2639 }
2640
2641 /*
2642  * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
2643  */
2644 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
2645 {
2646         if (!loaded_vmcs->vmcs)
2647                 return;
2648         loaded_vmcs_clear(loaded_vmcs);
2649         free_vmcs(loaded_vmcs->vmcs);
2650         loaded_vmcs->vmcs = NULL;
2651         if (loaded_vmcs->msr_bitmap)
2652                 free_page((unsigned long)loaded_vmcs->msr_bitmap);
2653         WARN_ON(loaded_vmcs->shadow_vmcs != NULL);
2654 }
2655
2656 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
2657 {
2658         loaded_vmcs->vmcs = alloc_vmcs(false);
2659         if (!loaded_vmcs->vmcs)
2660                 return -ENOMEM;
2661
2662         vmcs_clear(loaded_vmcs->vmcs);
2663
2664         loaded_vmcs->shadow_vmcs = NULL;
2665         loaded_vmcs->hv_timer_soft_disabled = false;
2666         loaded_vmcs->cpu = -1;
2667         loaded_vmcs->launched = 0;
2668
2669         if (cpu_has_vmx_msr_bitmap()) {
2670                 loaded_vmcs->msr_bitmap = (unsigned long *)
2671                                 __get_free_page(GFP_KERNEL_ACCOUNT);
2672                 if (!loaded_vmcs->msr_bitmap)
2673                         goto out_vmcs;
2674                 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
2675         }
2676
2677         memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state));
2678         memset(&loaded_vmcs->controls_shadow, 0,
2679                 sizeof(struct vmcs_controls_shadow));
2680
2681         return 0;
2682
2683 out_vmcs:
2684         free_loaded_vmcs(loaded_vmcs);
2685         return -ENOMEM;
2686 }
2687
2688 static void free_kvm_area(void)
2689 {
2690         int cpu;
2691
2692         for_each_possible_cpu(cpu) {
2693                 free_vmcs(per_cpu(vmxarea, cpu));
2694                 per_cpu(vmxarea, cpu) = NULL;
2695         }
2696 }
2697
2698 static __init int alloc_kvm_area(void)
2699 {
2700         int cpu;
2701
2702         for_each_possible_cpu(cpu) {
2703                 struct vmcs *vmcs;
2704
2705                 vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL);
2706                 if (!vmcs) {
2707                         free_kvm_area();
2708                         return -ENOMEM;
2709                 }
2710
2711                 /*
2712                  * When eVMCS is enabled, alloc_vmcs_cpu() sets
2713                  * vmcs->revision_id to KVM_EVMCS_VERSION instead of
2714                  * revision_id reported by MSR_IA32_VMX_BASIC.
2715                  *
2716                  * However, even though not explicitly documented by
2717                  * TLFS, VMXArea passed as VMXON argument should
2718                  * still be marked with revision_id reported by
2719                  * physical CPU.
2720                  */
2721                 if (static_branch_unlikely(&enable_evmcs))
2722                         vmcs->hdr.revision_id = vmcs_config.revision_id;
2723
2724                 per_cpu(vmxarea, cpu) = vmcs;
2725         }
2726         return 0;
2727 }
2728
2729 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
2730                 struct kvm_segment *save)
2731 {
2732         if (!emulate_invalid_guest_state) {
2733                 /*
2734                  * CS and SS RPL should be equal during guest entry according
2735                  * to VMX spec, but in reality it is not always so. Since vcpu
2736                  * is in the middle of the transition from real mode to
2737                  * protected mode it is safe to assume that RPL 0 is a good
2738                  * default value.
2739                  */
2740                 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
2741                         save->selector &= ~SEGMENT_RPL_MASK;
2742                 save->dpl = save->selector & SEGMENT_RPL_MASK;
2743                 save->s = 1;
2744         }
2745         __vmx_set_segment(vcpu, save, seg);
2746 }
2747
2748 static void enter_pmode(struct kvm_vcpu *vcpu)
2749 {
2750         unsigned long flags;
2751         struct vcpu_vmx *vmx = to_vmx(vcpu);
2752
2753         /*
2754          * Update real mode segment cache. It may be not up-to-date if segment
2755          * register was written while vcpu was in a guest mode.
2756          */
2757         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
2758         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
2759         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
2760         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
2761         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
2762         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
2763
2764         vmx->rmode.vm86_active = 0;
2765
2766         __vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
2767
2768         flags = vmcs_readl(GUEST_RFLAGS);
2769         flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
2770         flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
2771         vmcs_writel(GUEST_RFLAGS, flags);
2772
2773         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
2774                         (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
2775
2776         vmx_update_exception_bitmap(vcpu);
2777
2778         fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
2779         fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
2780         fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
2781         fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
2782         fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
2783         fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
2784 }
2785
2786 static void fix_rmode_seg(int seg, struct kvm_segment *save)
2787 {
2788         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2789         struct kvm_segment var = *save;
2790
2791         var.dpl = 0x3;
2792         if (seg == VCPU_SREG_CS)
2793                 var.type = 0x3;
2794
2795         if (!emulate_invalid_guest_state) {
2796                 var.selector = var.base >> 4;
2797                 var.base = var.base & 0xffff0;
2798                 var.limit = 0xffff;
2799                 var.g = 0;
2800                 var.db = 0;
2801                 var.present = 1;
2802                 var.s = 1;
2803                 var.l = 0;
2804                 var.unusable = 0;
2805                 var.type = 0x3;
2806                 var.avl = 0;
2807                 if (save->base & 0xf)
2808                         printk_once(KERN_WARNING "kvm: segment base is not "
2809                                         "paragraph aligned when entering "
2810                                         "protected mode (seg=%d)", seg);
2811         }
2812
2813         vmcs_write16(sf->selector, var.selector);
2814         vmcs_writel(sf->base, var.base);
2815         vmcs_write32(sf->limit, var.limit);
2816         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
2817 }
2818
2819 static void enter_rmode(struct kvm_vcpu *vcpu)
2820 {
2821         unsigned long flags;
2822         struct vcpu_vmx *vmx = to_vmx(vcpu);
2823         struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm);
2824
2825         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
2826         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
2827         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
2828         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
2829         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
2830         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
2831         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
2832
2833         vmx->rmode.vm86_active = 1;
2834
2835         /*
2836          * Very old userspace does not call KVM_SET_TSS_ADDR before entering
2837          * vcpu. Warn the user that an update is overdue.
2838          */
2839         if (!kvm_vmx->tss_addr)
2840                 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
2841                              "called before entering vcpu\n");
2842
2843         vmx_segment_cache_clear(vmx);
2844
2845         vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr);
2846         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
2847         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
2848
2849         flags = vmcs_readl(GUEST_RFLAGS);
2850         vmx->rmode.save_rflags = flags;
2851
2852         flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
2853
2854         vmcs_writel(GUEST_RFLAGS, flags);
2855         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
2856         vmx_update_exception_bitmap(vcpu);
2857
2858         fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
2859         fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
2860         fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
2861         fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
2862         fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
2863         fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
2864 }
2865
2866 int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
2867 {
2868         struct vcpu_vmx *vmx = to_vmx(vcpu);
2869         struct vmx_uret_msr *msr = vmx_find_uret_msr(vmx, MSR_EFER);
2870
2871         /* Nothing to do if hardware doesn't support EFER. */
2872         if (!msr)
2873                 return 0;
2874
2875         vcpu->arch.efer = efer;
2876         if (efer & EFER_LMA) {
2877                 vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
2878                 msr->data = efer;
2879         } else {
2880                 vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
2881
2882                 msr->data = efer & ~EFER_LME;
2883         }
2884         vmx_setup_uret_msrs(vmx);
2885         return 0;
2886 }
2887
2888 #ifdef CONFIG_X86_64
2889
2890 static void enter_lmode(struct kvm_vcpu *vcpu)
2891 {
2892         u32 guest_tr_ar;
2893
2894         vmx_segment_cache_clear(to_vmx(vcpu));
2895
2896         guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
2897         if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) {
2898                 pr_debug_ratelimited("%s: tss fixup for long mode. \n",
2899                                      __func__);
2900                 vmcs_write32(GUEST_TR_AR_BYTES,
2901                              (guest_tr_ar & ~VMX_AR_TYPE_MASK)
2902                              | VMX_AR_TYPE_BUSY_64_TSS);
2903         }
2904         vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
2905 }
2906
2907 static void exit_lmode(struct kvm_vcpu *vcpu)
2908 {
2909         vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
2910         vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
2911 }
2912
2913 #endif
2914
2915 static void vmx_flush_tlb_all(struct kvm_vcpu *vcpu)
2916 {
2917         struct vcpu_vmx *vmx = to_vmx(vcpu);
2918
2919         /*
2920          * INVEPT must be issued when EPT is enabled, irrespective of VPID, as
2921          * the CPU is not required to invalidate guest-physical mappings on
2922          * VM-Entry, even if VPID is disabled.  Guest-physical mappings are
2923          * associated with the root EPT structure and not any particular VPID
2924          * (INVVPID also isn't required to invalidate guest-physical mappings).
2925          */
2926         if (enable_ept) {
2927                 ept_sync_global();
2928         } else if (enable_vpid) {
2929                 if (cpu_has_vmx_invvpid_global()) {
2930                         vpid_sync_vcpu_global();
2931                 } else {
2932                         vpid_sync_vcpu_single(vmx->vpid);
2933                         vpid_sync_vcpu_single(vmx->nested.vpid02);
2934                 }
2935         }
2936 }
2937
2938 static inline int vmx_get_current_vpid(struct kvm_vcpu *vcpu)
2939 {
2940         if (is_guest_mode(vcpu))
2941                 return nested_get_vpid02(vcpu);
2942         return to_vmx(vcpu)->vpid;
2943 }
2944
2945 static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
2946 {
2947         struct kvm_mmu *mmu = vcpu->arch.mmu;
2948         u64 root_hpa = mmu->root.hpa;
2949
2950         /* No flush required if the current context is invalid. */
2951         if (!VALID_PAGE(root_hpa))
2952                 return;
2953
2954         if (enable_ept)
2955                 ept_sync_context(construct_eptp(vcpu, root_hpa,
2956                                                 mmu->shadow_root_level));
2957         else
2958                 vpid_sync_context(vmx_get_current_vpid(vcpu));
2959 }
2960
2961 static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
2962 {
2963         /*
2964          * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in
2965          * vmx_flush_tlb_guest() for an explanation of why this is ok.
2966          */
2967         vpid_sync_vcpu_addr(vmx_get_current_vpid(vcpu), addr);
2968 }
2969
2970 static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu)
2971 {
2972         /*
2973          * vpid_sync_context() is a nop if vpid==0, e.g. if enable_vpid==0 or a
2974          * vpid couldn't be allocated for this vCPU.  VM-Enter and VM-Exit are
2975          * required to flush GVA->{G,H}PA mappings from the TLB if vpid is
2976          * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed),
2977          * i.e. no explicit INVVPID is necessary.
2978          */
2979         vpid_sync_context(vmx_get_current_vpid(vcpu));
2980 }
2981
2982 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu)
2983 {
2984         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
2985
2986         if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR))
2987                 return;
2988
2989         if (is_pae_paging(vcpu)) {
2990                 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
2991                 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
2992                 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
2993                 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
2994         }
2995 }
2996
2997 void ept_save_pdptrs(struct kvm_vcpu *vcpu)
2998 {
2999         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
3000
3001         if (WARN_ON_ONCE(!is_pae_paging(vcpu)))
3002                 return;
3003
3004         mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
3005         mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
3006         mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
3007         mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
3008
3009         kvm_register_mark_available(vcpu, VCPU_EXREG_PDPTR);
3010 }
3011
3012 #define CR3_EXITING_BITS (CPU_BASED_CR3_LOAD_EXITING | \
3013                           CPU_BASED_CR3_STORE_EXITING)
3014
3015 void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
3016 {
3017         struct vcpu_vmx *vmx = to_vmx(vcpu);
3018         unsigned long hw_cr0, old_cr0_pg;
3019         u32 tmp;
3020
3021         old_cr0_pg = kvm_read_cr0_bits(vcpu, X86_CR0_PG);
3022
3023         hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF);
3024         if (is_unrestricted_guest(vcpu))
3025                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
3026         else {
3027                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
3028                 if (!enable_ept)
3029                         hw_cr0 |= X86_CR0_WP;
3030
3031                 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
3032                         enter_pmode(vcpu);
3033
3034                 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
3035                         enter_rmode(vcpu);
3036         }
3037
3038         vmcs_writel(CR0_READ_SHADOW, cr0);
3039         vmcs_writel(GUEST_CR0, hw_cr0);
3040         vcpu->arch.cr0 = cr0;
3041         kvm_register_mark_available(vcpu, VCPU_EXREG_CR0);
3042
3043 #ifdef CONFIG_X86_64
3044         if (vcpu->arch.efer & EFER_LME) {
3045                 if (!old_cr0_pg && (cr0 & X86_CR0_PG))
3046                         enter_lmode(vcpu);
3047                 else if (old_cr0_pg && !(cr0 & X86_CR0_PG))
3048                         exit_lmode(vcpu);
3049         }
3050 #endif
3051
3052         if (enable_ept && !is_unrestricted_guest(vcpu)) {
3053                 /*
3054                  * Ensure KVM has an up-to-date snapshot of the guest's CR3.  If
3055                  * the below code _enables_ CR3 exiting, vmx_cache_reg() will
3056                  * (correctly) stop reading vmcs.GUEST_CR3 because it thinks
3057                  * KVM's CR3 is installed.
3058                  */
3059                 if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
3060                         vmx_cache_reg(vcpu, VCPU_EXREG_CR3);
3061
3062                 /*
3063                  * When running with EPT but not unrestricted guest, KVM must
3064                  * intercept CR3 accesses when paging is _disabled_.  This is
3065                  * necessary because restricted guests can't actually run with
3066                  * paging disabled, and so KVM stuffs its own CR3 in order to
3067                  * run the guest when identity mapped page tables.
3068                  *
3069                  * Do _NOT_ check the old CR0.PG, e.g. to optimize away the
3070                  * update, it may be stale with respect to CR3 interception,
3071                  * e.g. after nested VM-Enter.
3072                  *
3073                  * Lastly, honor L1's desires, i.e. intercept CR3 loads and/or
3074                  * stores to forward them to L1, even if KVM does not need to
3075                  * intercept them to preserve its identity mapped page tables.
3076                  */
3077                 if (!(cr0 & X86_CR0_PG)) {
3078                         exec_controls_setbit(vmx, CR3_EXITING_BITS);
3079                 } else if (!is_guest_mode(vcpu)) {
3080                         exec_controls_clearbit(vmx, CR3_EXITING_BITS);
3081                 } else {
3082                         tmp = exec_controls_get(vmx);
3083                         tmp &= ~CR3_EXITING_BITS;
3084                         tmp |= get_vmcs12(vcpu)->cpu_based_vm_exec_control & CR3_EXITING_BITS;
3085                         exec_controls_set(vmx, tmp);
3086                 }
3087
3088                 /* Note, vmx_set_cr4() consumes the new vcpu->arch.cr0. */
3089                 if ((old_cr0_pg ^ cr0) & X86_CR0_PG)
3090                         vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3091
3092                 /*
3093                  * When !CR0_PG -> CR0_PG, vcpu->arch.cr3 becomes active, but
3094                  * GUEST_CR3 is still vmx->ept_identity_map_addr if EPT + !URG.
3095                  */
3096                 if (!(old_cr0_pg & X86_CR0_PG) && (cr0 & X86_CR0_PG))
3097                         kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
3098         }
3099
3100         /* depends on vcpu->arch.cr0 to be set to a new value */
3101         vmx->emulation_required = vmx_emulation_required(vcpu);
3102 }
3103
3104 static int vmx_get_max_tdp_level(void)
3105 {
3106         if (cpu_has_vmx_ept_5levels())
3107                 return 5;
3108         return 4;
3109 }
3110
3111 u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level)
3112 {
3113         u64 eptp = VMX_EPTP_MT_WB;
3114
3115         eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;
3116
3117         if (enable_ept_ad_bits &&
3118             (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu)))
3119                 eptp |= VMX_EPTP_AD_ENABLE_BIT;
3120         eptp |= root_hpa;
3121
3122         return eptp;
3123 }
3124
3125 static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
3126                              int root_level)
3127 {
3128         struct kvm *kvm = vcpu->kvm;
3129         bool update_guest_cr3 = true;
3130         unsigned long guest_cr3;
3131         u64 eptp;
3132
3133         if (enable_ept) {
3134                 eptp = construct_eptp(vcpu, root_hpa, root_level);
3135                 vmcs_write64(EPT_POINTER, eptp);
3136
3137                 hv_track_root_tdp(vcpu, root_hpa);
3138
3139                 if (!enable_unrestricted_guest && !is_paging(vcpu))
3140                         guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr;
3141                 else if (kvm_register_is_dirty(vcpu, VCPU_EXREG_CR3))
3142                         guest_cr3 = vcpu->arch.cr3;
3143                 else /* vmcs.GUEST_CR3 is already up-to-date. */
3144                         update_guest_cr3 = false;
3145                 vmx_ept_load_pdptrs(vcpu);
3146         } else {
3147                 guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu);
3148         }
3149
3150         if (update_guest_cr3)
3151                 vmcs_writel(GUEST_CR3, guest_cr3);
3152 }
3153
3154
3155 static bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
3156 {
3157         /*
3158          * We operate under the default treatment of SMM, so VMX cannot be
3159          * enabled under SMM.  Note, whether or not VMXE is allowed at all is
3160          * handled by kvm_is_valid_cr4().
3161          */
3162         if ((cr4 & X86_CR4_VMXE) && is_smm(vcpu))
3163                 return false;
3164
3165         if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4))
3166                 return false;
3167
3168         return true;
3169 }
3170
3171 void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
3172 {
3173         unsigned long old_cr4 = vcpu->arch.cr4;
3174         struct vcpu_vmx *vmx = to_vmx(vcpu);
3175         /*
3176          * Pass through host's Machine Check Enable value to hw_cr4, which
3177          * is in force while we are in guest mode.  Do not let guests control
3178          * this bit, even if host CR4.MCE == 0.
3179          */
3180         unsigned long hw_cr4;
3181
3182         hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE);
3183         if (is_unrestricted_guest(vcpu))
3184                 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST;
3185         else if (vmx->rmode.vm86_active)
3186                 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON;
3187         else
3188                 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON;
3189
3190         if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) {
3191                 if (cr4 & X86_CR4_UMIP) {
3192                         secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC);
3193                         hw_cr4 &= ~X86_CR4_UMIP;
3194                 } else if (!is_guest_mode(vcpu) ||
3195                         !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) {
3196                         secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC);
3197                 }
3198         }
3199
3200         vcpu->arch.cr4 = cr4;
3201         kvm_register_mark_available(vcpu, VCPU_EXREG_CR4);
3202
3203         if (!is_unrestricted_guest(vcpu)) {
3204                 if (enable_ept) {
3205                         if (!is_paging(vcpu)) {
3206                                 hw_cr4 &= ~X86_CR4_PAE;
3207                                 hw_cr4 |= X86_CR4_PSE;
3208                         } else if (!(cr4 & X86_CR4_PAE)) {
3209                                 hw_cr4 &= ~X86_CR4_PAE;
3210                         }
3211                 }
3212
3213                 /*
3214                  * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in
3215                  * hardware.  To emulate this behavior, SMEP/SMAP/PKU needs
3216                  * to be manually disabled when guest switches to non-paging
3217                  * mode.
3218                  *
3219                  * If !enable_unrestricted_guest, the CPU is always running
3220                  * with CR0.PG=1 and CR4 needs to be modified.
3221                  * If enable_unrestricted_guest, the CPU automatically
3222                  * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0.
3223                  */
3224                 if (!is_paging(vcpu))
3225                         hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
3226         }
3227
3228         vmcs_writel(CR4_READ_SHADOW, cr4);
3229         vmcs_writel(GUEST_CR4, hw_cr4);
3230
3231         if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
3232                 kvm_update_cpuid_runtime(vcpu);
3233 }
3234
3235 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
3236 {
3237         struct vcpu_vmx *vmx = to_vmx(vcpu);
3238         u32 ar;
3239
3240         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
3241                 *var = vmx->rmode.segs[seg];
3242                 if (seg == VCPU_SREG_TR
3243                     || var->selector == vmx_read_guest_seg_selector(vmx, seg))
3244                         return;
3245                 var->base = vmx_read_guest_seg_base(vmx, seg);
3246                 var->selector = vmx_read_guest_seg_selector(vmx, seg);
3247                 return;
3248         }
3249         var->base = vmx_read_guest_seg_base(vmx, seg);
3250         var->limit = vmx_read_guest_seg_limit(vmx, seg);
3251         var->selector = vmx_read_guest_seg_selector(vmx, seg);
3252         ar = vmx_read_guest_seg_ar(vmx, seg);
3253         var->unusable = (ar >> 16) & 1;
3254         var->type = ar & 15;
3255         var->s = (ar >> 4) & 1;
3256         var->dpl = (ar >> 5) & 3;
3257         /*
3258          * Some userspaces do not preserve unusable property. Since usable
3259          * segment has to be present according to VMX spec we can use present
3260          * property to amend userspace bug by making unusable segment always
3261          * nonpresent. vmx_segment_access_rights() already marks nonpresent
3262          * segment as unusable.
3263          */
3264         var->present = !var->unusable;
3265         var->avl = (ar >> 12) & 1;
3266         var->l = (ar >> 13) & 1;
3267         var->db = (ar >> 14) & 1;
3268         var->g = (ar >> 15) & 1;
3269 }
3270
3271 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
3272 {
3273         struct kvm_segment s;
3274
3275         if (to_vmx(vcpu)->rmode.vm86_active) {
3276                 vmx_get_segment(vcpu, &s, seg);
3277                 return s.base;
3278         }
3279         return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
3280 }
3281
3282 int vmx_get_cpl(struct kvm_vcpu *vcpu)
3283 {
3284         struct vcpu_vmx *vmx = to_vmx(vcpu);
3285
3286         if (unlikely(vmx->rmode.vm86_active))
3287                 return 0;
3288         else {
3289                 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
3290                 return VMX_AR_DPL(ar);
3291         }
3292 }
3293
3294 static u32 vmx_segment_access_rights(struct kvm_segment *var)
3295 {
3296         u32 ar;
3297
3298         if (var->unusable || !var->present)
3299                 ar = 1 << 16;
3300         else {
3301                 ar = var->type & 15;
3302                 ar |= (var->s & 1) << 4;
3303                 ar |= (var->dpl & 3) << 5;
3304                 ar |= (var->present & 1) << 7;
3305                 ar |= (var->avl & 1) << 12;
3306                 ar |= (var->l & 1) << 13;
3307                 ar |= (var->db & 1) << 14;
3308                 ar |= (var->g & 1) << 15;
3309         }
3310
3311         return ar;
3312 }
3313
3314 void __vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
3315 {
3316         struct vcpu_vmx *vmx = to_vmx(vcpu);
3317         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3318
3319         vmx_segment_cache_clear(vmx);
3320
3321         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
3322                 vmx->rmode.segs[seg] = *var;
3323                 if (seg == VCPU_SREG_TR)
3324                         vmcs_write16(sf->selector, var->selector);
3325                 else if (var->s)
3326                         fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
3327                 return;
3328         }
3329
3330         vmcs_writel(sf->base, var->base);
3331         vmcs_write32(sf->limit, var->limit);
3332         vmcs_write16(sf->selector, var->selector);
3333
3334         /*
3335          *   Fix the "Accessed" bit in AR field of segment registers for older
3336          * qemu binaries.
3337          *   IA32 arch specifies that at the time of processor reset the
3338          * "Accessed" bit in the AR field of segment registers is 1. And qemu
3339          * is setting it to 0 in the userland code. This causes invalid guest
3340          * state vmexit when "unrestricted guest" mode is turned on.
3341          *    Fix for this setup issue in cpu_reset is being pushed in the qemu
3342          * tree. Newer qemu binaries with that qemu fix would not need this
3343          * kvm hack.
3344          */
3345         if (is_unrestricted_guest(vcpu) && (seg != VCPU_SREG_LDTR))
3346                 var->type |= 0x1; /* Accessed */
3347
3348         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
3349 }
3350
3351 static void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
3352 {
3353         __vmx_set_segment(vcpu, var, seg);
3354
3355         to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu);
3356 }
3357
3358 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
3359 {
3360         u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
3361
3362         *db = (ar >> 14) & 1;
3363         *l = (ar >> 13) & 1;
3364 }
3365
3366 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3367 {
3368         dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
3369         dt->address = vmcs_readl(GUEST_IDTR_BASE);
3370 }
3371
3372 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3373 {
3374         vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
3375         vmcs_writel(GUEST_IDTR_BASE, dt->address);
3376 }
3377
3378 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3379 {
3380         dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
3381         dt->address = vmcs_readl(GUEST_GDTR_BASE);
3382 }
3383
3384 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3385 {
3386         vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
3387         vmcs_writel(GUEST_GDTR_BASE, dt->address);
3388 }
3389
3390 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
3391 {
3392         struct kvm_segment var;
3393         u32 ar;
3394
3395         vmx_get_segment(vcpu, &var, seg);
3396         var.dpl = 0x3;
3397         if (seg == VCPU_SREG_CS)
3398                 var.type = 0x3;
3399         ar = vmx_segment_access_rights(&var);
3400
3401         if (var.base != (var.selector << 4))
3402                 return false;
3403         if (var.limit != 0xffff)
3404                 return false;
3405         if (ar != 0xf3)
3406                 return false;
3407
3408         return true;
3409 }
3410
3411 static bool code_segment_valid(struct kvm_vcpu *vcpu)
3412 {
3413         struct kvm_segment cs;
3414         unsigned int cs_rpl;
3415
3416         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3417         cs_rpl = cs.selector & SEGMENT_RPL_MASK;
3418
3419         if (cs.unusable)
3420                 return false;
3421         if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK))
3422                 return false;
3423         if (!cs.s)
3424                 return false;
3425         if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) {
3426                 if (cs.dpl > cs_rpl)
3427                         return false;
3428         } else {
3429                 if (cs.dpl != cs_rpl)
3430                         return false;
3431         }
3432         if (!cs.present)
3433                 return false;
3434
3435         /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
3436         return true;
3437 }
3438
3439 static bool stack_segment_valid(struct kvm_vcpu *vcpu)
3440 {
3441         struct kvm_segment ss;
3442         unsigned int ss_rpl;
3443
3444         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3445         ss_rpl = ss.selector & SEGMENT_RPL_MASK;
3446
3447         if (ss.unusable)
3448                 return true;
3449         if (ss.type != 3 && ss.type != 7)
3450                 return false;
3451         if (!ss.s)
3452                 return false;
3453         if (ss.dpl != ss_rpl) /* DPL != RPL */
3454                 return false;
3455         if (!ss.present)
3456                 return false;
3457
3458         return true;
3459 }
3460
3461 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
3462 {
3463         struct kvm_segment var;
3464         unsigned int rpl;
3465
3466         vmx_get_segment(vcpu, &var, seg);
3467         rpl = var.selector & SEGMENT_RPL_MASK;
3468
3469         if (var.unusable)
3470                 return true;
3471         if (!var.s)
3472                 return false;
3473         if (!var.present)
3474                 return false;
3475         if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) {
3476                 if (var.dpl < rpl) /* DPL < RPL */
3477                         return false;
3478         }
3479
3480         /* TODO: Add other members to kvm_segment_field to allow checking for other access
3481          * rights flags
3482          */
3483         return true;
3484 }
3485
3486 static bool tr_valid(struct kvm_vcpu *vcpu)
3487 {
3488         struct kvm_segment tr;
3489
3490         vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
3491
3492         if (tr.unusable)
3493                 return false;
3494         if (tr.selector & SEGMENT_TI_MASK)      /* TI = 1 */
3495                 return false;
3496         if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
3497                 return false;
3498         if (!tr.present)
3499                 return false;
3500
3501         return true;
3502 }
3503
3504 static bool ldtr_valid(struct kvm_vcpu *vcpu)
3505 {
3506         struct kvm_segment ldtr;
3507
3508         vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
3509
3510         if (ldtr.unusable)
3511                 return true;
3512         if (ldtr.selector & SEGMENT_TI_MASK)    /* TI = 1 */
3513                 return false;
3514         if (ldtr.type != 2)
3515                 return false;
3516         if (!ldtr.present)
3517                 return false;
3518
3519         return true;
3520 }
3521
3522 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
3523 {
3524         struct kvm_segment cs, ss;
3525
3526         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3527         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3528
3529         return ((cs.selector & SEGMENT_RPL_MASK) ==
3530                  (ss.selector & SEGMENT_RPL_MASK));
3531 }
3532
3533 /*
3534  * Check if guest state is valid. Returns true if valid, false if
3535  * not.
3536  * We assume that registers are always usable
3537  */
3538 bool __vmx_guest_state_valid(struct kvm_vcpu *vcpu)
3539 {
3540         /* real mode guest state checks */
3541         if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
3542                 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
3543                         return false;
3544                 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
3545                         return false;
3546                 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
3547                         return false;
3548                 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
3549                         return false;
3550                 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
3551                         return false;
3552                 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
3553                         return false;
3554         } else {
3555         /* protected mode guest state checks */
3556                 if (!cs_ss_rpl_check(vcpu))
3557                         return false;
3558                 if (!code_segment_valid(vcpu))
3559                         return false;
3560                 if (!stack_segment_valid(vcpu))
3561                         return false;
3562                 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
3563                         return false;
3564                 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
3565                         return false;
3566                 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
3567                         return false;
3568                 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
3569                         return false;
3570                 if (!tr_valid(vcpu))
3571                         return false;
3572                 if (!ldtr_valid(vcpu))
3573                         return false;
3574         }
3575         /* TODO:
3576          * - Add checks on RIP
3577          * - Add checks on RFLAGS
3578          */
3579
3580         return true;
3581 }
3582
3583 static int init_rmode_tss(struct kvm *kvm, void __user *ua)
3584 {
3585         const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
3586         u16 data;
3587         int i;
3588
3589         for (i = 0; i < 3; i++) {
3590                 if (__copy_to_user(ua + PAGE_SIZE * i, zero_page, PAGE_SIZE))
3591                         return -EFAULT;
3592         }
3593
3594         data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
3595         if (__copy_to_user(ua + TSS_IOPB_BASE_OFFSET, &data, sizeof(u16)))
3596                 return -EFAULT;
3597
3598         data = ~0;
3599         if (__copy_to_user(ua + RMODE_TSS_SIZE - 1, &data, sizeof(u8)))
3600                 return -EFAULT;
3601
3602         return 0;
3603 }
3604
3605 static int init_rmode_identity_map(struct kvm *kvm)
3606 {
3607         struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
3608         int i, r = 0;
3609         void __user *uaddr;
3610         u32 tmp;
3611
3612         /* Protect kvm_vmx->ept_identity_pagetable_done. */
3613         mutex_lock(&kvm->slots_lock);
3614
3615         if (likely(kvm_vmx->ept_identity_pagetable_done))
3616                 goto out;
3617
3618         if (!kvm_vmx->ept_identity_map_addr)
3619                 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR;
3620
3621         uaddr = __x86_set_memory_region(kvm,
3622                                         IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
3623                                         kvm_vmx->ept_identity_map_addr,
3624                                         PAGE_SIZE);
3625         if (IS_ERR(uaddr)) {
3626                 r = PTR_ERR(uaddr);
3627                 goto out;
3628         }
3629
3630         /* Set up identity-mapping pagetable for EPT in real mode */
3631         for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
3632                 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
3633                         _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
3634                 if (__copy_to_user(uaddr + i * sizeof(tmp), &tmp, sizeof(tmp))) {
3635                         r = -EFAULT;
3636                         goto out;
3637                 }
3638         }
3639         kvm_vmx->ept_identity_pagetable_done = true;
3640
3641 out:
3642         mutex_unlock(&kvm->slots_lock);
3643         return r;
3644 }
3645
3646 static void seg_setup(int seg)
3647 {
3648         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3649         unsigned int ar;
3650
3651         vmcs_write16(sf->selector, 0);
3652         vmcs_writel(sf->base, 0);
3653         vmcs_write32(sf->limit, 0xffff);
3654         ar = 0x93;
3655         if (seg == VCPU_SREG_CS)
3656                 ar |= 0x08; /* code segment */
3657
3658         vmcs_write32(sf->ar_bytes, ar);
3659 }
3660
3661 static int alloc_apic_access_page(struct kvm *kvm)
3662 {
3663         struct page *page;
3664         void __user *hva;
3665         int ret = 0;
3666
3667         mutex_lock(&kvm->slots_lock);
3668         if (kvm->arch.apic_access_memslot_enabled)
3669                 goto out;
3670         hva = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
3671                                       APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
3672         if (IS_ERR(hva)) {
3673                 ret = PTR_ERR(hva);
3674                 goto out;
3675         }
3676
3677         page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
3678         if (is_error_page(page)) {
3679                 ret = -EFAULT;
3680                 goto out;
3681         }
3682
3683         /*
3684          * Do not pin the page in memory, so that memory hot-unplug
3685          * is able to migrate it.
3686          */
3687         put_page(page);
3688         kvm->arch.apic_access_memslot_enabled = true;
3689 out:
3690         mutex_unlock(&kvm->slots_lock);
3691         return ret;
3692 }
3693
3694 int allocate_vpid(void)
3695 {
3696         int vpid;
3697
3698         if (!enable_vpid)
3699                 return 0;
3700         spin_lock(&vmx_vpid_lock);
3701         vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
3702         if (vpid < VMX_NR_VPIDS)
3703                 __set_bit(vpid, vmx_vpid_bitmap);
3704         else
3705                 vpid = 0;
3706         spin_unlock(&vmx_vpid_lock);
3707         return vpid;
3708 }
3709
3710 void free_vpid(int vpid)
3711 {
3712         if (!enable_vpid || vpid == 0)
3713                 return;
3714         spin_lock(&vmx_vpid_lock);
3715         __clear_bit(vpid, vmx_vpid_bitmap);
3716         spin_unlock(&vmx_vpid_lock);
3717 }
3718
3719 static void vmx_msr_bitmap_l01_changed(struct vcpu_vmx *vmx)
3720 {
3721         /*
3722          * When KVM is a nested hypervisor on top of Hyper-V and uses
3723          * 'Enlightened MSR Bitmap' feature L0 needs to know that MSR
3724          * bitmap has changed.
3725          */
3726         if (static_branch_unlikely(&enable_evmcs))
3727                 evmcs_touch_msr_bitmap();
3728
3729         vmx->nested.force_msr_bitmap_recalc = true;
3730 }
3731
3732 void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type)
3733 {
3734         struct vcpu_vmx *vmx = to_vmx(vcpu);
3735         unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
3736
3737         if (!cpu_has_vmx_msr_bitmap())
3738                 return;
3739
3740         vmx_msr_bitmap_l01_changed(vmx);
3741
3742         /*
3743          * Mark the desired intercept state in shadow bitmap, this is needed
3744          * for resync when the MSR filters change.
3745         */
3746         if (is_valid_passthrough_msr(msr)) {
3747                 int idx = possible_passthrough_msr_slot(msr);
3748
3749                 if (idx != -ENOENT) {
3750                         if (type & MSR_TYPE_R)
3751                                 clear_bit(idx, vmx->shadow_msr_intercept.read);
3752                         if (type & MSR_TYPE_W)
3753                                 clear_bit(idx, vmx->shadow_msr_intercept.write);
3754                 }
3755         }
3756
3757         if ((type & MSR_TYPE_R) &&
3758             !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) {
3759                 vmx_set_msr_bitmap_read(msr_bitmap, msr);
3760                 type &= ~MSR_TYPE_R;
3761         }
3762
3763         if ((type & MSR_TYPE_W) &&
3764             !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) {
3765                 vmx_set_msr_bitmap_write(msr_bitmap, msr);
3766                 type &= ~MSR_TYPE_W;
3767         }
3768
3769         if (type & MSR_TYPE_R)
3770                 vmx_clear_msr_bitmap_read(msr_bitmap, msr);
3771
3772         if (type & MSR_TYPE_W)
3773                 vmx_clear_msr_bitmap_write(msr_bitmap, msr);
3774 }
3775
3776 void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type)
3777 {
3778         struct vcpu_vmx *vmx = to_vmx(vcpu);
3779         unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
3780
3781         if (!cpu_has_vmx_msr_bitmap())
3782                 return;
3783
3784         vmx_msr_bitmap_l01_changed(vmx);
3785
3786         /*
3787          * Mark the desired intercept state in shadow bitmap, this is needed
3788          * for resync when the MSR filter changes.
3789         */
3790         if (is_valid_passthrough_msr(msr)) {
3791                 int idx = possible_passthrough_msr_slot(msr);
3792
3793                 if (idx != -ENOENT) {
3794                         if (type & MSR_TYPE_R)
3795                                 set_bit(idx, vmx->shadow_msr_intercept.read);
3796                         if (type & MSR_TYPE_W)
3797                                 set_bit(idx, vmx->shadow_msr_intercept.write);
3798                 }
3799         }
3800
3801         if (type & MSR_TYPE_R)
3802                 vmx_set_msr_bitmap_read(msr_bitmap, msr);
3803
3804         if (type & MSR_TYPE_W)
3805                 vmx_set_msr_bitmap_write(msr_bitmap, msr);
3806 }
3807
3808 static void vmx_reset_x2apic_msrs(struct kvm_vcpu *vcpu, u8 mode)
3809 {
3810         unsigned long *msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
3811         unsigned long read_intercept;
3812         int msr;
3813
3814         read_intercept = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0;
3815
3816         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
3817                 unsigned int read_idx = msr / BITS_PER_LONG;
3818                 unsigned int write_idx = read_idx + (0x800 / sizeof(long));
3819
3820                 msr_bitmap[read_idx] = read_intercept;
3821                 msr_bitmap[write_idx] = ~0ul;
3822         }
3823 }
3824
3825 static void vmx_update_msr_bitmap_x2apic(struct kvm_vcpu *vcpu)
3826 {
3827         struct vcpu_vmx *vmx = to_vmx(vcpu);
3828         u8 mode;
3829
3830         if (!cpu_has_vmx_msr_bitmap())
3831                 return;
3832
3833         if (cpu_has_secondary_exec_ctrls() &&
3834             (secondary_exec_controls_get(vmx) &
3835              SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
3836                 mode = MSR_BITMAP_MODE_X2APIC;
3837                 if (enable_apicv && kvm_vcpu_apicv_active(vcpu))
3838                         mode |= MSR_BITMAP_MODE_X2APIC_APICV;
3839         } else {
3840                 mode = 0;
3841         }
3842
3843         if (mode == vmx->x2apic_msr_bitmap_mode)
3844                 return;
3845
3846         vmx->x2apic_msr_bitmap_mode = mode;
3847
3848         vmx_reset_x2apic_msrs(vcpu, mode);
3849
3850         /*
3851          * TPR reads and writes can be virtualized even if virtual interrupt
3852          * delivery is not in use.
3853          */
3854         vmx_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW,
3855                                   !(mode & MSR_BITMAP_MODE_X2APIC));
3856
3857         if (mode & MSR_BITMAP_MODE_X2APIC_APICV) {
3858                 vmx_enable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_RW);
3859                 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_EOI), MSR_TYPE_W);
3860                 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W);
3861         }
3862 }
3863
3864 void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu)
3865 {
3866         struct vcpu_vmx *vmx = to_vmx(vcpu);
3867         bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN);
3868         u32 i;
3869
3870         vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_STATUS, MSR_TYPE_RW, flag);
3871         vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_BASE, MSR_TYPE_RW, flag);
3872         vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_MASK, MSR_TYPE_RW, flag);
3873         vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_CR3_MATCH, MSR_TYPE_RW, flag);
3874         for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) {
3875                 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag);
3876                 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag);
3877         }
3878 }
3879
3880 static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
3881 {
3882         struct vcpu_vmx *vmx = to_vmx(vcpu);
3883         void *vapic_page;
3884         u32 vppr;
3885         int rvi;
3886
3887         if (WARN_ON_ONCE(!is_guest_mode(vcpu)) ||
3888                 !nested_cpu_has_vid(get_vmcs12(vcpu)) ||
3889                 WARN_ON_ONCE(!vmx->nested.virtual_apic_map.gfn))
3890                 return false;
3891
3892         rvi = vmx_get_rvi();
3893
3894         vapic_page = vmx->nested.virtual_apic_map.hva;
3895         vppr = *((u32 *)(vapic_page + APIC_PROCPRI));
3896
3897         return ((rvi & 0xf0) > (vppr & 0xf0));
3898 }
3899
3900 static void vmx_msr_filter_changed(struct kvm_vcpu *vcpu)
3901 {
3902         struct vcpu_vmx *vmx = to_vmx(vcpu);
3903         u32 i;
3904
3905         /*
3906          * Set intercept permissions for all potentially passed through MSRs
3907          * again. They will automatically get filtered through the MSR filter,
3908          * so we are back in sync after this.
3909          */
3910         for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++) {
3911                 u32 msr = vmx_possible_passthrough_msrs[i];
3912                 bool read = test_bit(i, vmx->shadow_msr_intercept.read);
3913                 bool write = test_bit(i, vmx->shadow_msr_intercept.write);
3914
3915                 vmx_set_intercept_for_msr(vcpu, msr, MSR_TYPE_R, read);
3916                 vmx_set_intercept_for_msr(vcpu, msr, MSR_TYPE_W, write);
3917         }
3918
3919         pt_update_intercept_for_msr(vcpu);
3920 }
3921
3922 static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
3923                                                      int pi_vec)
3924 {
3925 #ifdef CONFIG_SMP
3926         if (vcpu->mode == IN_GUEST_MODE) {
3927                 /*
3928                  * The vector of the virtual has already been set in the PIR.
3929                  * Send a notification event to deliver the virtual interrupt
3930                  * unless the vCPU is the currently running vCPU, i.e. the
3931                  * event is being sent from a fastpath VM-Exit handler, in
3932                  * which case the PIR will be synced to the vIRR before
3933                  * re-entering the guest.
3934                  *
3935                  * When the target is not the running vCPU, the following
3936                  * possibilities emerge:
3937                  *
3938                  * Case 1: vCPU stays in non-root mode. Sending a notification
3939                  * event posts the interrupt to the vCPU.
3940                  *
3941                  * Case 2: vCPU exits to root mode and is still runnable. The
3942                  * PIR will be synced to the vIRR before re-entering the guest.
3943                  * Sending a notification event is ok as the host IRQ handler
3944                  * will ignore the spurious event.
3945                  *
3946                  * Case 3: vCPU exits to root mode and is blocked. vcpu_block()
3947                  * has already synced PIR to vIRR and never blocks the vCPU if
3948                  * the vIRR is not empty. Therefore, a blocked vCPU here does
3949                  * not wait for any requested interrupts in PIR, and sending a
3950                  * notification event also results in a benign, spurious event.
3951                  */
3952
3953                 if (vcpu != kvm_get_running_vcpu())
3954                         apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
3955                 return;
3956         }
3957 #endif
3958         /*
3959          * The vCPU isn't in the guest; wake the vCPU in case it is blocking,
3960          * otherwise do nothing as KVM will grab the highest priority pending
3961          * IRQ via ->sync_pir_to_irr() in vcpu_enter_guest().
3962          */
3963         kvm_vcpu_wake_up(vcpu);
3964 }
3965
3966 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
3967                                                 int vector)
3968 {
3969         struct vcpu_vmx *vmx = to_vmx(vcpu);
3970
3971         if (is_guest_mode(vcpu) &&
3972             vector == vmx->nested.posted_intr_nv) {
3973                 /*
3974                  * If a posted intr is not recognized by hardware,
3975                  * we will accomplish it in the next vmentry.
3976                  */
3977                 vmx->nested.pi_pending = true;
3978                 kvm_make_request(KVM_REQ_EVENT, vcpu);
3979
3980                 /*
3981                  * This pairs with the smp_mb_*() after setting vcpu->mode in
3982                  * vcpu_enter_guest() to guarantee the vCPU sees the event
3983                  * request if triggering a posted interrupt "fails" because
3984                  * vcpu->mode != IN_GUEST_MODE.  The extra barrier is needed as
3985                  * the smb_wmb() in kvm_make_request() only ensures everything
3986                  * done before making the request is visible when the request
3987                  * is visible, it doesn't ensure ordering between the store to
3988                  * vcpu->requests and the load from vcpu->mode.
3989                  */
3990                 smp_mb__after_atomic();
3991
3992                 /* the PIR and ON have been set by L1. */
3993                 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_NESTED_VECTOR);
3994                 return 0;
3995         }
3996         return -1;
3997 }
3998 /*
3999  * Send interrupt to vcpu via posted interrupt way.
4000  * 1. If target vcpu is running(non-root mode), send posted interrupt
4001  * notification to vcpu and hardware will sync PIR to vIRR atomically.
4002  * 2. If target vcpu isn't running(root mode), kick it to pick up the
4003  * interrupt from PIR in next vmentry.
4004  */
4005 static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
4006 {
4007         struct vcpu_vmx *vmx = to_vmx(vcpu);
4008         int r;
4009
4010         r = vmx_deliver_nested_posted_interrupt(vcpu, vector);
4011         if (!r)
4012                 return 0;
4013
4014         if (!vcpu->arch.apicv_active)
4015                 return -1;
4016
4017         if (pi_test_and_set_pir(vector, &vmx->pi_desc))
4018                 return 0;
4019
4020         /* If a previous notification has sent the IPI, nothing to do.  */
4021         if (pi_test_and_set_on(&vmx->pi_desc))
4022                 return 0;
4023
4024         /*
4025          * The implied barrier in pi_test_and_set_on() pairs with the smp_mb_*()
4026          * after setting vcpu->mode in vcpu_enter_guest(), thus the vCPU is
4027          * guaranteed to see PID.ON=1 and sync the PIR to IRR if triggering a
4028          * posted interrupt "fails" because vcpu->mode != IN_GUEST_MODE.
4029          */
4030         kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR);
4031         return 0;
4032 }
4033
4034 static void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
4035                                   int trig_mode, int vector)
4036 {
4037         struct kvm_vcpu *vcpu = apic->vcpu;
4038
4039         if (vmx_deliver_posted_interrupt(vcpu, vector)) {
4040                 kvm_lapic_set_irr(vector, apic);
4041                 kvm_make_request(KVM_REQ_EVENT, vcpu);
4042                 kvm_vcpu_kick(vcpu);
4043         } else {
4044                 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode,
4045                                            trig_mode, vector);
4046         }
4047 }
4048
4049 /*
4050  * Set up the vmcs's constant host-state fields, i.e., host-state fields that
4051  * will not change in the lifetime of the guest.
4052  * Note that host-state that does change is set elsewhere. E.g., host-state
4053  * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
4054  */
4055 void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
4056 {
4057         u32 low32, high32;
4058         unsigned long tmpl;
4059         unsigned long cr0, cr3, cr4;
4060
4061         cr0 = read_cr0();
4062         WARN_ON(cr0 & X86_CR0_TS);
4063         vmcs_writel(HOST_CR0, cr0);  /* 22.2.3 */
4064
4065         /*
4066          * Save the most likely value for this task's CR3 in the VMCS.
4067          * We can't use __get_current_cr3_fast() because we're not atomic.
4068          */
4069         cr3 = __read_cr3();
4070         vmcs_writel(HOST_CR3, cr3);             /* 22.2.3  FIXME: shadow tables */
4071         vmx->loaded_vmcs->host_state.cr3 = cr3;
4072
4073         /* Save the most likely value for this task's CR4 in the VMCS. */
4074         cr4 = cr4_read_shadow();
4075         vmcs_writel(HOST_CR4, cr4);                     /* 22.2.3, 22.2.5 */
4076         vmx->loaded_vmcs->host_state.cr4 = cr4;
4077
4078         vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
4079 #ifdef CONFIG_X86_64
4080         /*
4081          * Load null selectors, so we can avoid reloading them in
4082          * vmx_prepare_switch_to_host(), in case userspace uses
4083          * the null selectors too (the expected case).
4084          */
4085         vmcs_write16(HOST_DS_SELECTOR, 0);
4086         vmcs_write16(HOST_ES_SELECTOR, 0);
4087 #else
4088         vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
4089         vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
4090 #endif
4091         vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
4092         vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */
4093
4094         vmcs_writel(HOST_IDTR_BASE, host_idt_base);   /* 22.2.4 */
4095
4096         vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */
4097
4098         rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
4099         vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
4100
4101         /*
4102          * SYSENTER is used for 32-bit system calls on either 32-bit or
4103          * 64-bit kernels.  It is always zero If neither is allowed, otherwise
4104          * vmx_vcpu_load_vmcs loads it with the per-CPU entry stack (and may
4105          * have already done so!).
4106          */
4107         if (!IS_ENABLED(CONFIG_IA32_EMULATION) && !IS_ENABLED(CONFIG_X86_32))
4108                 vmcs_writel(HOST_IA32_SYSENTER_ESP, 0);
4109
4110         rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
4111         vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */
4112
4113         if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
4114                 rdmsr(MSR_IA32_CR_PAT, low32, high32);
4115                 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
4116         }
4117
4118         if (cpu_has_load_ia32_efer())
4119                 vmcs_write64(HOST_IA32_EFER, host_efer);
4120 }
4121
4122 void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
4123 {
4124         struct kvm_vcpu *vcpu = &vmx->vcpu;
4125
4126         vcpu->arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS &
4127                                           ~vcpu->arch.cr4_guest_rsvd_bits;
4128         if (!enable_ept) {
4129                 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_TLBFLUSH_BITS;
4130                 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PDPTR_BITS;
4131         }
4132         if (is_guest_mode(&vmx->vcpu))
4133                 vcpu->arch.cr4_guest_owned_bits &=
4134                         ~get_vmcs12(vcpu)->cr4_guest_host_mask;
4135         vmcs_writel(CR4_GUEST_HOST_MASK, ~vcpu->arch.cr4_guest_owned_bits);
4136 }
4137
4138 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
4139 {
4140         u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
4141
4142         if (!kvm_vcpu_apicv_active(&vmx->vcpu))
4143                 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
4144
4145         if (!enable_vnmi)
4146                 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS;
4147
4148         if (!enable_preemption_timer)
4149                 pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
4150
4151         return pin_based_exec_ctrl;
4152 }
4153
4154 static u32 vmx_vmentry_ctrl(void)
4155 {
4156         u32 vmentry_ctrl = vmcs_config.vmentry_ctrl;
4157
4158         if (vmx_pt_mode_is_system())
4159                 vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP |
4160                                   VM_ENTRY_LOAD_IA32_RTIT_CTL);
4161         /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */
4162         return vmentry_ctrl &
4163                 ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | VM_ENTRY_LOAD_IA32_EFER);
4164 }
4165
4166 static u32 vmx_vmexit_ctrl(void)
4167 {
4168         u32 vmexit_ctrl = vmcs_config.vmexit_ctrl;
4169
4170         if (vmx_pt_mode_is_system())
4171                 vmexit_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP |
4172                                  VM_EXIT_CLEAR_IA32_RTIT_CTL);
4173         /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */
4174         return vmexit_ctrl &
4175                 ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER);
4176 }
4177
4178 static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
4179 {
4180         struct vcpu_vmx *vmx = to_vmx(vcpu);
4181
4182         pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));
4183         if (cpu_has_secondary_exec_ctrls()) {
4184                 if (kvm_vcpu_apicv_active(vcpu))
4185                         secondary_exec_controls_setbit(vmx,
4186                                       SECONDARY_EXEC_APIC_REGISTER_VIRT |
4187                                       SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
4188                 else
4189                         secondary_exec_controls_clearbit(vmx,
4190                                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
4191                                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
4192         }
4193
4194         vmx_update_msr_bitmap_x2apic(vcpu);
4195 }
4196
4197 static u32 vmx_exec_control(struct vcpu_vmx *vmx)
4198 {
4199         u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
4200
4201         if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
4202                 exec_control &= ~CPU_BASED_MOV_DR_EXITING;
4203
4204         if (!cpu_need_tpr_shadow(&vmx->vcpu)) {
4205                 exec_control &= ~CPU_BASED_TPR_SHADOW;
4206 #ifdef CONFIG_X86_64
4207                 exec_control |= CPU_BASED_CR8_STORE_EXITING |
4208                                 CPU_BASED_CR8_LOAD_EXITING;
4209 #endif
4210         }
4211         if (!enable_ept)
4212                 exec_control |= CPU_BASED_CR3_STORE_EXITING |
4213                                 CPU_BASED_CR3_LOAD_EXITING  |
4214                                 CPU_BASED_INVLPG_EXITING;
4215         if (kvm_mwait_in_guest(vmx->vcpu.kvm))
4216                 exec_control &= ~(CPU_BASED_MWAIT_EXITING |
4217                                 CPU_BASED_MONITOR_EXITING);
4218         if (kvm_hlt_in_guest(vmx->vcpu.kvm))
4219                 exec_control &= ~CPU_BASED_HLT_EXITING;
4220         return exec_control;
4221 }
4222
4223 /*
4224  * Adjust a single secondary execution control bit to intercept/allow an
4225  * instruction in the guest.  This is usually done based on whether or not a
4226  * feature has been exposed to the guest in order to correctly emulate faults.
4227  */
4228 static inline void
4229 vmx_adjust_secondary_exec_control(struct vcpu_vmx *vmx, u32 *exec_control,
4230                                   u32 control, bool enabled, bool exiting)
4231 {
4232         /*
4233          * If the control is for an opt-in feature, clear the control if the
4234          * feature is not exposed to the guest, i.e. not enabled.  If the
4235          * control is opt-out, i.e. an exiting control, clear the control if
4236          * the feature _is_ exposed to the guest, i.e. exiting/interception is
4237          * disabled for the associated instruction.  Note, the caller is
4238          * responsible presetting exec_control to set all supported bits.
4239          */
4240         if (enabled == exiting)
4241                 *exec_control &= ~control;
4242
4243         /*
4244          * Update the nested MSR settings so that a nested VMM can/can't set
4245          * controls for features that are/aren't exposed to the guest.
4246          */
4247         if (nested) {
4248                 if (enabled)
4249                         vmx->nested.msrs.secondary_ctls_high |= control;
4250                 else
4251                         vmx->nested.msrs.secondary_ctls_high &= ~control;
4252         }
4253 }
4254
4255 /*
4256  * Wrapper macro for the common case of adjusting a secondary execution control
4257  * based on a single guest CPUID bit, with a dedicated feature bit.  This also
4258  * verifies that the control is actually supported by KVM and hardware.
4259  */
4260 #define vmx_adjust_sec_exec_control(vmx, exec_control, name, feat_name, ctrl_name, exiting) \
4261 ({                                                                       \
4262         bool __enabled;                                                  \
4263                                                                          \
4264         if (cpu_has_vmx_##name()) {                                      \
4265                 __enabled = guest_cpuid_has(&(vmx)->vcpu,                \
4266                                             X86_FEATURE_##feat_name);    \
4267                 vmx_adjust_secondary_exec_control(vmx, exec_control,     \
4268                         SECONDARY_EXEC_##ctrl_name, __enabled, exiting); \
4269         }                                                                \
4270 })
4271
4272 /* More macro magic for ENABLE_/opt-in versus _EXITING/opt-out controls. */
4273 #define vmx_adjust_sec_exec_feature(vmx, exec_control, lname, uname) \
4274         vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, ENABLE_##uname, false)
4275
4276 #define vmx_adjust_sec_exec_exiting(vmx, exec_control, lname, uname) \
4277         vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, uname##_EXITING, true)
4278
4279 static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
4280 {
4281         struct kvm_vcpu *vcpu = &vmx->vcpu;
4282
4283         u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
4284
4285         if (vmx_pt_mode_is_system())
4286                 exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX);
4287         if (!cpu_need_virtualize_apic_accesses(vcpu))
4288                 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
4289         if (vmx->vpid == 0)
4290                 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
4291         if (!enable_ept) {
4292                 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
4293                 enable_unrestricted_guest = 0;
4294         }
4295         if (!enable_unrestricted_guest)
4296                 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
4297         if (kvm_pause_in_guest(vmx->vcpu.kvm))
4298                 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
4299         if (!kvm_vcpu_apicv_active(vcpu))
4300                 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
4301                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
4302         exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
4303
4304         /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP,
4305          * in vmx_set_cr4.  */
4306         exec_control &= ~SECONDARY_EXEC_DESC;
4307
4308         /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
4309            (handle_vmptrld).
4310            We can NOT enable shadow_vmcs here because we don't have yet
4311            a current VMCS12
4312         */
4313         exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
4314
4315         /*
4316          * PML is enabled/disabled when dirty logging of memsmlots changes, but
4317          * it needs to be set here when dirty logging is already active, e.g.
4318          * if this vCPU was created after dirty logging was enabled.
4319          */
4320         if (!vcpu->kvm->arch.cpu_dirty_logging_count)
4321                 exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
4322
4323         if (cpu_has_vmx_xsaves()) {
4324                 /* Exposing XSAVES only when XSAVE is exposed */
4325                 bool xsaves_enabled =
4326                         boot_cpu_has(X86_FEATURE_XSAVE) &&
4327                         guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
4328                         guest_cpuid_has(vcpu, X86_FEATURE_XSAVES);
4329
4330                 vcpu->arch.xsaves_enabled = xsaves_enabled;
4331
4332                 vmx_adjust_secondary_exec_control(vmx, &exec_control,
4333                                                   SECONDARY_EXEC_XSAVES,
4334                                                   xsaves_enabled, false);
4335         }
4336
4337         /*
4338          * RDPID is also gated by ENABLE_RDTSCP, turn on the control if either
4339          * feature is exposed to the guest.  This creates a virtualization hole
4340          * if both are supported in hardware but only one is exposed to the
4341          * guest, but letting the guest execute RDTSCP or RDPID when either one
4342          * is advertised is preferable to emulating the advertised instruction
4343          * in KVM on #UD, and obviously better than incorrectly injecting #UD.
4344          */
4345         if (cpu_has_vmx_rdtscp()) {
4346                 bool rdpid_or_rdtscp_enabled =
4347                         guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) ||
4348                         guest_cpuid_has(vcpu, X86_FEATURE_RDPID);
4349
4350                 vmx_adjust_secondary_exec_control(vmx, &exec_control,
4351                                                   SECONDARY_EXEC_ENABLE_RDTSCP,
4352                                                   rdpid_or_rdtscp_enabled, false);
4353         }
4354         vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID);
4355
4356         vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND);
4357         vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdseed, RDSEED);
4358
4359         vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG,
4360                                     ENABLE_USR_WAIT_PAUSE, false);
4361
4362         if (!vcpu->kvm->arch.bus_lock_detection_enabled)
4363                 exec_control &= ~SECONDARY_EXEC_BUS_LOCK_DETECTION;
4364
4365         return exec_control;
4366 }
4367
4368 #define VMX_XSS_EXIT_BITMAP 0
4369
4370 static void init_vmcs(struct vcpu_vmx *vmx)
4371 {
4372         if (nested)
4373                 nested_vmx_set_vmcs_shadowing_bitmap();
4374
4375         if (cpu_has_vmx_msr_bitmap())
4376                 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap));
4377
4378         vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); /* 22.3.1.5 */
4379
4380         /* Control */
4381         pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));
4382
4383         exec_controls_set(vmx, vmx_exec_control(vmx));
4384
4385         if (cpu_has_secondary_exec_ctrls())
4386                 secondary_exec_controls_set(vmx, vmx_secondary_exec_control(vmx));
4387
4388         if (kvm_vcpu_apicv_active(&vmx->vcpu)) {
4389                 vmcs_write64(EOI_EXIT_BITMAP0, 0);
4390                 vmcs_write64(EOI_EXIT_BITMAP1, 0);
4391                 vmcs_write64(EOI_EXIT_BITMAP2, 0);
4392                 vmcs_write64(EOI_EXIT_BITMAP3, 0);
4393
4394                 vmcs_write16(GUEST_INTR_STATUS, 0);
4395
4396                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
4397                 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
4398         }
4399
4400         if (!kvm_pause_in_guest(vmx->vcpu.kvm)) {
4401                 vmcs_write32(PLE_GAP, ple_gap);
4402                 vmx->ple_window = ple_window;
4403                 vmx->ple_window_dirty = true;
4404         }
4405
4406         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
4407         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
4408         vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */
4409
4410         vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
4411         vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
4412         vmx_set_constant_host_state(vmx);
4413         vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
4414         vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
4415
4416         if (cpu_has_vmx_vmfunc())
4417                 vmcs_write64(VM_FUNCTION_CONTROL, 0);
4418
4419         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
4420         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
4421         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
4422         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
4423         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
4424
4425         if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
4426                 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
4427
4428         vm_exit_controls_set(vmx, vmx_vmexit_ctrl());
4429
4430         /* 22.2.1, 20.8.1 */
4431         vm_entry_controls_set(vmx, vmx_vmentry_ctrl());
4432
4433         vmx->vcpu.arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS;
4434         vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits);
4435
4436         set_cr4_guest_host_mask(vmx);
4437
4438         if (vmx->vpid != 0)
4439                 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
4440
4441         if (cpu_has_vmx_xsaves())
4442                 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);
4443
4444         if (enable_pml) {
4445                 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
4446                 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
4447         }
4448
4449         vmx_write_encls_bitmap(&vmx->vcpu, NULL);
4450
4451         if (vmx_pt_mode_is_host_guest()) {
4452                 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc));
4453                 /* Bit[6~0] are forced to 1, writes are ignored. */
4454                 vmx->pt_desc.guest.output_mask = 0x7F;
4455                 vmcs_write64(GUEST_IA32_RTIT_CTL, 0);
4456         }
4457
4458         vmcs_write32(GUEST_SYSENTER_CS, 0);
4459         vmcs_writel(GUEST_SYSENTER_ESP, 0);
4460         vmcs_writel(GUEST_SYSENTER_EIP, 0);
4461         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
4462
4463         if (cpu_has_vmx_tpr_shadow()) {
4464                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
4465                 if (cpu_need_tpr_shadow(&vmx->vcpu))
4466                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
4467                                      __pa(vmx->vcpu.arch.apic->regs));
4468                 vmcs_write32(TPR_THRESHOLD, 0);
4469         }
4470
4471         vmx_setup_uret_msrs(vmx);
4472 }
4473
4474 static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu)
4475 {
4476         struct vcpu_vmx *vmx = to_vmx(vcpu);
4477
4478         init_vmcs(vmx);
4479
4480         if (nested)
4481                 memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs));
4482
4483         vcpu_setup_sgx_lepubkeyhash(vcpu);
4484
4485         vmx->nested.posted_intr_nv = -1;
4486         vmx->nested.vmxon_ptr = INVALID_GPA;
4487         vmx->nested.current_vmptr = INVALID_GPA;
4488         vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID;
4489
4490         vcpu->arch.microcode_version = 0x100000000ULL;
4491         vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED;
4492
4493         /*
4494          * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR
4495          * or POSTED_INTR_WAKEUP_VECTOR.
4496          */
4497         vmx->pi_desc.nv = POSTED_INTR_VECTOR;
4498         vmx->pi_desc.sn = 1;
4499 }
4500
4501 static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
4502 {
4503         struct vcpu_vmx *vmx = to_vmx(vcpu);
4504
4505         if (!init_event)
4506                 __vmx_vcpu_reset(vcpu);
4507
4508         vmx->rmode.vm86_active = 0;
4509         vmx->spec_ctrl = 0;
4510
4511         vmx->msr_ia32_umwait_control = 0;
4512
4513         vmx->hv_deadline_tsc = -1;
4514         kvm_set_cr8(vcpu, 0);
4515
4516         vmx_segment_cache_clear(vmx);
4517         kvm_register_mark_available(vcpu, VCPU_EXREG_SEGMENTS);
4518
4519         seg_setup(VCPU_SREG_CS);
4520         vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
4521         vmcs_writel(GUEST_CS_BASE, 0xffff0000ul);
4522
4523         seg_setup(VCPU_SREG_DS);
4524         seg_setup(VCPU_SREG_ES);
4525         seg_setup(VCPU_SREG_FS);
4526         seg_setup(VCPU_SREG_GS);
4527         seg_setup(VCPU_SREG_SS);
4528
4529         vmcs_write16(GUEST_TR_SELECTOR, 0);
4530         vmcs_writel(GUEST_TR_BASE, 0);
4531         vmcs_write32(GUEST_TR_LIMIT, 0xffff);
4532         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
4533
4534         vmcs_write16(GUEST_LDTR_SELECTOR, 0);
4535         vmcs_writel(GUEST_LDTR_BASE, 0);
4536         vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
4537         vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
4538
4539         vmcs_writel(GUEST_GDTR_BASE, 0);
4540         vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
4541
4542         vmcs_writel(GUEST_IDTR_BASE, 0);
4543         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
4544
4545         vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
4546         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
4547         vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0);
4548         if (kvm_mpx_supported())
4549                 vmcs_write64(GUEST_BNDCFGS, 0);
4550
4551         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
4552
4553         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
4554
4555         vpid_sync_context(vmx->vpid);
4556 }
4557
4558 static void vmx_enable_irq_window(struct kvm_vcpu *vcpu)
4559 {
4560         exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);
4561 }
4562
4563 static void vmx_enable_nmi_window(struct kvm_vcpu *vcpu)
4564 {
4565         if (!enable_vnmi ||
4566             vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
4567                 vmx_enable_irq_window(vcpu);
4568                 return;
4569         }
4570
4571         exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING);
4572 }
4573
4574 static void vmx_inject_irq(struct kvm_vcpu *vcpu)
4575 {
4576         struct vcpu_vmx *vmx = to_vmx(vcpu);
4577         uint32_t intr;
4578         int irq = vcpu->arch.interrupt.nr;
4579
4580         trace_kvm_inj_virq(irq);
4581
4582         ++vcpu->stat.irq_injections;
4583         if (vmx->rmode.vm86_active) {
4584                 int inc_eip = 0;
4585                 if (vcpu->arch.interrupt.soft)
4586                         inc_eip = vcpu->arch.event_exit_inst_len;
4587                 kvm_inject_realmode_interrupt(vcpu, irq, inc_eip);
4588                 return;
4589         }
4590         intr = irq | INTR_INFO_VALID_MASK;
4591         if (vcpu->arch.interrupt.soft) {
4592                 intr |= INTR_TYPE_SOFT_INTR;
4593                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
4594                              vmx->vcpu.arch.event_exit_inst_len);
4595         } else
4596                 intr |= INTR_TYPE_EXT_INTR;
4597         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
4598
4599         vmx_clear_hlt(vcpu);
4600 }
4601
4602 static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
4603 {
4604         struct vcpu_vmx *vmx = to_vmx(vcpu);
4605
4606         if (!enable_vnmi) {
4607                 /*
4608                  * Tracking the NMI-blocked state in software is built upon
4609                  * finding the next open IRQ window. This, in turn, depends on
4610                  * well-behaving guests: They have to keep IRQs disabled at
4611                  * least as long as the NMI handler runs. Otherwise we may
4612                  * cause NMI nesting, maybe breaking the guest. But as this is
4613                  * highly unlikely, we can live with the residual risk.
4614                  */
4615                 vmx->loaded_vmcs->soft_vnmi_blocked = 1;
4616                 vmx->loaded_vmcs->vnmi_blocked_time = 0;
4617         }
4618
4619         ++vcpu->stat.nmi_injections;
4620         vmx->loaded_vmcs->nmi_known_unmasked = false;
4621
4622         if (vmx->rmode.vm86_active) {
4623                 kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0);
4624                 return;
4625         }
4626
4627         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
4628                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
4629
4630         vmx_clear_hlt(vcpu);
4631 }
4632
4633 bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
4634 {
4635         struct vcpu_vmx *vmx = to_vmx(vcpu);
4636         bool masked;
4637
4638         if (!enable_vnmi)
4639                 return vmx->loaded_vmcs->soft_vnmi_blocked;
4640         if (vmx->loaded_vmcs->nmi_known_unmasked)
4641                 return false;
4642         masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI;
4643         vmx->loaded_vmcs->nmi_known_unmasked = !masked;
4644         return masked;
4645 }
4646
4647 void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
4648 {
4649         struct vcpu_vmx *vmx = to_vmx(vcpu);
4650
4651         if (!enable_vnmi) {
4652                 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) {
4653                         vmx->loaded_vmcs->soft_vnmi_blocked = masked;
4654                         vmx->loaded_vmcs->vnmi_blocked_time = 0;
4655                 }
4656         } else {
4657                 vmx->loaded_vmcs->nmi_known_unmasked = !masked;
4658                 if (masked)
4659                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
4660                                       GUEST_INTR_STATE_NMI);
4661                 else
4662                         vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
4663                                         GUEST_INTR_STATE_NMI);
4664         }
4665 }
4666
4667 bool vmx_nmi_blocked(struct kvm_vcpu *vcpu)
4668 {
4669         if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu))
4670                 return false;
4671
4672         if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked)
4673                 return true;
4674
4675         return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
4676                 (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI |
4677                  GUEST_INTR_STATE_NMI));
4678 }
4679
4680 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4681 {
4682         if (to_vmx(vcpu)->nested.nested_run_pending)
4683                 return -EBUSY;
4684
4685         /* An NMI must not be injected into L2 if it's supposed to VM-Exit.  */
4686         if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu))
4687                 return -EBUSY;
4688
4689         return !vmx_nmi_blocked(vcpu);
4690 }
4691
4692 bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu)
4693 {
4694         if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu))
4695                 return false;
4696
4697         return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) ||
4698                (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
4699                 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
4700 }
4701
4702 static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4703 {
4704         if (to_vmx(vcpu)->nested.nested_run_pending)
4705                 return -EBUSY;
4706
4707        /*
4708         * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
4709         * e.g. if the IRQ arrived asynchronously after checking nested events.
4710         */
4711         if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu))
4712                 return -EBUSY;
4713
4714         return !vmx_interrupt_blocked(vcpu);
4715 }
4716
4717 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
4718 {
4719         void __user *ret;
4720
4721         if (enable_unrestricted_guest)
4722                 return 0;
4723
4724         mutex_lock(&kvm->slots_lock);
4725         ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr,
4726                                       PAGE_SIZE * 3);
4727         mutex_unlock(&kvm->slots_lock);
4728
4729         if (IS_ERR(ret))
4730                 return PTR_ERR(ret);
4731
4732         to_kvm_vmx(kvm)->tss_addr = addr;
4733
4734         return init_rmode_tss(kvm, ret);
4735 }
4736
4737 static int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
4738 {
4739         to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr;
4740         return 0;
4741 }
4742
4743 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
4744 {
4745         switch (vec) {
4746         case BP_VECTOR:
4747                 /*
4748                  * Update instruction length as we may reinject the exception
4749                  * from user space while in guest debugging mode.
4750                  */
4751                 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
4752                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
4753                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
4754                         return false;
4755                 fallthrough;
4756         case DB_VECTOR:
4757                 return !(vcpu->guest_debug &
4758                         (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP));
4759         case DE_VECTOR:
4760         case OF_VECTOR:
4761         case BR_VECTOR:
4762         case UD_VECTOR:
4763         case DF_VECTOR:
4764         case SS_VECTOR:
4765         case GP_VECTOR:
4766         case MF_VECTOR:
4767                 return true;
4768         }
4769         return false;
4770 }
4771
4772 static int handle_rmode_exception(struct kvm_vcpu *vcpu,
4773                                   int vec, u32 err_code)
4774 {
4775         /*
4776          * Instruction with address size override prefix opcode 0x67
4777          * Cause the #SS fault with 0 error code in VM86 mode.
4778          */
4779         if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
4780                 if (kvm_emulate_instruction(vcpu, 0)) {
4781                         if (vcpu->arch.halt_request) {
4782                                 vcpu->arch.halt_request = 0;
4783                                 return kvm_emulate_halt_noskip(vcpu);
4784                         }
4785                         return 1;
4786                 }
4787                 return 0;
4788         }
4789
4790         /*
4791          * Forward all other exceptions that are valid in real mode.
4792          * FIXME: Breaks guest debugging in real mode, needs to be fixed with
4793          *        the required debugging infrastructure rework.
4794          */
4795         kvm_queue_exception(vcpu, vec);
4796         return 1;
4797 }
4798
4799 static int handle_machine_check(struct kvm_vcpu *vcpu)
4800 {
4801         /* handled by vmx_vcpu_run() */
4802         return 1;
4803 }
4804
4805 /*
4806  * If the host has split lock detection disabled, then #AC is
4807  * unconditionally injected into the guest, which is the pre split lock
4808  * detection behaviour.
4809  *
4810  * If the host has split lock detection enabled then #AC is
4811  * only injected into the guest when:
4812  *  - Guest CPL == 3 (user mode)
4813  *  - Guest has #AC detection enabled in CR0
4814  *  - Guest EFLAGS has AC bit set
4815  */
4816 bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu)
4817 {
4818         if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
4819                 return true;
4820
4821         return vmx_get_cpl(vcpu) == 3 && kvm_read_cr0_bits(vcpu, X86_CR0_AM) &&
4822                (kvm_get_rflags(vcpu) & X86_EFLAGS_AC);
4823 }
4824
4825 static int handle_exception_nmi(struct kvm_vcpu *vcpu)
4826 {
4827         struct vcpu_vmx *vmx = to_vmx(vcpu);
4828         struct kvm_run *kvm_run = vcpu->run;
4829         u32 intr_info, ex_no, error_code;
4830         unsigned long cr2, dr6;
4831         u32 vect_info;
4832
4833         vect_info = vmx->idt_vectoring_info;
4834         intr_info = vmx_get_intr_info(vcpu);
4835
4836         if (is_machine_check(intr_info) || is_nmi(intr_info))
4837                 return 1; /* handled by handle_exception_nmi_irqoff() */
4838
4839         /*
4840          * Queue the exception here instead of in handle_nm_fault_irqoff().
4841          * This ensures the nested_vmx check is not skipped so vmexit can
4842          * be reflected to L1 (when it intercepts #NM) before reaching this
4843          * point.
4844          */
4845         if (is_nm_fault(intr_info)) {
4846                 kvm_queue_exception(vcpu, NM_VECTOR);
4847                 return 1;
4848         }
4849
4850         if (is_invalid_opcode(intr_info))
4851                 return handle_ud(vcpu);
4852
4853         error_code = 0;
4854         if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
4855                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
4856
4857         if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) {
4858                 WARN_ON_ONCE(!enable_vmware_backdoor);
4859
4860                 /*
4861                  * VMware backdoor emulation on #GP interception only handles
4862                  * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero
4863                  * error code on #GP.
4864                  */
4865                 if (error_code) {
4866                         kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
4867                         return 1;
4868                 }
4869                 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP);
4870         }
4871
4872         /*
4873          * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
4874          * MMIO, it is better to report an internal error.
4875          * See the comments in vmx_handle_exit.
4876          */
4877         if ((vect_info & VECTORING_INFO_VALID_MASK) &&
4878             !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
4879                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4880                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
4881                 vcpu->run->internal.ndata = 4;
4882                 vcpu->run->internal.data[0] = vect_info;
4883                 vcpu->run->internal.data[1] = intr_info;
4884                 vcpu->run->internal.data[2] = error_code;
4885                 vcpu->run->internal.data[3] = vcpu->arch.last_vmentry_cpu;
4886                 return 0;
4887         }
4888
4889         if (is_page_fault(intr_info)) {
4890                 cr2 = vmx_get_exit_qual(vcpu);
4891                 if (enable_ept && !vcpu->arch.apf.host_apf_flags) {
4892                         /*
4893                          * EPT will cause page fault only if we need to
4894                          * detect illegal GPAs.
4895                          */
4896                         WARN_ON_ONCE(!allow_smaller_maxphyaddr);
4897                         kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code);
4898                         return 1;
4899                 } else
4900                         return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0);
4901         }
4902
4903         ex_no = intr_info & INTR_INFO_VECTOR_MASK;
4904
4905         if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
4906                 return handle_rmode_exception(vcpu, ex_no, error_code);
4907
4908         switch (ex_no) {
4909         case DB_VECTOR:
4910                 dr6 = vmx_get_exit_qual(vcpu);
4911                 if (!(vcpu->guest_debug &
4912                       (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
4913                         /*
4914                          * If the #DB was due to ICEBP, a.k.a. INT1, skip the
4915                          * instruction.  ICEBP generates a trap-like #DB, but
4916                          * despite its interception control being tied to #DB,
4917                          * is an instruction intercept, i.e. the VM-Exit occurs
4918                          * on the ICEBP itself.  Note, skipping ICEBP also
4919                          * clears STI and MOVSS blocking.
4920                          *
4921                          * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS
4922                          * if single-step is enabled in RFLAGS and STI or MOVSS
4923                          * blocking is active, as the CPU doesn't set the bit
4924                          * on VM-Exit due to #DB interception.  VM-Entry has a
4925                          * consistency check that a single-step #DB is pending
4926                          * in this scenario as the previous instruction cannot
4927                          * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV
4928                          * don't modify RFLAGS), therefore the one instruction
4929                          * delay when activating single-step breakpoints must
4930                          * have already expired.  Note, the CPU sets/clears BS
4931                          * as appropriate for all other VM-Exits types.
4932                          */
4933                         if (is_icebp(intr_info))
4934                                 WARN_ON(!skip_emulated_instruction(vcpu));
4935                         else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) &&
4936                                  (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
4937                                   (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)))
4938                                 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
4939                                             vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS);
4940
4941                         kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
4942                         return 1;
4943                 }
4944                 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW;
4945                 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
4946                 fallthrough;
4947         case BP_VECTOR:
4948                 /*
4949                  * Update instruction length as we may reinject #BP from
4950                  * user space while in guest debugging mode. Reading it for
4951                  * #DB as well causes no harm, it is not used in that case.
4952                  */
4953                 vmx->vcpu.arch.event_exit_inst_len =
4954                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
4955                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
4956                 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
4957                 kvm_run->debug.arch.exception = ex_no;
4958                 break;
4959         case AC_VECTOR:
4960                 if (vmx_guest_inject_ac(vcpu)) {
4961                         kvm_queue_exception_e(vcpu, AC_VECTOR, error_code);
4962                         return 1;
4963                 }
4964
4965                 /*
4966                  * Handle split lock. Depending on detection mode this will
4967                  * either warn and disable split lock detection for this
4968                  * task or force SIGBUS on it.
4969                  */
4970                 if (handle_guest_split_lock(kvm_rip_read(vcpu)))
4971                         return 1;
4972                 fallthrough;
4973         default:
4974                 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
4975                 kvm_run->ex.exception = ex_no;
4976                 kvm_run->ex.error_code = error_code;
4977                 break;
4978         }
4979         return 0;
4980 }
4981
4982 static __always_inline int handle_external_interrupt(struct kvm_vcpu *vcpu)
4983 {
4984         ++vcpu->stat.irq_exits;
4985         return 1;
4986 }
4987
4988 static int handle_triple_fault(struct kvm_vcpu *vcpu)
4989 {
4990         vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
4991         vcpu->mmio_needed = 0;
4992         return 0;
4993 }
4994
4995 static int handle_io(struct kvm_vcpu *vcpu)
4996 {
4997         unsigned long exit_qualification;
4998         int size, in, string;
4999         unsigned port;
5000
5001         exit_qualification = vmx_get_exit_qual(vcpu);
5002         string = (exit_qualification & 16) != 0;
5003
5004         ++vcpu->stat.io_exits;
5005
5006         if (string)
5007                 return kvm_emulate_instruction(vcpu, 0);
5008
5009         port = exit_qualification >> 16;
5010         size = (exit_qualification & 7) + 1;
5011         in = (exit_qualification & 8) != 0;
5012
5013         return kvm_fast_pio(vcpu, size, port, in);
5014 }
5015
5016 static void
5017 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
5018 {
5019         /*
5020          * Patch in the VMCALL instruction:
5021          */
5022         hypercall[0] = 0x0f;
5023         hypercall[1] = 0x01;
5024         hypercall[2] = 0xc1;
5025 }
5026
5027 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */
5028 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
5029 {
5030         if (is_guest_mode(vcpu)) {
5031                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5032                 unsigned long orig_val = val;
5033
5034                 /*
5035                  * We get here when L2 changed cr0 in a way that did not change
5036                  * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
5037                  * but did change L0 shadowed bits. So we first calculate the
5038                  * effective cr0 value that L1 would like to write into the
5039                  * hardware. It consists of the L2-owned bits from the new
5040                  * value combined with the L1-owned bits from L1's guest_cr0.
5041                  */
5042                 val = (val & ~vmcs12->cr0_guest_host_mask) |
5043                         (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);
5044
5045                 if (!nested_guest_cr0_valid(vcpu, val))
5046                         return 1;
5047
5048                 if (kvm_set_cr0(vcpu, val))
5049                         return 1;
5050                 vmcs_writel(CR0_READ_SHADOW, orig_val);
5051                 return 0;
5052         } else {
5053                 if (to_vmx(vcpu)->nested.vmxon &&
5054                     !nested_host_cr0_valid(vcpu, val))
5055                         return 1;
5056
5057                 return kvm_set_cr0(vcpu, val);
5058         }
5059 }
5060
5061 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
5062 {
5063         if (is_guest_mode(vcpu)) {
5064                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5065                 unsigned long orig_val = val;
5066
5067                 /* analogously to handle_set_cr0 */
5068                 val = (val & ~vmcs12->cr4_guest_host_mask) |
5069                         (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
5070                 if (kvm_set_cr4(vcpu, val))
5071                         return 1;
5072                 vmcs_writel(CR4_READ_SHADOW, orig_val);
5073                 return 0;
5074         } else
5075                 return kvm_set_cr4(vcpu, val);
5076 }
5077
5078 static int handle_desc(struct kvm_vcpu *vcpu)
5079 {
5080         WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP));
5081         return kvm_emulate_instruction(vcpu, 0);
5082 }
5083
5084 static int handle_cr(struct kvm_vcpu *vcpu)
5085 {
5086         unsigned long exit_qualification, val;
5087         int cr;
5088         int reg;
5089         int err;
5090         int ret;
5091
5092         exit_qualification = vmx_get_exit_qual(vcpu);
5093         cr = exit_qualification & 15;
5094         reg = (exit_qualification >> 8) & 15;
5095         switch ((exit_qualification >> 4) & 3) {
5096         case 0: /* mov to cr */
5097                 val = kvm_register_read(vcpu, reg);
5098                 trace_kvm_cr_write(cr, val);
5099                 switch (cr) {
5100                 case 0:
5101                         err = handle_set_cr0(vcpu, val);
5102                         return kvm_complete_insn_gp(vcpu, err);
5103                 case 3:
5104                         WARN_ON_ONCE(enable_unrestricted_guest);
5105
5106                         err = kvm_set_cr3(vcpu, val);
5107                         return kvm_complete_insn_gp(vcpu, err);
5108                 case 4:
5109                         err = handle_set_cr4(vcpu, val);
5110                         return kvm_complete_insn_gp(vcpu, err);
5111                 case 8: {
5112                                 u8 cr8_prev = kvm_get_cr8(vcpu);
5113                                 u8 cr8 = (u8)val;
5114                                 err = kvm_set_cr8(vcpu, cr8);
5115                                 ret = kvm_complete_insn_gp(vcpu, err);
5116                                 if (lapic_in_kernel(vcpu))
5117                                         return ret;
5118                                 if (cr8_prev <= cr8)
5119                                         return ret;
5120                                 /*
5121                                  * TODO: we might be squashing a
5122                                  * KVM_GUESTDBG_SINGLESTEP-triggered
5123                                  * KVM_EXIT_DEBUG here.
5124                                  */
5125                                 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
5126                                 return 0;
5127                         }
5128                 }
5129                 break;
5130         case 2: /* clts */
5131                 KVM_BUG(1, vcpu->kvm, "Guest always owns CR0.TS");
5132                 return -EIO;
5133         case 1: /*mov from cr*/
5134                 switch (cr) {
5135                 case 3:
5136                         WARN_ON_ONCE(enable_unrestricted_guest);
5137
5138                         val = kvm_read_cr3(vcpu);
5139                         kvm_register_write(vcpu, reg, val);
5140                         trace_kvm_cr_read(cr, val);
5141                         return kvm_skip_emulated_instruction(vcpu);
5142                 case 8:
5143                         val = kvm_get_cr8(vcpu);
5144                         kvm_register_write(vcpu, reg, val);
5145                         trace_kvm_cr_read(cr, val);
5146                         return kvm_skip_emulated_instruction(vcpu);
5147                 }
5148                 break;
5149         case 3: /* lmsw */
5150                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
5151                 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
5152                 kvm_lmsw(vcpu, val);
5153
5154                 return kvm_skip_emulated_instruction(vcpu);
5155         default:
5156                 break;
5157         }
5158         vcpu->run->exit_reason = 0;
5159         vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
5160                (int)(exit_qualification >> 4) & 3, cr);
5161         return 0;
5162 }
5163
5164 static int handle_dr(struct kvm_vcpu *vcpu)
5165 {
5166         unsigned long exit_qualification;
5167         int dr, dr7, reg;
5168         int err = 1;
5169
5170         exit_qualification = vmx_get_exit_qual(vcpu);
5171         dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
5172
5173         /* First, if DR does not exist, trigger UD */
5174         if (!kvm_require_dr(vcpu, dr))
5175                 return 1;
5176
5177         if (vmx_get_cpl(vcpu) > 0)
5178                 goto out;
5179
5180         dr7 = vmcs_readl(GUEST_DR7);
5181         if (dr7 & DR7_GD) {
5182                 /*
5183                  * As the vm-exit takes precedence over the debug trap, we
5184                  * need to emulate the latter, either for the host or the
5185                  * guest debugging itself.
5186                  */
5187                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
5188                         vcpu->run->debug.arch.dr6 = DR6_BD | DR6_ACTIVE_LOW;
5189                         vcpu->run->debug.arch.dr7 = dr7;
5190                         vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
5191                         vcpu->run->debug.arch.exception = DB_VECTOR;
5192                         vcpu->run->exit_reason = KVM_EXIT_DEBUG;
5193                         return 0;
5194                 } else {
5195                         kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BD);
5196                         return 1;
5197                 }
5198         }
5199
5200         if (vcpu->guest_debug == 0) {
5201                 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING);
5202
5203                 /*
5204                  * No more DR vmexits; force a reload of the debug registers
5205                  * and reenter on this instruction.  The next vmexit will
5206                  * retrieve the full state of the debug registers.
5207                  */
5208                 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
5209                 return 1;
5210         }
5211
5212         reg = DEBUG_REG_ACCESS_REG(exit_qualification);
5213         if (exit_qualification & TYPE_MOV_FROM_DR) {
5214                 unsigned long val;
5215
5216                 kvm_get_dr(vcpu, dr, &val);
5217                 kvm_register_write(vcpu, reg, val);
5218                 err = 0;
5219         } else {
5220                 err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg));
5221         }
5222
5223 out:
5224         return kvm_complete_insn_gp(vcpu, err);
5225 }
5226
5227 static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
5228 {
5229         get_debugreg(vcpu->arch.db[0], 0);
5230         get_debugreg(vcpu->arch.db[1], 1);
5231         get_debugreg(vcpu->arch.db[2], 2);
5232         get_debugreg(vcpu->arch.db[3], 3);
5233         get_debugreg(vcpu->arch.dr6, 6);
5234         vcpu->arch.dr7 = vmcs_readl(GUEST_DR7);
5235
5236         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
5237         exec_controls_setbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING);
5238
5239         /*
5240          * exc_debug expects dr6 to be cleared after it runs, avoid that it sees
5241          * a stale dr6 from the guest.
5242          */
5243         set_debugreg(DR6_RESERVED, 6);
5244 }
5245
5246 static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
5247 {
5248         vmcs_writel(GUEST_DR7, val);
5249 }
5250
5251 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
5252 {
5253         kvm_apic_update_ppr(vcpu);
5254         return 1;
5255 }
5256
5257 static int handle_interrupt_window(struct kvm_vcpu *vcpu)
5258 {
5259         exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);
5260
5261         kvm_make_request(KVM_REQ_EVENT, vcpu);
5262
5263         ++vcpu->stat.irq_window_exits;
5264         return 1;
5265 }
5266
5267 static int handle_invlpg(struct kvm_vcpu *vcpu)
5268 {
5269         unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
5270
5271         kvm_mmu_invlpg(vcpu, exit_qualification);
5272         return kvm_skip_emulated_instruction(vcpu);
5273 }
5274
5275 static int handle_apic_access(struct kvm_vcpu *vcpu)
5276 {
5277         if (likely(fasteoi)) {
5278                 unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
5279                 int access_type, offset;
5280
5281                 access_type = exit_qualification & APIC_ACCESS_TYPE;
5282                 offset = exit_qualification & APIC_ACCESS_OFFSET;
5283                 /*
5284                  * Sane guest uses MOV to write EOI, with written value
5285                  * not cared. So make a short-circuit here by avoiding
5286                  * heavy instruction emulation.
5287                  */
5288                 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
5289                     (offset == APIC_EOI)) {
5290                         kvm_lapic_set_eoi(vcpu);
5291                         return kvm_skip_emulated_instruction(vcpu);
5292                 }
5293         }
5294         return kvm_emulate_instruction(vcpu, 0);
5295 }
5296
5297 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
5298 {
5299         unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
5300         int vector = exit_qualification & 0xff;
5301
5302         /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
5303         kvm_apic_set_eoi_accelerated(vcpu, vector);
5304         return 1;
5305 }
5306
5307 static int handle_apic_write(struct kvm_vcpu *vcpu)
5308 {
5309         unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
5310
5311         /*
5312          * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and
5313          * hardware has done any necessary aliasing, offset adjustments, etc...
5314          * for the access.  I.e. the correct value has already been  written to
5315          * the vAPIC page for the correct 16-byte chunk.  KVM needs only to
5316          * retrieve the register value and emulate the access.
5317          */
5318         u32 offset = exit_qualification & 0xff0;
5319
5320         kvm_apic_write_nodecode(vcpu, offset);
5321         return 1;
5322 }
5323
5324 static int handle_task_switch(struct kvm_vcpu *vcpu)
5325 {
5326         struct vcpu_vmx *vmx = to_vmx(vcpu);
5327         unsigned long exit_qualification;
5328         bool has_error_code = false;
5329         u32 error_code = 0;
5330         u16 tss_selector;
5331         int reason, type, idt_v, idt_index;
5332
5333         idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
5334         idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
5335         type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
5336
5337         exit_qualification = vmx_get_exit_qual(vcpu);
5338
5339         reason = (u32)exit_qualification >> 30;
5340         if (reason == TASK_SWITCH_GATE && idt_v) {
5341                 switch (type) {
5342                 case INTR_TYPE_NMI_INTR:
5343                         vcpu->arch.nmi_injected = false;
5344                         vmx_set_nmi_mask(vcpu, true);
5345                         break;
5346                 case INTR_TYPE_EXT_INTR:
5347                 case INTR_TYPE_SOFT_INTR:
5348                         kvm_clear_interrupt_queue(vcpu);
5349                         break;
5350                 case INTR_TYPE_HARD_EXCEPTION:
5351                         if (vmx->idt_vectoring_info &
5352                             VECTORING_INFO_DELIVER_CODE_MASK) {
5353                                 has_error_code = true;
5354                                 error_code =
5355                                         vmcs_read32(IDT_VECTORING_ERROR_CODE);
5356                         }
5357                         fallthrough;
5358                 case INTR_TYPE_SOFT_EXCEPTION:
5359                         kvm_clear_exception_queue(vcpu);
5360                         break;
5361                 default:
5362                         break;
5363                 }
5364         }
5365         tss_selector = exit_qualification;
5366
5367         if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
5368                        type != INTR_TYPE_EXT_INTR &&
5369                        type != INTR_TYPE_NMI_INTR))
5370                 WARN_ON(!skip_emulated_instruction(vcpu));
5371
5372         /*
5373          * TODO: What about debug traps on tss switch?
5374          *       Are we supposed to inject them and update dr6?
5375          */
5376         return kvm_task_switch(vcpu, tss_selector,
5377                                type == INTR_TYPE_SOFT_INTR ? idt_index : -1,
5378                                reason, has_error_code, error_code);
5379 }
5380
5381 static int handle_ept_violation(struct kvm_vcpu *vcpu)
5382 {
5383         unsigned long exit_qualification;
5384         gpa_t gpa;
5385         u64 error_code;
5386
5387         exit_qualification = vmx_get_exit_qual(vcpu);
5388
5389         /*
5390          * EPT violation happened while executing iret from NMI,
5391          * "blocked by NMI" bit has to be set before next VM entry.
5392          * There are errata that may cause this bit to not be set:
5393          * AAK134, BY25.
5394          */
5395         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
5396                         enable_vnmi &&
5397                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
5398                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);
5399
5400         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5401         trace_kvm_page_fault(gpa, exit_qualification);
5402
5403         /* Is it a read fault? */
5404         error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
5405                      ? PFERR_USER_MASK : 0;
5406         /* Is it a write fault? */
5407         error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
5408                       ? PFERR_WRITE_MASK : 0;
5409         /* Is it a fetch fault? */
5410         error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
5411                       ? PFERR_FETCH_MASK : 0;
5412         /* ept page table entry is present? */
5413         error_code |= (exit_qualification &
5414                        (EPT_VIOLATION_READABLE | EPT_VIOLATION_WRITABLE |
5415                         EPT_VIOLATION_EXECUTABLE))
5416                       ? PFERR_PRESENT_MASK : 0;
5417
5418         error_code |= (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) != 0 ?
5419                PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;
5420
5421         vcpu->arch.exit_qualification = exit_qualification;
5422
5423         /*
5424          * Check that the GPA doesn't exceed physical memory limits, as that is
5425          * a guest page fault.  We have to emulate the instruction here, because
5426          * if the illegal address is that of a paging structure, then
5427          * EPT_VIOLATION_ACC_WRITE bit is set.  Alternatively, if supported we
5428          * would also use advanced VM-exit information for EPT violations to
5429          * reconstruct the page fault error code.
5430          */
5431         if (unlikely(allow_smaller_maxphyaddr && kvm_vcpu_is_illegal_gpa(vcpu, gpa)))
5432                 return kvm_emulate_instruction(vcpu, 0);
5433
5434         return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
5435 }
5436
5437 static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
5438 {
5439         gpa_t gpa;
5440
5441         if (!vmx_can_emulate_instruction(vcpu, EMULTYPE_PF, NULL, 0))
5442                 return 1;
5443
5444         /*
5445          * A nested guest cannot optimize MMIO vmexits, because we have an
5446          * nGPA here instead of the required GPA.
5447          */
5448         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5449         if (!is_guest_mode(vcpu) &&
5450             !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
5451                 trace_kvm_fast_mmio(gpa);
5452                 return kvm_skip_emulated_instruction(vcpu);
5453         }
5454
5455         return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0);
5456 }
5457
5458 static int handle_nmi_window(struct kvm_vcpu *vcpu)
5459 {
5460         if (KVM_BUG_ON(!enable_vnmi, vcpu->kvm))
5461                 return -EIO;
5462
5463         exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING);
5464         ++vcpu->stat.nmi_window_exits;
5465         kvm_make_request(KVM_REQ_EVENT, vcpu);
5466
5467         return 1;
5468 }
5469
5470 static bool vmx_emulation_required_with_pending_exception(struct kvm_vcpu *vcpu)
5471 {
5472         struct vcpu_vmx *vmx = to_vmx(vcpu);
5473
5474         return vmx->emulation_required && !vmx->rmode.vm86_active &&
5475                vcpu->arch.exception.pending;
5476 }
5477
5478 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
5479 {
5480         struct vcpu_vmx *vmx = to_vmx(vcpu);
5481         bool intr_window_requested;
5482         unsigned count = 130;
5483
5484         intr_window_requested = exec_controls_get(vmx) &
5485                                 CPU_BASED_INTR_WINDOW_EXITING;
5486
5487         while (vmx->emulation_required && count-- != 0) {
5488                 if (intr_window_requested && !vmx_interrupt_blocked(vcpu))
5489                         return handle_interrupt_window(&vmx->vcpu);
5490
5491                 if (kvm_test_request(KVM_REQ_EVENT, vcpu))
5492                         return 1;
5493
5494                 if (!kvm_emulate_instruction(vcpu, 0))
5495                         return 0;
5496
5497                 if (vmx_emulation_required_with_pending_exception(vcpu)) {
5498                         kvm_prepare_emulation_failure_exit(vcpu);
5499                         return 0;
5500                 }
5501
5502                 if (vcpu->arch.halt_request) {
5503                         vcpu->arch.halt_request = 0;
5504                         return kvm_emulate_halt_noskip(vcpu);
5505                 }
5506
5507                 /*
5508                  * Note, return 1 and not 0, vcpu_run() will invoke
5509                  * xfer_to_guest_mode() which will create a proper return
5510                  * code.
5511                  */
5512                 if (__xfer_to_guest_mode_work_pending())
5513                         return 1;
5514         }
5515
5516         return 1;
5517 }
5518
5519 static int vmx_vcpu_pre_run(struct kvm_vcpu *vcpu)
5520 {
5521         if (vmx_emulation_required_with_pending_exception(vcpu)) {
5522                 kvm_prepare_emulation_failure_exit(vcpu);
5523                 return 0;
5524         }
5525
5526         return 1;
5527 }
5528
5529 static void grow_ple_window(struct kvm_vcpu *vcpu)
5530 {
5531         struct vcpu_vmx *vmx = to_vmx(vcpu);
5532         unsigned int old = vmx->ple_window;
5533
5534         vmx->ple_window = __grow_ple_window(old, ple_window,
5535                                             ple_window_grow,
5536                                             ple_window_max);
5537
5538         if (vmx->ple_window != old) {
5539                 vmx->ple_window_dirty = true;
5540                 trace_kvm_ple_window_update(vcpu->vcpu_id,
5541                                             vmx->ple_window, old);
5542         }
5543 }
5544
5545 static void shrink_ple_window(struct kvm_vcpu *vcpu)
5546 {
5547         struct vcpu_vmx *vmx = to_vmx(vcpu);
5548         unsigned int old = vmx->ple_window;
5549
5550         vmx->ple_window = __shrink_ple_window(old, ple_window,
5551                                               ple_window_shrink,
5552                                               ple_window);
5553
5554         if (vmx->ple_window != old) {
5555                 vmx->ple_window_dirty = true;
5556                 trace_kvm_ple_window_update(vcpu->vcpu_id,
5557                                             vmx->ple_window, old);
5558         }
5559 }
5560
5561 /*
5562  * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
5563  * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
5564  */
5565 static int handle_pause(struct kvm_vcpu *vcpu)
5566 {
5567         if (!kvm_pause_in_guest(vcpu->kvm))
5568                 grow_ple_window(vcpu);
5569
5570         /*
5571          * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting"
5572          * VM-execution control is ignored if CPL > 0. OTOH, KVM
5573          * never set PAUSE_EXITING and just set PLE if supported,
5574          * so the vcpu must be CPL=0 if it gets a PAUSE exit.
5575          */
5576         kvm_vcpu_on_spin(vcpu, true);
5577         return kvm_skip_emulated_instruction(vcpu);
5578 }
5579
5580 static int handle_monitor_trap(struct kvm_vcpu *vcpu)
5581 {
5582         return 1;
5583 }
5584
5585 static int handle_invpcid(struct kvm_vcpu *vcpu)
5586 {
5587         u32 vmx_instruction_info;
5588         unsigned long type;
5589         gva_t gva;
5590         struct {
5591                 u64 pcid;
5592                 u64 gla;
5593         } operand;
5594         int gpr_index;
5595
5596         if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
5597                 kvm_queue_exception(vcpu, UD_VECTOR);
5598                 return 1;
5599         }
5600
5601         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5602         gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info);
5603         type = kvm_register_read(vcpu, gpr_index);
5604
5605         /* According to the Intel instruction reference, the memory operand
5606          * is read even if it isn't needed (e.g., for type==all)
5607          */
5608         if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
5609                                 vmx_instruction_info, false,
5610                                 sizeof(operand), &gva))
5611                 return 1;
5612
5613         return kvm_handle_invpcid(vcpu, type, gva);
5614 }
5615
5616 static int handle_pml_full(struct kvm_vcpu *vcpu)
5617 {
5618         unsigned long exit_qualification;
5619
5620         trace_kvm_pml_full(vcpu->vcpu_id);
5621
5622         exit_qualification = vmx_get_exit_qual(vcpu);
5623
5624         /*
5625          * PML buffer FULL happened while executing iret from NMI,
5626          * "blocked by NMI" bit has to be set before next VM entry.
5627          */
5628         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
5629                         enable_vnmi &&
5630                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
5631                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
5632                                 GUEST_INTR_STATE_NMI);
5633
5634         /*
5635          * PML buffer already flushed at beginning of VMEXIT. Nothing to do
5636          * here.., and there's no userspace involvement needed for PML.
5637          */
5638         return 1;
5639 }
5640
5641 static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu)
5642 {
5643         struct vcpu_vmx *vmx = to_vmx(vcpu);
5644
5645         if (!vmx->req_immediate_exit &&
5646             !unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) {
5647                 kvm_lapic_expired_hv_timer(vcpu);
5648                 return EXIT_FASTPATH_REENTER_GUEST;
5649         }
5650
5651         return EXIT_FASTPATH_NONE;
5652 }
5653
5654 static int handle_preemption_timer(struct kvm_vcpu *vcpu)
5655 {
5656         handle_fastpath_preemption_timer(vcpu);
5657         return 1;
5658 }
5659
5660 /*
5661  * When nested=0, all VMX instruction VM Exits filter here.  The handlers
5662  * are overwritten by nested_vmx_setup() when nested=1.
5663  */
5664 static int handle_vmx_instruction(struct kvm_vcpu *vcpu)
5665 {
5666         kvm_queue_exception(vcpu, UD_VECTOR);
5667         return 1;
5668 }
5669
5670 #ifndef CONFIG_X86_SGX_KVM
5671 static int handle_encls(struct kvm_vcpu *vcpu)
5672 {
5673         /*
5674          * SGX virtualization is disabled.  There is no software enable bit for
5675          * SGX, so KVM intercepts all ENCLS leafs and injects a #UD to prevent
5676          * the guest from executing ENCLS (when SGX is supported by hardware).
5677          */
5678         kvm_queue_exception(vcpu, UD_VECTOR);
5679         return 1;
5680 }
5681 #endif /* CONFIG_X86_SGX_KVM */
5682
5683 static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu)
5684 {
5685         /*
5686          * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK
5687          * VM-Exits. Unconditionally set the flag here and leave the handling to
5688          * vmx_handle_exit().
5689          */
5690         to_vmx(vcpu)->exit_reason.bus_lock_detected = true;
5691         return 1;
5692 }
5693
5694 /*
5695  * The exit handlers return 1 if the exit was handled fully and guest execution
5696  * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
5697  * to be done to userspace and return 0.
5698  */
5699 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
5700         [EXIT_REASON_EXCEPTION_NMI]           = handle_exception_nmi,
5701         [EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
5702         [EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
5703         [EXIT_REASON_NMI_WINDOW]              = handle_nmi_window,
5704         [EXIT_REASON_IO_INSTRUCTION]          = handle_io,
5705         [EXIT_REASON_CR_ACCESS]               = handle_cr,
5706         [EXIT_REASON_DR_ACCESS]               = handle_dr,
5707         [EXIT_REASON_CPUID]                   = kvm_emulate_cpuid,
5708         [EXIT_REASON_MSR_READ]                = kvm_emulate_rdmsr,
5709         [EXIT_REASON_MSR_WRITE]               = kvm_emulate_wrmsr,
5710         [EXIT_REASON_INTERRUPT_WINDOW]        = handle_interrupt_window,
5711         [EXIT_REASON_HLT]                     = kvm_emulate_halt,
5712         [EXIT_REASON_INVD]                    = kvm_emulate_invd,
5713         [EXIT_REASON_INVLPG]                  = handle_invlpg,
5714         [EXIT_REASON_RDPMC]                   = kvm_emulate_rdpmc,
5715         [EXIT_REASON_VMCALL]                  = kvm_emulate_hypercall,
5716         [EXIT_REASON_VMCLEAR]                 = handle_vmx_instruction,
5717         [EXIT_REASON_VMLAUNCH]                = handle_vmx_instruction,
5718         [EXIT_REASON_VMPTRLD]                 = handle_vmx_instruction,
5719         [EXIT_REASON_VMPTRST]                 = handle_vmx_instruction,
5720         [EXIT_REASON_VMREAD]                  = handle_vmx_instruction,
5721         [EXIT_REASON_VMRESUME]                = handle_vmx_instruction,
5722         [EXIT_REASON_VMWRITE]                 = handle_vmx_instruction,
5723         [EXIT_REASON_VMOFF]                   = handle_vmx_instruction,
5724         [EXIT_REASON_VMON]                    = handle_vmx_instruction,
5725         [EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
5726         [EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
5727         [EXIT_REASON_APIC_WRITE]              = handle_apic_write,
5728         [EXIT_REASON_EOI_INDUCED]             = handle_apic_eoi_induced,
5729         [EXIT_REASON_WBINVD]                  = kvm_emulate_wbinvd,
5730         [EXIT_REASON_XSETBV]                  = kvm_emulate_xsetbv,
5731         [EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
5732         [EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
5733         [EXIT_REASON_GDTR_IDTR]               = handle_desc,
5734         [EXIT_REASON_LDTR_TR]                 = handle_desc,
5735         [EXIT_REASON_EPT_VIOLATION]           = handle_ept_violation,
5736         [EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
5737         [EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
5738         [EXIT_REASON_MWAIT_INSTRUCTION]       = kvm_emulate_mwait,
5739         [EXIT_REASON_MONITOR_TRAP_FLAG]       = handle_monitor_trap,
5740         [EXIT_REASON_MONITOR_INSTRUCTION]     = kvm_emulate_monitor,
5741         [EXIT_REASON_INVEPT]                  = handle_vmx_instruction,
5742         [EXIT_REASON_INVVPID]                 = handle_vmx_instruction,
5743         [EXIT_REASON_RDRAND]                  = kvm_handle_invalid_op,
5744         [EXIT_REASON_RDSEED]                  = kvm_handle_invalid_op,
5745         [EXIT_REASON_PML_FULL]                = handle_pml_full,
5746         [EXIT_REASON_INVPCID]                 = handle_invpcid,
5747         [EXIT_REASON_VMFUNC]                  = handle_vmx_instruction,
5748         [EXIT_REASON_PREEMPTION_TIMER]        = handle_preemption_timer,
5749         [EXIT_REASON_ENCLS]                   = handle_encls,
5750         [EXIT_REASON_BUS_LOCK]                = handle_bus_lock_vmexit,
5751 };
5752
5753 static const int kvm_vmx_max_exit_handlers =
5754         ARRAY_SIZE(kvm_vmx_exit_handlers);
5755
5756 static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
5757                               u64 *info1, u64 *info2,
5758                               u32 *intr_info, u32 *error_code)
5759 {
5760         struct vcpu_vmx *vmx = to_vmx(vcpu);
5761
5762         *reason = vmx->exit_reason.full;
5763         *info1 = vmx_get_exit_qual(vcpu);
5764         if (!(vmx->exit_reason.failed_vmentry)) {
5765                 *info2 = vmx->idt_vectoring_info;
5766                 *intr_info = vmx_get_intr_info(vcpu);
5767                 if (is_exception_with_error_code(*intr_info))
5768                         *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
5769                 else
5770                         *error_code = 0;
5771         } else {
5772                 *info2 = 0;
5773                 *intr_info = 0;
5774                 *error_code = 0;
5775         }
5776 }
5777
5778 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx)
5779 {
5780         if (vmx->pml_pg) {
5781                 __free_page(vmx->pml_pg);
5782                 vmx->pml_pg = NULL;
5783         }
5784 }
5785
5786 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu)
5787 {
5788         struct vcpu_vmx *vmx = to_vmx(vcpu);
5789         u64 *pml_buf;
5790         u16 pml_idx;
5791
5792         pml_idx = vmcs_read16(GUEST_PML_INDEX);
5793
5794         /* Do nothing if PML buffer is empty */
5795         if (pml_idx == (PML_ENTITY_NUM - 1))
5796                 return;
5797
5798         /* PML index always points to next available PML buffer entity */
5799         if (pml_idx >= PML_ENTITY_NUM)
5800                 pml_idx = 0;
5801         else
5802                 pml_idx++;
5803
5804         pml_buf = page_address(vmx->pml_pg);
5805         for (; pml_idx < PML_ENTITY_NUM; pml_idx++) {
5806                 u64 gpa;
5807
5808                 gpa = pml_buf[pml_idx];
5809                 WARN_ON(gpa & (PAGE_SIZE - 1));
5810                 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
5811         }
5812
5813         /* reset PML index */
5814         vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
5815 }
5816
5817 static void vmx_dump_sel(char *name, uint32_t sel)
5818 {
5819         pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n",
5820                name, vmcs_read16(sel),
5821                vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR),
5822                vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR),
5823                vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR));
5824 }
5825
5826 static void vmx_dump_dtsel(char *name, uint32_t limit)
5827 {
5828         pr_err("%s                           limit=0x%08x, base=0x%016lx\n",
5829                name, vmcs_read32(limit),
5830                vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT));
5831 }
5832
5833 static void vmx_dump_msrs(char *name, struct vmx_msrs *m)
5834 {
5835         unsigned int i;
5836         struct vmx_msr_entry *e;
5837
5838         pr_err("MSR %s:\n", name);
5839         for (i = 0, e = m->val; i < m->nr; ++i, ++e)
5840                 pr_err("  %2d: msr=0x%08x value=0x%016llx\n", i, e->index, e->value);
5841 }
5842
5843 void dump_vmcs(struct kvm_vcpu *vcpu)
5844 {
5845         struct vcpu_vmx *vmx = to_vmx(vcpu);
5846         u32 vmentry_ctl, vmexit_ctl;
5847         u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control;
5848         unsigned long cr4;
5849         int efer_slot;
5850
5851         if (!dump_invalid_vmcs) {
5852                 pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n");
5853                 return;
5854         }
5855
5856         vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS);
5857         vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS);
5858         cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
5859         pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);
5860         cr4 = vmcs_readl(GUEST_CR4);
5861         secondary_exec_control = 0;
5862         if (cpu_has_secondary_exec_ctrls())
5863                 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
5864
5865         pr_err("VMCS %p, last attempted VM-entry on CPU %d\n",
5866                vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu);
5867         pr_err("*** Guest State ***\n");
5868         pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
5869                vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW),
5870                vmcs_readl(CR0_GUEST_HOST_MASK));
5871         pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
5872                cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK));
5873         pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3));
5874         if (cpu_has_vmx_ept()) {
5875                 pr_err("PDPTR0 = 0x%016llx  PDPTR1 = 0x%016llx\n",
5876                        vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1));
5877                 pr_err("PDPTR2 = 0x%016llx  PDPTR3 = 0x%016llx\n",
5878                        vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3));
5879         }
5880         pr_err("RSP = 0x%016lx  RIP = 0x%016lx\n",
5881                vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP));
5882         pr_err("RFLAGS=0x%08lx         DR7 = 0x%016lx\n",
5883                vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7));
5884         pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
5885                vmcs_readl(GUEST_SYSENTER_ESP),
5886                vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP));
5887         vmx_dump_sel("CS:  ", GUEST_CS_SELECTOR);
5888         vmx_dump_sel("DS:  ", GUEST_DS_SELECTOR);
5889         vmx_dump_sel("SS:  ", GUEST_SS_SELECTOR);
5890         vmx_dump_sel("ES:  ", GUEST_ES_SELECTOR);
5891         vmx_dump_sel("FS:  ", GUEST_FS_SELECTOR);
5892         vmx_dump_sel("GS:  ", GUEST_GS_SELECTOR);
5893         vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT);
5894         vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR);
5895         vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT);
5896         vmx_dump_sel("TR:  ", GUEST_TR_SELECTOR);
5897         efer_slot = vmx_find_loadstore_msr_slot(&vmx->msr_autoload.guest, MSR_EFER);
5898         if (vmentry_ctl & VM_ENTRY_LOAD_IA32_EFER)
5899                 pr_err("EFER= 0x%016llx\n", vmcs_read64(GUEST_IA32_EFER));
5900         else if (efer_slot >= 0)
5901                 pr_err("EFER= 0x%016llx (autoload)\n",
5902                        vmx->msr_autoload.guest.val[efer_slot].value);
5903         else if (vmentry_ctl & VM_ENTRY_IA32E_MODE)
5904                 pr_err("EFER= 0x%016llx (effective)\n",
5905                        vcpu->arch.efer | (EFER_LMA | EFER_LME));
5906         else
5907                 pr_err("EFER= 0x%016llx (effective)\n",
5908                        vcpu->arch.efer & ~(EFER_LMA | EFER_LME));
5909         if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PAT)
5910                 pr_err("PAT = 0x%016llx\n", vmcs_read64(GUEST_IA32_PAT));
5911         pr_err("DebugCtl = 0x%016llx  DebugExceptions = 0x%016lx\n",
5912                vmcs_read64(GUEST_IA32_DEBUGCTL),
5913                vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS));
5914         if (cpu_has_load_perf_global_ctrl() &&
5915             vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
5916                 pr_err("PerfGlobCtl = 0x%016llx\n",
5917                        vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL));
5918         if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
5919                 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS));
5920         pr_err("Interruptibility = %08x  ActivityState = %08x\n",
5921                vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
5922                vmcs_read32(GUEST_ACTIVITY_STATE));
5923         if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
5924                 pr_err("InterruptStatus = %04x\n",
5925                        vmcs_read16(GUEST_INTR_STATUS));
5926         if (vmcs_read32(VM_ENTRY_MSR_LOAD_COUNT) > 0)
5927                 vmx_dump_msrs("guest autoload", &vmx->msr_autoload.guest);
5928         if (vmcs_read32(VM_EXIT_MSR_STORE_COUNT) > 0)
5929                 vmx_dump_msrs("guest autostore", &vmx->msr_autostore.guest);
5930
5931         pr_err("*** Host State ***\n");
5932         pr_err("RIP = 0x%016lx  RSP = 0x%016lx\n",
5933                vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP));
5934         pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n",
5935                vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR),
5936                vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR),
5937                vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR),
5938                vmcs_read16(HOST_TR_SELECTOR));
5939         pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n",
5940                vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE),
5941                vmcs_readl(HOST_TR_BASE));
5942         pr_err("GDTBase=%016lx IDTBase=%016lx\n",
5943                vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE));
5944         pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n",
5945                vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3),
5946                vmcs_readl(HOST_CR4));
5947         pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
5948                vmcs_readl(HOST_IA32_SYSENTER_ESP),
5949                vmcs_read32(HOST_IA32_SYSENTER_CS),
5950                vmcs_readl(HOST_IA32_SYSENTER_EIP));
5951         if (vmexit_ctl & VM_EXIT_LOAD_IA32_EFER)
5952                 pr_err("EFER= 0x%016llx\n", vmcs_read64(HOST_IA32_EFER));
5953         if (vmexit_ctl & VM_EXIT_LOAD_IA32_PAT)
5954                 pr_err("PAT = 0x%016llx\n", vmcs_read64(HOST_IA32_PAT));
5955         if (cpu_has_load_perf_global_ctrl() &&
5956             vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
5957                 pr_err("PerfGlobCtl = 0x%016llx\n",
5958                        vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL));
5959         if (vmcs_read32(VM_EXIT_MSR_LOAD_COUNT) > 0)
5960                 vmx_dump_msrs("host autoload", &vmx->msr_autoload.host);
5961
5962         pr_err("*** Control State ***\n");
5963         pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
5964                pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control);
5965         pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl);
5966         pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n",
5967                vmcs_read32(EXCEPTION_BITMAP),
5968                vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK),
5969                vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH));
5970         pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n",
5971                vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
5972                vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE),
5973                vmcs_read32(VM_ENTRY_INSTRUCTION_LEN));
5974         pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n",
5975                vmcs_read32(VM_EXIT_INTR_INFO),
5976                vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
5977                vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
5978         pr_err("        reason=%08x qualification=%016lx\n",
5979                vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION));
5980         pr_err("IDTVectoring: info=%08x errcode=%08x\n",
5981                vmcs_read32(IDT_VECTORING_INFO_FIELD),
5982                vmcs_read32(IDT_VECTORING_ERROR_CODE));
5983         pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET));
5984         if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING)
5985                 pr_err("TSC Multiplier = 0x%016llx\n",
5986                        vmcs_read64(TSC_MULTIPLIER));
5987         if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW) {
5988                 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) {
5989                         u16 status = vmcs_read16(GUEST_INTR_STATUS);
5990                         pr_err("SVI|RVI = %02x|%02x ", status >> 8, status & 0xff);
5991                 }
5992                 pr_cont("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD));
5993                 if (secondary_exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)
5994                         pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR));
5995                 pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR));
5996         }
5997         if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR)
5998                 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV));
5999         if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT))
6000                 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER));
6001         if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING)
6002                 pr_err("PLE Gap=%08x Window=%08x\n",
6003                        vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW));
6004         if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID)
6005                 pr_err("Virtual processor ID = 0x%04x\n",
6006                        vmcs_read16(VIRTUAL_PROCESSOR_ID));
6007 }
6008
6009 /*
6010  * The guest has exited.  See if we can fix it or if we need userspace
6011  * assistance.
6012  */
6013 static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
6014 {
6015         struct vcpu_vmx *vmx = to_vmx(vcpu);
6016         union vmx_exit_reason exit_reason = vmx->exit_reason;
6017         u32 vectoring_info = vmx->idt_vectoring_info;
6018         u16 exit_handler_index;
6019
6020         /*
6021          * Flush logged GPAs PML buffer, this will make dirty_bitmap more
6022          * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before
6023          * querying dirty_bitmap, we only need to kick all vcpus out of guest
6024          * mode as if vcpus is in root mode, the PML buffer must has been
6025          * flushed already.  Note, PML is never enabled in hardware while
6026          * running L2.
6027          */
6028         if (enable_pml && !is_guest_mode(vcpu))
6029                 vmx_flush_pml_buffer(vcpu);
6030
6031         /*
6032          * KVM should never reach this point with a pending nested VM-Enter.
6033          * More specifically, short-circuiting VM-Entry to emulate L2 due to
6034          * invalid guest state should never happen as that means KVM knowingly
6035          * allowed a nested VM-Enter with an invalid vmcs12.  More below.
6036          */
6037         if (KVM_BUG_ON(vmx->nested.nested_run_pending, vcpu->kvm))
6038                 return -EIO;
6039
6040         if (is_guest_mode(vcpu)) {
6041                 /*
6042                  * PML is never enabled when running L2, bail immediately if a
6043                  * PML full exit occurs as something is horribly wrong.
6044                  */
6045                 if (exit_reason.basic == EXIT_REASON_PML_FULL)
6046                         goto unexpected_vmexit;
6047
6048                 /*
6049                  * The host physical addresses of some pages of guest memory
6050                  * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC
6051                  * Page). The CPU may write to these pages via their host
6052                  * physical address while L2 is running, bypassing any
6053                  * address-translation-based dirty tracking (e.g. EPT write
6054                  * protection).
6055                  *
6056                  * Mark them dirty on every exit from L2 to prevent them from
6057                  * getting out of sync with dirty tracking.
6058                  */
6059                 nested_mark_vmcs12_pages_dirty(vcpu);
6060
6061                 /*
6062                  * Synthesize a triple fault if L2 state is invalid.  In normal
6063                  * operation, nested VM-Enter rejects any attempt to enter L2
6064                  * with invalid state.  However, those checks are skipped if
6065                  * state is being stuffed via RSM or KVM_SET_NESTED_STATE.  If
6066                  * L2 state is invalid, it means either L1 modified SMRAM state
6067                  * or userspace provided bad state.  Synthesize TRIPLE_FAULT as
6068                  * doing so is architecturally allowed in the RSM case, and is
6069                  * the least awful solution for the userspace case without
6070                  * risking false positives.
6071                  */
6072                 if (vmx->emulation_required) {
6073                         nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0);
6074                         return 1;
6075                 }
6076
6077                 if (nested_vmx_reflect_vmexit(vcpu))
6078                         return 1;
6079         }
6080
6081         /* If guest state is invalid, start emulating.  L2 is handled above. */
6082         if (vmx->emulation_required)
6083                 return handle_invalid_guest_state(vcpu);
6084
6085         if (exit_reason.failed_vmentry) {
6086                 dump_vmcs(vcpu);
6087                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
6088                 vcpu->run->fail_entry.hardware_entry_failure_reason
6089                         = exit_reason.full;
6090                 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
6091                 return 0;
6092         }
6093
6094         if (unlikely(vmx->fail)) {
6095                 dump_vmcs(vcpu);
6096                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
6097                 vcpu->run->fail_entry.hardware_entry_failure_reason
6098                         = vmcs_read32(VM_INSTRUCTION_ERROR);
6099                 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
6100                 return 0;
6101         }
6102
6103         /*
6104          * Note:
6105          * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
6106          * delivery event since it indicates guest is accessing MMIO.
6107          * The vm-exit can be triggered again after return to guest that
6108          * will cause infinite loop.
6109          */
6110         if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
6111             (exit_reason.basic != EXIT_REASON_EXCEPTION_NMI &&
6112              exit_reason.basic != EXIT_REASON_EPT_VIOLATION &&
6113              exit_reason.basic != EXIT_REASON_PML_FULL &&
6114              exit_reason.basic != EXIT_REASON_APIC_ACCESS &&
6115              exit_reason.basic != EXIT_REASON_TASK_SWITCH)) {
6116                 int ndata = 3;
6117
6118                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6119                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
6120                 vcpu->run->internal.data[0] = vectoring_info;
6121                 vcpu->run->internal.data[1] = exit_reason.full;
6122                 vcpu->run->internal.data[2] = vcpu->arch.exit_qualification;
6123                 if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) {
6124                         vcpu->run->internal.data[ndata++] =
6125                                 vmcs_read64(GUEST_PHYSICAL_ADDRESS);
6126                 }
6127                 vcpu->run->internal.data[ndata++] = vcpu->arch.last_vmentry_cpu;
6128                 vcpu->run->internal.ndata = ndata;
6129                 return 0;
6130         }
6131
6132         if (unlikely(!enable_vnmi &&
6133                      vmx->loaded_vmcs->soft_vnmi_blocked)) {
6134                 if (!vmx_interrupt_blocked(vcpu)) {
6135                         vmx->loaded_vmcs->soft_vnmi_blocked = 0;
6136                 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL &&
6137                            vcpu->arch.nmi_pending) {
6138                         /*
6139                          * This CPU don't support us in finding the end of an
6140                          * NMI-blocked window if the guest runs with IRQs
6141                          * disabled. So we pull the trigger after 1 s of
6142                          * futile waiting, but inform the user about this.
6143                          */
6144                         printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
6145                                "state on VCPU %d after 1 s timeout\n",
6146                                __func__, vcpu->vcpu_id);
6147                         vmx->loaded_vmcs->soft_vnmi_blocked = 0;
6148                 }
6149         }
6150
6151         if (exit_fastpath != EXIT_FASTPATH_NONE)
6152                 return 1;
6153
6154         if (exit_reason.basic >= kvm_vmx_max_exit_handlers)
6155                 goto unexpected_vmexit;
6156 #ifdef CONFIG_RETPOLINE
6157         if (exit_reason.basic == EXIT_REASON_MSR_WRITE)
6158                 return kvm_emulate_wrmsr(vcpu);
6159         else if (exit_reason.basic == EXIT_REASON_PREEMPTION_TIMER)
6160                 return handle_preemption_timer(vcpu);
6161         else if (exit_reason.basic == EXIT_REASON_INTERRUPT_WINDOW)
6162                 return handle_interrupt_window(vcpu);
6163         else if (exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT)
6164                 return handle_external_interrupt(vcpu);
6165         else if (exit_reason.basic == EXIT_REASON_HLT)
6166                 return kvm_emulate_halt(vcpu);
6167         else if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG)
6168                 return handle_ept_misconfig(vcpu);
6169 #endif
6170
6171         exit_handler_index = array_index_nospec((u16)exit_reason.basic,
6172                                                 kvm_vmx_max_exit_handlers);
6173         if (!kvm_vmx_exit_handlers[exit_handler_index])
6174                 goto unexpected_vmexit;
6175
6176         return kvm_vmx_exit_handlers[exit_handler_index](vcpu);
6177
6178 unexpected_vmexit:
6179         vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
6180                     exit_reason.full);
6181         dump_vmcs(vcpu);
6182         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6183         vcpu->run->internal.suberror =
6184                         KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
6185         vcpu->run->internal.ndata = 2;
6186         vcpu->run->internal.data[0] = exit_reason.full;
6187         vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
6188         return 0;
6189 }
6190
6191 static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
6192 {
6193         int ret = __vmx_handle_exit(vcpu, exit_fastpath);
6194
6195         /*
6196          * Exit to user space when bus lock detected to inform that there is
6197          * a bus lock in guest.
6198          */
6199         if (to_vmx(vcpu)->exit_reason.bus_lock_detected) {
6200                 if (ret > 0)
6201                         vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
6202
6203                 vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK;
6204                 return 0;
6205         }
6206         return ret;
6207 }
6208
6209 /*
6210  * Software based L1D cache flush which is used when microcode providing
6211  * the cache control MSR is not loaded.
6212  *
6213  * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to
6214  * flush it is required to read in 64 KiB because the replacement algorithm
6215  * is not exactly LRU. This could be sized at runtime via topology
6216  * information but as all relevant affected CPUs have 32KiB L1D cache size
6217  * there is no point in doing so.
6218  */
6219 static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu)
6220 {
6221         int size = PAGE_SIZE << L1D_CACHE_ORDER;
6222
6223         /*
6224          * This code is only executed when the the flush mode is 'cond' or
6225          * 'always'
6226          */
6227         if (static_branch_likely(&vmx_l1d_flush_cond)) {
6228                 bool flush_l1d;
6229
6230                 /*
6231                  * Clear the per-vcpu flush bit, it gets set again
6232                  * either from vcpu_run() or from one of the unsafe
6233                  * VMEXIT handlers.
6234                  */
6235                 flush_l1d = vcpu->arch.l1tf_flush_l1d;
6236                 vcpu->arch.l1tf_flush_l1d = false;
6237
6238                 /*
6239                  * Clear the per-cpu flush bit, it gets set again from
6240                  * the interrupt handlers.
6241                  */
6242                 flush_l1d |= kvm_get_cpu_l1tf_flush_l1d();
6243                 kvm_clear_cpu_l1tf_flush_l1d();
6244
6245                 if (!flush_l1d)
6246                         return;
6247         }
6248
6249         vcpu->stat.l1d_flush++;
6250
6251         if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) {
6252                 native_wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
6253                 return;
6254         }
6255
6256         asm volatile(
6257                 /* First ensure the pages are in the TLB */
6258                 "xorl   %%eax, %%eax\n"
6259                 ".Lpopulate_tlb:\n\t"
6260                 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
6261                 "addl   $4096, %%eax\n\t"
6262                 "cmpl   %%eax, %[size]\n\t"
6263                 "jne    .Lpopulate_tlb\n\t"
6264                 "xorl   %%eax, %%eax\n\t"
6265                 "cpuid\n\t"
6266                 /* Now fill the cache */
6267                 "xorl   %%eax, %%eax\n"
6268                 ".Lfill_cache:\n"
6269                 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
6270                 "addl   $64, %%eax\n\t"
6271                 "cmpl   %%eax, %[size]\n\t"
6272                 "jne    .Lfill_cache\n\t"
6273                 "lfence\n"
6274                 :: [flush_pages] "r" (vmx_l1d_flush_pages),
6275                     [size] "r" (size)
6276                 : "eax", "ebx", "ecx", "edx");
6277 }
6278
6279 static void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
6280 {
6281         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6282         int tpr_threshold;
6283
6284         if (is_guest_mode(vcpu) &&
6285                 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
6286                 return;
6287
6288         tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr;
6289         if (is_guest_mode(vcpu))
6290                 to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold;
6291         else
6292                 vmcs_write32(TPR_THRESHOLD, tpr_threshold);
6293 }
6294
6295 void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
6296 {
6297         struct vcpu_vmx *vmx = to_vmx(vcpu);
6298         u32 sec_exec_control;
6299
6300         if (!lapic_in_kernel(vcpu))
6301                 return;
6302
6303         if (!flexpriority_enabled &&
6304             !cpu_has_vmx_virtualize_x2apic_mode())
6305                 return;
6306
6307         /* Postpone execution until vmcs01 is the current VMCS. */
6308         if (is_guest_mode(vcpu)) {
6309                 vmx->nested.change_vmcs01_virtual_apic_mode = true;
6310                 return;
6311         }
6312
6313         sec_exec_control = secondary_exec_controls_get(vmx);
6314         sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
6315                               SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
6316
6317         switch (kvm_get_apic_mode(vcpu)) {
6318         case LAPIC_MODE_INVALID:
6319                 WARN_ONCE(true, "Invalid local APIC state");
6320                 break;
6321         case LAPIC_MODE_DISABLED:
6322                 break;
6323         case LAPIC_MODE_XAPIC:
6324                 if (flexpriority_enabled) {
6325                         sec_exec_control |=
6326                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
6327                         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
6328
6329                         /*
6330                          * Flush the TLB, reloading the APIC access page will
6331                          * only do so if its physical address has changed, but
6332                          * the guest may have inserted a non-APIC mapping into
6333                          * the TLB while the APIC access page was disabled.
6334                          */
6335                         kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
6336                 }
6337                 break;
6338         case LAPIC_MODE_X2APIC:
6339                 if (cpu_has_vmx_virtualize_x2apic_mode())
6340                         sec_exec_control |=
6341                                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
6342                 break;
6343         }
6344         secondary_exec_controls_set(vmx, sec_exec_control);
6345
6346         vmx_update_msr_bitmap_x2apic(vcpu);
6347 }
6348
6349 static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
6350 {
6351         struct page *page;
6352
6353         /* Defer reload until vmcs01 is the current VMCS. */
6354         if (is_guest_mode(vcpu)) {
6355                 to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true;
6356                 return;
6357         }
6358
6359         if (!(secondary_exec_controls_get(to_vmx(vcpu)) &
6360             SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
6361                 return;
6362
6363         page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
6364         if (is_error_page(page))
6365                 return;
6366
6367         vmcs_write64(APIC_ACCESS_ADDR, page_to_phys(page));
6368         vmx_flush_tlb_current(vcpu);
6369
6370         /*
6371          * Do not pin apic access page in memory, the MMU notifier
6372          * will call us again if it is migrated or swapped out.
6373          */
6374         put_page(page);
6375 }
6376
6377 static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
6378 {
6379         u16 status;
6380         u8 old;
6381
6382         if (max_isr == -1)
6383                 max_isr = 0;
6384
6385         status = vmcs_read16(GUEST_INTR_STATUS);
6386         old = status >> 8;
6387         if (max_isr != old) {
6388                 status &= 0xff;
6389                 status |= max_isr << 8;
6390                 vmcs_write16(GUEST_INTR_STATUS, status);
6391         }
6392 }
6393
6394 static void vmx_set_rvi(int vector)
6395 {
6396         u16 status;
6397         u8 old;
6398
6399         if (vector == -1)
6400                 vector = 0;
6401
6402         status = vmcs_read16(GUEST_INTR_STATUS);
6403         old = (u8)status & 0xff;
6404         if ((u8)vector != old) {
6405                 status &= ~0xff;
6406                 status |= (u8)vector;
6407                 vmcs_write16(GUEST_INTR_STATUS, status);
6408         }
6409 }
6410
6411 static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
6412 {
6413         /*
6414          * When running L2, updating RVI is only relevant when
6415          * vmcs12 virtual-interrupt-delivery enabled.
6416          * However, it can be enabled only when L1 also
6417          * intercepts external-interrupts and in that case
6418          * we should not update vmcs02 RVI but instead intercept
6419          * interrupt. Therefore, do nothing when running L2.
6420          */
6421         if (!is_guest_mode(vcpu))
6422                 vmx_set_rvi(max_irr);
6423 }
6424
6425 static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
6426 {
6427         struct vcpu_vmx *vmx = to_vmx(vcpu);
6428         int max_irr;
6429         bool got_posted_interrupt;
6430
6431         if (KVM_BUG_ON(!enable_apicv, vcpu->kvm))
6432                 return -EIO;
6433
6434         if (pi_test_on(&vmx->pi_desc)) {
6435                 pi_clear_on(&vmx->pi_desc);
6436                 /*
6437                  * IOMMU can write to PID.ON, so the barrier matters even on UP.
6438                  * But on x86 this is just a compiler barrier anyway.
6439                  */
6440                 smp_mb__after_atomic();
6441                 got_posted_interrupt =
6442                         kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr);
6443         } else {
6444                 max_irr = kvm_lapic_find_highest_irr(vcpu);
6445                 got_posted_interrupt = false;
6446         }
6447
6448         /*
6449          * Newly recognized interrupts are injected via either virtual interrupt
6450          * delivery (RVI) or KVM_REQ_EVENT.  Virtual interrupt delivery is
6451          * disabled in two cases:
6452          *
6453          * 1) If L2 is running and the vCPU has a new pending interrupt.  If L1
6454          * wants to exit on interrupts, KVM_REQ_EVENT is needed to synthesize a
6455          * VM-Exit to L1.  If L1 doesn't want to exit, the interrupt is injected
6456          * into L2, but KVM doesn't use virtual interrupt delivery to inject
6457          * interrupts into L2, and so KVM_REQ_EVENT is again needed.
6458          *
6459          * 2) If APICv is disabled for this vCPU, assigned devices may still
6460          * attempt to post interrupts.  The posted interrupt vector will cause
6461          * a VM-Exit and the subsequent entry will call sync_pir_to_irr.
6462          */
6463         if (!is_guest_mode(vcpu) && kvm_vcpu_apicv_active(vcpu))
6464                 vmx_set_rvi(max_irr);
6465         else if (got_posted_interrupt)
6466                 kvm_make_request(KVM_REQ_EVENT, vcpu);
6467
6468         return max_irr;
6469 }
6470
6471 static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
6472 {
6473         if (!kvm_vcpu_apicv_active(vcpu))
6474                 return;
6475
6476         vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
6477         vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
6478         vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
6479         vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
6480 }
6481
6482 static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu)
6483 {
6484         struct vcpu_vmx *vmx = to_vmx(vcpu);
6485
6486         pi_clear_on(&vmx->pi_desc);
6487         memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir));
6488 }
6489
6490 void vmx_do_interrupt_nmi_irqoff(unsigned long entry);
6491
6492 static void handle_interrupt_nmi_irqoff(struct kvm_vcpu *vcpu,
6493                                         unsigned long entry)
6494 {
6495         bool is_nmi = entry == (unsigned long)asm_exc_nmi_noist;
6496
6497         kvm_before_interrupt(vcpu, is_nmi ? KVM_HANDLING_NMI : KVM_HANDLING_IRQ);
6498         vmx_do_interrupt_nmi_irqoff(entry);
6499         kvm_after_interrupt(vcpu);
6500 }
6501
6502 static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
6503 {
6504         /*
6505          * Save xfd_err to guest_fpu before interrupt is enabled, so the
6506          * MSR value is not clobbered by the host activity before the guest
6507          * has chance to consume it.
6508          *
6509          * Do not blindly read xfd_err here, since this exception might
6510          * be caused by L1 interception on a platform which doesn't
6511          * support xfd at all.
6512          *
6513          * Do it conditionally upon guest_fpu::xfd. xfd_err matters
6514          * only when xfd contains a non-zero value.
6515          *
6516          * Queuing exception is done in vmx_handle_exit. See comment there.
6517          */
6518         if (vcpu->arch.guest_fpu.fpstate->xfd)
6519                 rdmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
6520 }
6521
6522 static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx)
6523 {
6524         const unsigned long nmi_entry = (unsigned long)asm_exc_nmi_noist;
6525         u32 intr_info = vmx_get_intr_info(&vmx->vcpu);
6526
6527         /* if exit due to PF check for async PF */
6528         if (is_page_fault(intr_info))
6529                 vmx->vcpu.arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
6530         /* if exit due to NM, handle before interrupts are enabled */
6531         else if (is_nm_fault(intr_info))
6532                 handle_nm_fault_irqoff(&vmx->vcpu);
6533         /* Handle machine checks before interrupts are enabled */
6534         else if (is_machine_check(intr_info))
6535                 kvm_machine_check();
6536         /* We need to handle NMIs before interrupts are enabled */
6537         else if (is_nmi(intr_info))
6538                 handle_interrupt_nmi_irqoff(&vmx->vcpu, nmi_entry);
6539 }
6540
6541 static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu)
6542 {
6543         u32 intr_info = vmx_get_intr_info(vcpu);
6544         unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK;
6545         gate_desc *desc = (gate_desc *)host_idt_base + vector;
6546
6547         if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm,
6548             "KVM: unexpected VM-Exit interrupt info: 0x%x", intr_info))
6549                 return;
6550
6551         handle_interrupt_nmi_irqoff(vcpu, gate_offset(desc));
6552 }
6553
6554 static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu)
6555 {
6556         struct vcpu_vmx *vmx = to_vmx(vcpu);
6557
6558         if (vmx->emulation_required)
6559                 return;
6560
6561         if (vmx->exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT)
6562                 handle_external_interrupt_irqoff(vcpu);
6563         else if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI)
6564                 handle_exception_nmi_irqoff(vmx);
6565 }
6566
6567 /*
6568  * The kvm parameter can be NULL (module initialization, or invocation before
6569  * VM creation). Be sure to check the kvm parameter before using it.
6570  */
6571 static bool vmx_has_emulated_msr(struct kvm *kvm, u32 index)
6572 {
6573         switch (index) {
6574         case MSR_IA32_SMBASE:
6575                 /*
6576                  * We cannot do SMM unless we can run the guest in big
6577                  * real mode.
6578                  */
6579                 return enable_unrestricted_guest || emulate_invalid_guest_state;
6580         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
6581                 return nested;
6582         case MSR_AMD64_VIRT_SPEC_CTRL:
6583         case MSR_AMD64_TSC_RATIO:
6584                 /* This is AMD only.  */
6585                 return false;
6586         default:
6587                 return true;
6588         }
6589 }
6590
6591 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
6592 {
6593         u32 exit_intr_info;
6594         bool unblock_nmi;
6595         u8 vector;
6596         bool idtv_info_valid;
6597
6598         idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
6599
6600         if (enable_vnmi) {
6601                 if (vmx->loaded_vmcs->nmi_known_unmasked)
6602                         return;
6603
6604                 exit_intr_info = vmx_get_intr_info(&vmx->vcpu);
6605                 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
6606                 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
6607                 /*
6608                  * SDM 3: 27.7.1.2 (September 2008)
6609                  * Re-set bit "block by NMI" before VM entry if vmexit caused by
6610                  * a guest IRET fault.
6611                  * SDM 3: 23.2.2 (September 2008)
6612                  * Bit 12 is undefined in any of the following cases:
6613                  *  If the VM exit sets the valid bit in the IDT-vectoring
6614                  *   information field.
6615                  *  If the VM exit is due to a double fault.
6616                  */
6617                 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
6618                     vector != DF_VECTOR && !idtv_info_valid)
6619                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
6620                                       GUEST_INTR_STATE_NMI);
6621                 else
6622                         vmx->loaded_vmcs->nmi_known_unmasked =
6623                                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
6624                                   & GUEST_INTR_STATE_NMI);
6625         } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked))
6626                 vmx->loaded_vmcs->vnmi_blocked_time +=
6627                         ktime_to_ns(ktime_sub(ktime_get(),
6628                                               vmx->loaded_vmcs->entry_time));
6629 }
6630
6631 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
6632                                       u32 idt_vectoring_info,
6633                                       int instr_len_field,
6634                                       int error_code_field)
6635 {
6636         u8 vector;
6637         int type;
6638         bool idtv_info_valid;
6639
6640         idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
6641
6642         vcpu->arch.nmi_injected = false;
6643         kvm_clear_exception_queue(vcpu);
6644         kvm_clear_interrupt_queue(vcpu);
6645
6646         if (!idtv_info_valid)
6647                 return;
6648
6649         kvm_make_request(KVM_REQ_EVENT, vcpu);
6650
6651         vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
6652         type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
6653
6654         switch (type) {
6655         case INTR_TYPE_NMI_INTR:
6656                 vcpu->arch.nmi_injected = true;
6657                 /*
6658                  * SDM 3: 27.7.1.2 (September 2008)
6659                  * Clear bit "block by NMI" before VM entry if a NMI
6660                  * delivery faulted.
6661                  */
6662                 vmx_set_nmi_mask(vcpu, false);
6663                 break;
6664         case INTR_TYPE_SOFT_EXCEPTION:
6665                 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
6666                 fallthrough;
6667         case INTR_TYPE_HARD_EXCEPTION:
6668                 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
6669                         u32 err = vmcs_read32(error_code_field);
6670                         kvm_requeue_exception_e(vcpu, vector, err);
6671                 } else
6672                         kvm_requeue_exception(vcpu, vector);
6673                 break;
6674         case INTR_TYPE_SOFT_INTR:
6675                 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
6676                 fallthrough;
6677         case INTR_TYPE_EXT_INTR:
6678                 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR);
6679                 break;
6680         default:
6681                 break;
6682         }
6683 }
6684
6685 static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
6686 {
6687         __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info,
6688                                   VM_EXIT_INSTRUCTION_LEN,
6689                                   IDT_VECTORING_ERROR_CODE);
6690 }
6691
6692 static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
6693 {
6694         __vmx_complete_interrupts(vcpu,
6695                                   vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
6696                                   VM_ENTRY_INSTRUCTION_LEN,
6697                                   VM_ENTRY_EXCEPTION_ERROR_CODE);
6698
6699         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
6700 }
6701
6702 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
6703 {
6704         int i, nr_msrs;
6705         struct perf_guest_switch_msr *msrs;
6706
6707         /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */
6708         msrs = perf_guest_get_msrs(&nr_msrs);
6709         if (!msrs)
6710                 return;
6711
6712         for (i = 0; i < nr_msrs; i++)
6713                 if (msrs[i].host == msrs[i].guest)
6714                         clear_atomic_switch_msr(vmx, msrs[i].msr);
6715                 else
6716                         add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
6717                                         msrs[i].host, false);
6718 }
6719
6720 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu)
6721 {
6722         struct vcpu_vmx *vmx = to_vmx(vcpu);
6723         u64 tscl;
6724         u32 delta_tsc;
6725
6726         if (vmx->req_immediate_exit) {
6727                 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0);
6728                 vmx->loaded_vmcs->hv_timer_soft_disabled = false;
6729         } else if (vmx->hv_deadline_tsc != -1) {
6730                 tscl = rdtsc();
6731                 if (vmx->hv_deadline_tsc > tscl)
6732                         /* set_hv_timer ensures the delta fits in 32-bits */
6733                         delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >>
6734                                 cpu_preemption_timer_multi);
6735                 else
6736                         delta_tsc = 0;
6737
6738                 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc);
6739                 vmx->loaded_vmcs->hv_timer_soft_disabled = false;
6740         } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) {
6741                 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1);
6742                 vmx->loaded_vmcs->hv_timer_soft_disabled = true;
6743         }
6744 }
6745
6746 void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp)
6747 {
6748         if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) {
6749                 vmx->loaded_vmcs->host_state.rsp = host_rsp;
6750                 vmcs_writel(HOST_RSP, host_rsp);
6751         }
6752 }
6753
6754 static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
6755 {
6756         switch (to_vmx(vcpu)->exit_reason.basic) {
6757         case EXIT_REASON_MSR_WRITE:
6758                 return handle_fastpath_set_msr_irqoff(vcpu);
6759         case EXIT_REASON_PREEMPTION_TIMER:
6760                 return handle_fastpath_preemption_timer(vcpu);
6761         default:
6762                 return EXIT_FASTPATH_NONE;
6763         }
6764 }
6765
6766 static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
6767                                         struct vcpu_vmx *vmx)
6768 {
6769         guest_state_enter_irqoff();
6770
6771         /* L1D Flush includes CPU buffer clear to mitigate MDS */
6772         if (static_branch_unlikely(&vmx_l1d_should_flush))
6773                 vmx_l1d_flush(vcpu);
6774         else if (static_branch_unlikely(&mds_user_clear))
6775                 mds_clear_cpu_buffers();
6776
6777         if (vcpu->arch.cr2 != native_read_cr2())
6778                 native_write_cr2(vcpu->arch.cr2);
6779
6780         vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,
6781                                    vmx->loaded_vmcs->launched);
6782
6783         vcpu->arch.cr2 = native_read_cr2();
6784
6785         guest_state_exit_irqoff();
6786 }
6787
6788 static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)
6789 {
6790         struct vcpu_vmx *vmx = to_vmx(vcpu);
6791         unsigned long cr3, cr4;
6792
6793         /* Record the guest's net vcpu time for enforced NMI injections. */
6794         if (unlikely(!enable_vnmi &&
6795                      vmx->loaded_vmcs->soft_vnmi_blocked))
6796                 vmx->loaded_vmcs->entry_time = ktime_get();
6797
6798         /*
6799          * Don't enter VMX if guest state is invalid, let the exit handler
6800          * start emulation until we arrive back to a valid state.  Synthesize a
6801          * consistency check VM-Exit due to invalid guest state and bail.
6802          */
6803         if (unlikely(vmx->emulation_required)) {
6804                 vmx->fail = 0;
6805
6806                 vmx->exit_reason.full = EXIT_REASON_INVALID_STATE;
6807                 vmx->exit_reason.failed_vmentry = 1;
6808                 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1);
6809                 vmx->exit_qualification = ENTRY_FAIL_DEFAULT;
6810                 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2);
6811                 vmx->exit_intr_info = 0;
6812                 return EXIT_FASTPATH_NONE;
6813         }
6814
6815         trace_kvm_entry(vcpu);
6816
6817         if (vmx->ple_window_dirty) {
6818                 vmx->ple_window_dirty = false;
6819                 vmcs_write32(PLE_WINDOW, vmx->ple_window);
6820         }
6821
6822         /*
6823          * We did this in prepare_switch_to_guest, because it needs to
6824          * be within srcu_read_lock.
6825          */
6826         WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync);
6827
6828         if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP))
6829                 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
6830         if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP))
6831                 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
6832         vcpu->arch.regs_dirty = 0;
6833
6834         /*
6835          * Refresh vmcs.HOST_CR3 if necessary.  This must be done immediately
6836          * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time
6837          * it switches back to the current->mm, which can occur in KVM context
6838          * when switching to a temporary mm to patch kernel code, e.g. if KVM
6839          * toggles a static key while handling a VM-Exit.
6840          */
6841         cr3 = __get_current_cr3_fast();
6842         if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
6843                 vmcs_writel(HOST_CR3, cr3);
6844                 vmx->loaded_vmcs->host_state.cr3 = cr3;
6845         }
6846
6847         cr4 = cr4_read_shadow();
6848         if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
6849                 vmcs_writel(HOST_CR4, cr4);
6850                 vmx->loaded_vmcs->host_state.cr4 = cr4;
6851         }
6852
6853         /* When KVM_DEBUGREG_WONT_EXIT, dr6 is accessible in guest. */
6854         if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
6855                 set_debugreg(vcpu->arch.dr6, 6);
6856
6857         /* When single-stepping over STI and MOV SS, we must clear the
6858          * corresponding interruptibility bits in the guest state. Otherwise
6859          * vmentry fails as it then expects bit 14 (BS) in pending debug
6860          * exceptions being set, but that's not correct for the guest debugging
6861          * case. */
6862         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
6863                 vmx_set_interrupt_shadow(vcpu, 0);
6864
6865         kvm_load_guest_xsave_state(vcpu);
6866
6867         pt_guest_enter(vmx);
6868
6869         atomic_switch_perf_msrs(vmx);
6870         if (intel_pmu_lbr_is_enabled(vcpu))
6871                 vmx_passthrough_lbr_msrs(vcpu);
6872
6873         if (enable_preemption_timer)
6874                 vmx_update_hv_timer(vcpu);
6875
6876         kvm_wait_lapic_expire(vcpu);
6877
6878         /*
6879          * If this vCPU has touched SPEC_CTRL, restore the guest's value if
6880          * it's non-zero. Since vmentry is serialising on affected CPUs, there
6881          * is no need to worry about the conditional branch over the wrmsr
6882          * being speculatively taken.
6883          */
6884         x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0);
6885
6886         /* The actual VMENTER/EXIT is in the .noinstr.text section. */
6887         vmx_vcpu_enter_exit(vcpu, vmx);
6888
6889         /*
6890          * We do not use IBRS in the kernel. If this vCPU has used the
6891          * SPEC_CTRL MSR it may have left it on; save the value and
6892          * turn it off. This is much more efficient than blindly adding
6893          * it to the atomic save/restore list. Especially as the former
6894          * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
6895          *
6896          * For non-nested case:
6897          * If the L01 MSR bitmap does not intercept the MSR, then we need to
6898          * save it.
6899          *
6900          * For nested case:
6901          * If the L02 MSR bitmap does not intercept the MSR, then we need to
6902          * save it.
6903          */
6904         if (unlikely(!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL)))
6905                 vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
6906
6907         x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);
6908
6909         /* All fields are clean at this point */
6910         if (static_branch_unlikely(&enable_evmcs)) {
6911                 current_evmcs->hv_clean_fields |=
6912                         HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
6913
6914                 current_evmcs->hv_vp_id = kvm_hv_get_vpindex(vcpu);
6915         }
6916
6917         /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
6918         if (vmx->host_debugctlmsr)
6919                 update_debugctlmsr(vmx->host_debugctlmsr);
6920
6921 #ifndef CONFIG_X86_64
6922         /*
6923          * The sysexit path does not restore ds/es, so we must set them to
6924          * a reasonable value ourselves.
6925          *
6926          * We can't defer this to vmx_prepare_switch_to_host() since that
6927          * function may be executed in interrupt context, which saves and
6928          * restore segments around it, nullifying its effect.
6929          */
6930         loadsegment(ds, __USER_DS);
6931         loadsegment(es, __USER_DS);
6932 #endif
6933
6934         vcpu->arch.regs_avail &= ~VMX_REGS_LAZY_LOAD_SET;
6935
6936         pt_guest_exit(vmx);
6937
6938         kvm_load_host_xsave_state(vcpu);
6939
6940         if (is_guest_mode(vcpu)) {
6941                 /*
6942                  * Track VMLAUNCH/VMRESUME that have made past guest state
6943                  * checking.
6944                  */
6945                 if (vmx->nested.nested_run_pending &&
6946                     !vmx->exit_reason.failed_vmentry)
6947                         ++vcpu->stat.nested_run;
6948
6949                 vmx->nested.nested_run_pending = 0;
6950         }
6951
6952         vmx->idt_vectoring_info = 0;
6953
6954         if (unlikely(vmx->fail)) {
6955                 vmx->exit_reason.full = 0xdead;
6956                 return EXIT_FASTPATH_NONE;
6957         }
6958
6959         vmx->exit_reason.full = vmcs_read32(VM_EXIT_REASON);
6960         if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY))
6961                 kvm_machine_check();
6962
6963         if (likely(!vmx->exit_reason.failed_vmentry))
6964                 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
6965
6966         trace_kvm_exit(vcpu, KVM_ISA_VMX);
6967
6968         if (unlikely(vmx->exit_reason.failed_vmentry))
6969                 return EXIT_FASTPATH_NONE;
6970
6971         vmx->loaded_vmcs->launched = 1;
6972
6973         vmx_recover_nmi_blocking(vmx);
6974         vmx_complete_interrupts(vmx);
6975
6976         if (is_guest_mode(vcpu))
6977                 return EXIT_FASTPATH_NONE;
6978
6979         return vmx_exit_handlers_fastpath(vcpu);
6980 }
6981
6982 static void vmx_vcpu_free(struct kvm_vcpu *vcpu)
6983 {
6984         struct vcpu_vmx *vmx = to_vmx(vcpu);
6985
6986         if (enable_pml)
6987                 vmx_destroy_pml_buffer(vmx);
6988         free_vpid(vmx->vpid);
6989         nested_vmx_free_vcpu(vcpu);
6990         free_loaded_vmcs(vmx->loaded_vmcs);
6991 }
6992
6993 static int vmx_vcpu_create(struct kvm_vcpu *vcpu)
6994 {
6995         struct vmx_uret_msr *tsx_ctrl;
6996         struct vcpu_vmx *vmx;
6997         int i, err;
6998
6999         BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0);
7000         vmx = to_vmx(vcpu);
7001
7002         INIT_LIST_HEAD(&vmx->pi_wakeup_list);
7003
7004         err = -ENOMEM;
7005
7006         vmx->vpid = allocate_vpid();
7007
7008         /*
7009          * If PML is turned on, failure on enabling PML just results in failure
7010          * of creating the vcpu, therefore we can simplify PML logic (by
7011          * avoiding dealing with cases, such as enabling PML partially on vcpus
7012          * for the guest), etc.
7013          */
7014         if (enable_pml) {
7015                 vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
7016                 if (!vmx->pml_pg)
7017                         goto free_vpid;
7018         }
7019
7020         for (i = 0; i < kvm_nr_uret_msrs; ++i)
7021                 vmx->guest_uret_msrs[i].mask = -1ull;
7022         if (boot_cpu_has(X86_FEATURE_RTM)) {
7023                 /*
7024                  * TSX_CTRL_CPUID_CLEAR is handled in the CPUID interception.
7025                  * Keep the host value unchanged to avoid changing CPUID bits
7026                  * under the host kernel's feet.
7027                  */
7028                 tsx_ctrl = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL);
7029                 if (tsx_ctrl)
7030                         tsx_ctrl->mask = ~(u64)TSX_CTRL_CPUID_CLEAR;
7031         }
7032
7033         err = alloc_loaded_vmcs(&vmx->vmcs01);
7034         if (err < 0)
7035                 goto free_pml;
7036
7037         /*
7038          * Use Hyper-V 'Enlightened MSR Bitmap' feature when KVM runs as a
7039          * nested (L1) hypervisor and Hyper-V in L0 supports it. Enable the
7040          * feature only for vmcs01, KVM currently isn't equipped to realize any
7041          * performance benefits from enabling it for vmcs02.
7042          */
7043         if (IS_ENABLED(CONFIG_HYPERV) && static_branch_unlikely(&enable_evmcs) &&
7044             (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) {
7045                 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs;
7046
7047                 evmcs->hv_enlightenments_control.msr_bitmap = 1;
7048         }
7049
7050         /* The MSR bitmap starts with all ones */
7051         bitmap_fill(vmx->shadow_msr_intercept.read, MAX_POSSIBLE_PASSTHROUGH_MSRS);
7052         bitmap_fill(vmx->shadow_msr_intercept.write, MAX_POSSIBLE_PASSTHROUGH_MSRS);
7053
7054         vmx_disable_intercept_for_msr(vcpu, MSR_IA32_TSC, MSR_TYPE_R);
7055 #ifdef CONFIG_X86_64
7056         vmx_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW);
7057         vmx_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW);
7058         vmx_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
7059 #endif
7060         vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW);
7061         vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW);
7062         vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW);
7063         if (kvm_cstate_in_guest(vcpu->kvm)) {
7064                 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C1_RES, MSR_TYPE_R);
7065                 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C3_RESIDENCY, MSR_TYPE_R);
7066                 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C6_RESIDENCY, MSR_TYPE_R);
7067                 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C7_RESIDENCY, MSR_TYPE_R);
7068         }
7069
7070         vmx->loaded_vmcs = &vmx->vmcs01;
7071
7072         if (cpu_need_virtualize_apic_accesses(vcpu)) {
7073                 err = alloc_apic_access_page(vcpu->kvm);
7074                 if (err)
7075                         goto free_vmcs;
7076         }
7077
7078         if (enable_ept && !enable_unrestricted_guest) {
7079                 err = init_rmode_identity_map(vcpu->kvm);
7080                 if (err)
7081                         goto free_vmcs;
7082         }
7083
7084         return 0;
7085
7086 free_vmcs:
7087         free_loaded_vmcs(vmx->loaded_vmcs);
7088 free_pml:
7089         vmx_destroy_pml_buffer(vmx);
7090 free_vpid:
7091         free_vpid(vmx->vpid);
7092         return err;
7093 }
7094
7095 #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n"
7096 #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n"
7097
7098 static int vmx_vm_init(struct kvm *kvm)
7099 {
7100         if (!ple_gap)
7101                 kvm->arch.pause_in_guest = true;
7102
7103         if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) {
7104                 switch (l1tf_mitigation) {
7105                 case L1TF_MITIGATION_OFF:
7106                 case L1TF_MITIGATION_FLUSH_NOWARN:
7107                         /* 'I explicitly don't care' is set */
7108                         break;
7109                 case L1TF_MITIGATION_FLUSH:
7110                 case L1TF_MITIGATION_FLUSH_NOSMT:
7111                 case L1TF_MITIGATION_FULL:
7112                         /*
7113                          * Warn upon starting the first VM in a potentially
7114                          * insecure environment.
7115                          */
7116                         if (sched_smt_active())
7117                                 pr_warn_once(L1TF_MSG_SMT);
7118                         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER)
7119                                 pr_warn_once(L1TF_MSG_L1D);
7120                         break;
7121                 case L1TF_MITIGATION_FULL_FORCE:
7122                         /* Flush is enforced */
7123                         break;
7124                 }
7125         }
7126         return 0;
7127 }
7128
7129 static int __init vmx_check_processor_compat(void)
7130 {
7131         struct vmcs_config vmcs_conf;
7132         struct vmx_capability vmx_cap;
7133
7134         if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
7135             !this_cpu_has(X86_FEATURE_VMX)) {
7136                 pr_err("kvm: VMX is disabled on CPU %d\n", smp_processor_id());
7137                 return -EIO;
7138         }
7139
7140         if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0)
7141                 return -EIO;
7142         if (nested)
7143                 nested_vmx_setup_ctls_msrs(&vmcs_conf.nested, vmx_cap.ept);
7144         if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
7145                 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
7146                                 smp_processor_id());
7147                 return -EIO;
7148         }
7149         return 0;
7150 }
7151
7152 static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
7153 {
7154         u8 cache;
7155
7156         /* We wanted to honor guest CD/MTRR/PAT, but doing so could result in
7157          * memory aliases with conflicting memory types and sometimes MCEs.
7158          * We have to be careful as to what are honored and when.
7159          *
7160          * For MMIO, guest CD/MTRR are ignored.  The EPT memory type is set to
7161          * UC.  The effective memory type is UC or WC depending on guest PAT.
7162          * This was historically the source of MCEs and we want to be
7163          * conservative.
7164          *
7165          * When there is no need to deal with noncoherent DMA (e.g., no VT-d
7166          * or VT-d has snoop control), guest CD/MTRR/PAT are all ignored.  The
7167          * EPT memory type is set to WB.  The effective memory type is forced
7168          * WB.
7169          *
7170          * Otherwise, we trust guest.  Guest CD/MTRR/PAT are all honored.  The
7171          * EPT memory type is used to emulate guest CD/MTRR.
7172          */
7173
7174         if (is_mmio)
7175                 return MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
7176
7177         if (!kvm_arch_has_noncoherent_dma(vcpu->kvm))
7178                 return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT;
7179
7180         if (kvm_read_cr0(vcpu) & X86_CR0_CD) {
7181                 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
7182                         cache = MTRR_TYPE_WRBACK;
7183                 else
7184                         cache = MTRR_TYPE_UNCACHABLE;
7185
7186                 return (cache << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT;
7187         }
7188
7189         return kvm_mtrr_get_guest_memory_type(vcpu, gfn) << VMX_EPT_MT_EPTE_SHIFT;
7190 }
7191
7192 static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx, u32 new_ctl)
7193 {
7194         /*
7195          * These bits in the secondary execution controls field
7196          * are dynamic, the others are mostly based on the hypervisor
7197          * architecture and the guest's CPUID.  Do not touch the
7198          * dynamic bits.
7199          */
7200         u32 mask =
7201                 SECONDARY_EXEC_SHADOW_VMCS |
7202                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
7203                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
7204                 SECONDARY_EXEC_DESC;
7205
7206         u32 cur_ctl = secondary_exec_controls_get(vmx);
7207
7208         secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask));
7209 }
7210
7211 /*
7212  * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits
7213  * (indicating "allowed-1") if they are supported in the guest's CPUID.
7214  */
7215 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu)
7216 {
7217         struct vcpu_vmx *vmx = to_vmx(vcpu);
7218         struct kvm_cpuid_entry2 *entry;
7219
7220         vmx->nested.msrs.cr0_fixed1 = 0xffffffff;
7221         vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE;
7222
7223 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do {            \
7224         if (entry && (entry->_reg & (_cpuid_mask)))                     \
7225                 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask);     \
7226 } while (0)
7227
7228         entry = kvm_find_cpuid_entry(vcpu, 0x1, 0);
7229         cr4_fixed1_update(X86_CR4_VME,        edx, feature_bit(VME));
7230         cr4_fixed1_update(X86_CR4_PVI,        edx, feature_bit(VME));
7231         cr4_fixed1_update(X86_CR4_TSD,        edx, feature_bit(TSC));
7232         cr4_fixed1_update(X86_CR4_DE,         edx, feature_bit(DE));
7233         cr4_fixed1_update(X86_CR4_PSE,        edx, feature_bit(PSE));
7234         cr4_fixed1_update(X86_CR4_PAE,        edx, feature_bit(PAE));
7235         cr4_fixed1_update(X86_CR4_MCE,        edx, feature_bit(MCE));
7236         cr4_fixed1_update(X86_CR4_PGE,        edx, feature_bit(PGE));
7237         cr4_fixed1_update(X86_CR4_OSFXSR,     edx, feature_bit(FXSR));
7238         cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, feature_bit(XMM));
7239         cr4_fixed1_update(X86_CR4_VMXE,       ecx, feature_bit(VMX));
7240         cr4_fixed1_update(X86_CR4_SMXE,       ecx, feature_bit(SMX));
7241         cr4_fixed1_update(X86_CR4_PCIDE,      ecx, feature_bit(PCID));
7242         cr4_fixed1_update(X86_CR4_OSXSAVE,    ecx, feature_bit(XSAVE));
7243
7244         entry = kvm_find_cpuid_entry(vcpu, 0x7, 0);
7245         cr4_fixed1_update(X86_CR4_FSGSBASE,   ebx, feature_bit(FSGSBASE));
7246         cr4_fixed1_update(X86_CR4_SMEP,       ebx, feature_bit(SMEP));
7247         cr4_fixed1_update(X86_CR4_SMAP,       ebx, feature_bit(SMAP));
7248         cr4_fixed1_update(X86_CR4_PKE,        ecx, feature_bit(PKU));
7249         cr4_fixed1_update(X86_CR4_UMIP,       ecx, feature_bit(UMIP));
7250         cr4_fixed1_update(X86_CR4_LA57,       ecx, feature_bit(LA57));
7251
7252 #undef cr4_fixed1_update
7253 }
7254
7255 static void nested_vmx_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
7256 {
7257         struct vcpu_vmx *vmx = to_vmx(vcpu);
7258
7259         if (kvm_mpx_supported()) {
7260                 bool mpx_enabled = guest_cpuid_has(vcpu, X86_FEATURE_MPX);
7261
7262                 if (mpx_enabled) {
7263                         vmx->nested.msrs.entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
7264                         vmx->nested.msrs.exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
7265                 } else {
7266                         vmx->nested.msrs.entry_ctls_high &= ~VM_ENTRY_LOAD_BNDCFGS;
7267                         vmx->nested.msrs.exit_ctls_high &= ~VM_EXIT_CLEAR_BNDCFGS;
7268                 }
7269         }
7270 }
7271
7272 static void update_intel_pt_cfg(struct kvm_vcpu *vcpu)
7273 {
7274         struct vcpu_vmx *vmx = to_vmx(vcpu);
7275         struct kvm_cpuid_entry2 *best = NULL;
7276         int i;
7277
7278         for (i = 0; i < PT_CPUID_LEAVES; i++) {
7279                 best = kvm_find_cpuid_entry(vcpu, 0x14, i);
7280                 if (!best)
7281                         return;
7282                 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax;
7283                 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx;
7284                 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx;
7285                 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx;
7286         }
7287
7288         /* Get the number of configurable Address Ranges for filtering */
7289         vmx->pt_desc.num_address_ranges = intel_pt_validate_cap(vmx->pt_desc.caps,
7290                                                 PT_CAP_num_address_ranges);
7291
7292         /* Initialize and clear the no dependency bits */
7293         vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS |
7294                         RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC |
7295                         RTIT_CTL_BRANCH_EN);
7296
7297         /*
7298          * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise
7299          * will inject an #GP
7300          */
7301         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering))
7302                 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN;
7303
7304         /*
7305          * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and
7306          * PSBFreq can be set
7307          */
7308         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc))
7309                 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC |
7310                                 RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ);
7311
7312         /*
7313          * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn and MTCFreq can be set
7314          */
7315         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc))
7316                 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN |
7317                                               RTIT_CTL_MTC_RANGE);
7318
7319         /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */
7320         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite))
7321                 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW |
7322                                                         RTIT_CTL_PTW_EN);
7323
7324         /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */
7325         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace))
7326                 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN;
7327
7328         /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */
7329         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output))
7330                 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA;
7331
7332         /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabricEn can be set */
7333         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys))
7334                 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN;
7335
7336         /* unmask address range configure area */
7337         for (i = 0; i < vmx->pt_desc.num_address_ranges; i++)
7338                 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4));
7339 }
7340
7341 static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
7342 {
7343         struct vcpu_vmx *vmx = to_vmx(vcpu);
7344
7345         /* xsaves_enabled is recomputed in vmx_compute_secondary_exec_control(). */
7346         vcpu->arch.xsaves_enabled = false;
7347
7348         vmx_setup_uret_msrs(vmx);
7349
7350         if (cpu_has_secondary_exec_ctrls())
7351                 vmcs_set_secondary_exec_control(vmx,
7352                                                 vmx_secondary_exec_control(vmx));
7353
7354         if (nested_vmx_allowed(vcpu))
7355                 vmx->msr_ia32_feature_control_valid_bits |=
7356                         FEAT_CTL_VMX_ENABLED_INSIDE_SMX |
7357                         FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX;
7358         else
7359                 vmx->msr_ia32_feature_control_valid_bits &=
7360                         ~(FEAT_CTL_VMX_ENABLED_INSIDE_SMX |
7361                           FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX);
7362
7363         if (nested_vmx_allowed(vcpu)) {
7364                 nested_vmx_cr_fixed1_bits_update(vcpu);
7365                 nested_vmx_entry_exit_ctls_update(vcpu);
7366         }
7367
7368         if (boot_cpu_has(X86_FEATURE_INTEL_PT) &&
7369                         guest_cpuid_has(vcpu, X86_FEATURE_INTEL_PT))
7370                 update_intel_pt_cfg(vcpu);
7371
7372         if (boot_cpu_has(X86_FEATURE_RTM)) {
7373                 struct vmx_uret_msr *msr;
7374                 msr = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL);
7375                 if (msr) {
7376                         bool enabled = guest_cpuid_has(vcpu, X86_FEATURE_RTM);
7377                         vmx_set_guest_uret_msr(vmx, msr, enabled ? 0 : TSX_CTRL_RTM_DISABLE);
7378                 }
7379         }
7380
7381         if (kvm_cpu_cap_has(X86_FEATURE_XFD))
7382                 vmx_set_intercept_for_msr(vcpu, MSR_IA32_XFD_ERR, MSR_TYPE_R,
7383                                           !guest_cpuid_has(vcpu, X86_FEATURE_XFD));
7384
7385
7386         set_cr4_guest_host_mask(vmx);
7387
7388         vmx_write_encls_bitmap(vcpu, NULL);
7389         if (guest_cpuid_has(vcpu, X86_FEATURE_SGX))
7390                 vmx->msr_ia32_feature_control_valid_bits |= FEAT_CTL_SGX_ENABLED;
7391         else
7392                 vmx->msr_ia32_feature_control_valid_bits &= ~FEAT_CTL_SGX_ENABLED;
7393
7394         if (guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC))
7395                 vmx->msr_ia32_feature_control_valid_bits |=
7396                         FEAT_CTL_SGX_LC_ENABLED;
7397         else
7398                 vmx->msr_ia32_feature_control_valid_bits &=
7399                         ~FEAT_CTL_SGX_LC_ENABLED;
7400
7401         /* Refresh #PF interception to account for MAXPHYADDR changes. */
7402         vmx_update_exception_bitmap(vcpu);
7403 }
7404
7405 static __init void vmx_set_cpu_caps(void)
7406 {
7407         kvm_set_cpu_caps();
7408
7409         /* CPUID 0x1 */
7410         if (nested)
7411                 kvm_cpu_cap_set(X86_FEATURE_VMX);
7412
7413         /* CPUID 0x7 */
7414         if (kvm_mpx_supported())
7415                 kvm_cpu_cap_check_and_set(X86_FEATURE_MPX);
7416         if (!cpu_has_vmx_invpcid())
7417                 kvm_cpu_cap_clear(X86_FEATURE_INVPCID);
7418         if (vmx_pt_mode_is_host_guest())
7419                 kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT);
7420
7421         if (!enable_sgx) {
7422                 kvm_cpu_cap_clear(X86_FEATURE_SGX);
7423                 kvm_cpu_cap_clear(X86_FEATURE_SGX_LC);
7424                 kvm_cpu_cap_clear(X86_FEATURE_SGX1);
7425                 kvm_cpu_cap_clear(X86_FEATURE_SGX2);
7426         }
7427
7428         if (vmx_umip_emulated())
7429                 kvm_cpu_cap_set(X86_FEATURE_UMIP);
7430
7431         /* CPUID 0xD.1 */
7432         supported_xss = 0;
7433         if (!cpu_has_vmx_xsaves())
7434                 kvm_cpu_cap_clear(X86_FEATURE_XSAVES);
7435
7436         /* CPUID 0x80000001 and 0x7 (RDPID) */
7437         if (!cpu_has_vmx_rdtscp()) {
7438                 kvm_cpu_cap_clear(X86_FEATURE_RDTSCP);
7439                 kvm_cpu_cap_clear(X86_FEATURE_RDPID);
7440         }
7441
7442         if (cpu_has_vmx_waitpkg())
7443                 kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG);
7444 }
7445
7446 static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu)
7447 {
7448         to_vmx(vcpu)->req_immediate_exit = true;
7449 }
7450
7451 static int vmx_check_intercept_io(struct kvm_vcpu *vcpu,
7452                                   struct x86_instruction_info *info)
7453 {
7454         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
7455         unsigned short port;
7456         bool intercept;
7457         int size;
7458
7459         if (info->intercept == x86_intercept_in ||
7460             info->intercept == x86_intercept_ins) {
7461                 port = info->src_val;
7462                 size = info->dst_bytes;
7463         } else {
7464                 port = info->dst_val;
7465                 size = info->src_bytes;
7466         }
7467
7468         /*
7469          * If the 'use IO bitmaps' VM-execution control is 0, IO instruction
7470          * VM-exits depend on the 'unconditional IO exiting' VM-execution
7471          * control.
7472          *
7473          * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps.
7474          */
7475         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
7476                 intercept = nested_cpu_has(vmcs12,
7477                                            CPU_BASED_UNCOND_IO_EXITING);
7478         else
7479                 intercept = nested_vmx_check_io_bitmaps(vcpu, port, size);
7480
7481         /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED.  */
7482         return intercept ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
7483 }
7484
7485 static int vmx_check_intercept(struct kvm_vcpu *vcpu,
7486                                struct x86_instruction_info *info,
7487                                enum x86_intercept_stage stage,
7488                                struct x86_exception *exception)
7489 {
7490         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
7491
7492         switch (info->intercept) {
7493         /*
7494          * RDPID causes #UD if disabled through secondary execution controls.
7495          * Because it is marked as EmulateOnUD, we need to intercept it here.
7496          * Note, RDPID is hidden behind ENABLE_RDTSCP.
7497          */
7498         case x86_intercept_rdpid:
7499                 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_RDTSCP)) {
7500                         exception->vector = UD_VECTOR;
7501                         exception->error_code_valid = false;
7502                         return X86EMUL_PROPAGATE_FAULT;
7503                 }
7504                 break;
7505
7506         case x86_intercept_in:
7507         case x86_intercept_ins:
7508         case x86_intercept_out:
7509         case x86_intercept_outs:
7510                 return vmx_check_intercept_io(vcpu, info);
7511
7512         case x86_intercept_lgdt:
7513         case x86_intercept_lidt:
7514         case x86_intercept_lldt:
7515         case x86_intercept_ltr:
7516         case x86_intercept_sgdt:
7517         case x86_intercept_sidt:
7518         case x86_intercept_sldt:
7519         case x86_intercept_str:
7520                 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC))
7521                         return X86EMUL_CONTINUE;
7522
7523                 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED.  */
7524                 break;
7525
7526         /* TODO: check more intercepts... */
7527         default:
7528                 break;
7529         }
7530
7531         return X86EMUL_UNHANDLEABLE;
7532 }
7533
7534 #ifdef CONFIG_X86_64
7535 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */
7536 static inline int u64_shl_div_u64(u64 a, unsigned int shift,
7537                                   u64 divisor, u64 *result)
7538 {
7539         u64 low = a << shift, high = a >> (64 - shift);
7540
7541         /* To avoid the overflow on divq */
7542         if (high >= divisor)
7543                 return 1;
7544
7545         /* Low hold the result, high hold rem which is discarded */
7546         asm("divq %2\n\t" : "=a" (low), "=d" (high) :
7547             "rm" (divisor), "0" (low), "1" (high));
7548         *result = low;
7549
7550         return 0;
7551 }
7552
7553 static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
7554                             bool *expired)
7555 {
7556         struct vcpu_vmx *vmx;
7557         u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles;
7558         struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer;
7559
7560         vmx = to_vmx(vcpu);
7561         tscl = rdtsc();
7562         guest_tscl = kvm_read_l1_tsc(vcpu, tscl);
7563         delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl;
7564         lapic_timer_advance_cycles = nsec_to_cycles(vcpu,
7565                                                     ktimer->timer_advance_ns);
7566
7567         if (delta_tsc > lapic_timer_advance_cycles)
7568                 delta_tsc -= lapic_timer_advance_cycles;
7569         else
7570                 delta_tsc = 0;
7571
7572         /* Convert to host delta tsc if tsc scaling is enabled */
7573         if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio &&
7574             delta_tsc && u64_shl_div_u64(delta_tsc,
7575                                 kvm_tsc_scaling_ratio_frac_bits,
7576                                 vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc))
7577                 return -ERANGE;
7578
7579         /*
7580          * If the delta tsc can't fit in the 32 bit after the multi shift,
7581          * we can't use the preemption timer.
7582          * It's possible that it fits on later vmentries, but checking
7583          * on every vmentry is costly so we just use an hrtimer.
7584          */
7585         if (delta_tsc >> (cpu_preemption_timer_multi + 32))
7586                 return -ERANGE;
7587
7588         vmx->hv_deadline_tsc = tscl + delta_tsc;
7589         *expired = !delta_tsc;
7590         return 0;
7591 }
7592
7593 static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
7594 {
7595         to_vmx(vcpu)->hv_deadline_tsc = -1;
7596 }
7597 #endif
7598
7599 static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
7600 {
7601         if (!kvm_pause_in_guest(vcpu->kvm))
7602                 shrink_ple_window(vcpu);
7603 }
7604
7605 void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
7606 {
7607         struct vcpu_vmx *vmx = to_vmx(vcpu);
7608
7609         if (is_guest_mode(vcpu)) {
7610                 vmx->nested.update_vmcs01_cpu_dirty_logging = true;
7611                 return;
7612         }
7613
7614         /*
7615          * Note, cpu_dirty_logging_count can be changed concurrent with this
7616          * code, but in that case another update request will be made and so
7617          * the guest will never run with a stale PML value.
7618          */
7619         if (vcpu->kvm->arch.cpu_dirty_logging_count)
7620                 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML);
7621         else
7622                 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML);
7623 }
7624
7625 static void vmx_setup_mce(struct kvm_vcpu *vcpu)
7626 {
7627         if (vcpu->arch.mcg_cap & MCG_LMCE_P)
7628                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
7629                         FEAT_CTL_LMCE_ENABLED;
7630         else
7631                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
7632                         ~FEAT_CTL_LMCE_ENABLED;
7633 }
7634
7635 static int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
7636 {
7637         /* we need a nested vmexit to enter SMM, postpone if run is pending */
7638         if (to_vmx(vcpu)->nested.nested_run_pending)
7639                 return -EBUSY;
7640         return !is_smm(vcpu);
7641 }
7642
7643 static int vmx_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
7644 {
7645         struct vcpu_vmx *vmx = to_vmx(vcpu);
7646
7647         vmx->nested.smm.guest_mode = is_guest_mode(vcpu);
7648         if (vmx->nested.smm.guest_mode)
7649                 nested_vmx_vmexit(vcpu, -1, 0, 0);
7650
7651         vmx->nested.smm.vmxon = vmx->nested.vmxon;
7652         vmx->nested.vmxon = false;
7653         vmx_clear_hlt(vcpu);
7654         return 0;
7655 }
7656
7657 static int vmx_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
7658 {
7659         struct vcpu_vmx *vmx = to_vmx(vcpu);
7660         int ret;
7661
7662         if (vmx->nested.smm.vmxon) {
7663                 vmx->nested.vmxon = true;
7664                 vmx->nested.smm.vmxon = false;
7665         }
7666
7667         if (vmx->nested.smm.guest_mode) {
7668                 ret = nested_vmx_enter_non_root_mode(vcpu, false);
7669                 if (ret)
7670                         return ret;
7671
7672                 vmx->nested.nested_run_pending = 1;
7673                 vmx->nested.smm.guest_mode = false;
7674         }
7675         return 0;
7676 }
7677
7678 static void vmx_enable_smi_window(struct kvm_vcpu *vcpu)
7679 {
7680         /* RSM will cause a vmexit anyway.  */
7681 }
7682
7683 static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
7684 {
7685         return to_vmx(vcpu)->nested.vmxon && !is_guest_mode(vcpu);
7686 }
7687
7688 static void vmx_migrate_timers(struct kvm_vcpu *vcpu)
7689 {
7690         if (is_guest_mode(vcpu)) {
7691                 struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer;
7692
7693                 if (hrtimer_try_to_cancel(timer) == 1)
7694                         hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
7695         }
7696 }
7697
7698 static void vmx_hardware_unsetup(void)
7699 {
7700         kvm_set_posted_intr_wakeup_handler(NULL);
7701
7702         if (nested)
7703                 nested_vmx_hardware_unsetup();
7704
7705         free_kvm_area();
7706 }
7707
7708 static bool vmx_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason)
7709 {
7710         ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) |
7711                           BIT(APICV_INHIBIT_REASON_ABSENT) |
7712                           BIT(APICV_INHIBIT_REASON_HYPERV) |
7713                           BIT(APICV_INHIBIT_REASON_BLOCKIRQ);
7714
7715         return supported & BIT(reason);
7716 }
7717
7718 static struct kvm_x86_ops vmx_x86_ops __initdata = {
7719         .name = "kvm_intel",
7720
7721         .hardware_unsetup = vmx_hardware_unsetup,
7722
7723         .hardware_enable = vmx_hardware_enable,
7724         .hardware_disable = vmx_hardware_disable,
7725         .has_emulated_msr = vmx_has_emulated_msr,
7726
7727         .vm_size = sizeof(struct kvm_vmx),
7728         .vm_init = vmx_vm_init,
7729
7730         .vcpu_create = vmx_vcpu_create,
7731         .vcpu_free = vmx_vcpu_free,
7732         .vcpu_reset = vmx_vcpu_reset,
7733
7734         .prepare_switch_to_guest = vmx_prepare_switch_to_guest,
7735         .vcpu_load = vmx_vcpu_load,
7736         .vcpu_put = vmx_vcpu_put,
7737
7738         .update_exception_bitmap = vmx_update_exception_bitmap,
7739         .get_msr_feature = vmx_get_msr_feature,
7740         .get_msr = vmx_get_msr,
7741         .set_msr = vmx_set_msr,
7742         .get_segment_base = vmx_get_segment_base,
7743         .get_segment = vmx_get_segment,
7744         .set_segment = vmx_set_segment,
7745         .get_cpl = vmx_get_cpl,
7746         .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
7747         .set_cr0 = vmx_set_cr0,
7748         .is_valid_cr4 = vmx_is_valid_cr4,
7749         .set_cr4 = vmx_set_cr4,
7750         .set_efer = vmx_set_efer,
7751         .get_idt = vmx_get_idt,
7752         .set_idt = vmx_set_idt,
7753         .get_gdt = vmx_get_gdt,
7754         .set_gdt = vmx_set_gdt,
7755         .set_dr7 = vmx_set_dr7,
7756         .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
7757         .cache_reg = vmx_cache_reg,
7758         .get_rflags = vmx_get_rflags,
7759         .set_rflags = vmx_set_rflags,
7760         .get_if_flag = vmx_get_if_flag,
7761
7762         .flush_tlb_all = vmx_flush_tlb_all,
7763         .flush_tlb_current = vmx_flush_tlb_current,
7764         .flush_tlb_gva = vmx_flush_tlb_gva,
7765         .flush_tlb_guest = vmx_flush_tlb_guest,
7766
7767         .vcpu_pre_run = vmx_vcpu_pre_run,
7768         .vcpu_run = vmx_vcpu_run,
7769         .handle_exit = vmx_handle_exit,
7770         .skip_emulated_instruction = vmx_skip_emulated_instruction,
7771         .update_emulated_instruction = vmx_update_emulated_instruction,
7772         .set_interrupt_shadow = vmx_set_interrupt_shadow,
7773         .get_interrupt_shadow = vmx_get_interrupt_shadow,
7774         .patch_hypercall = vmx_patch_hypercall,
7775         .inject_irq = vmx_inject_irq,
7776         .inject_nmi = vmx_inject_nmi,
7777         .queue_exception = vmx_queue_exception,
7778         .cancel_injection = vmx_cancel_injection,
7779         .interrupt_allowed = vmx_interrupt_allowed,
7780         .nmi_allowed = vmx_nmi_allowed,
7781         .get_nmi_mask = vmx_get_nmi_mask,
7782         .set_nmi_mask = vmx_set_nmi_mask,
7783         .enable_nmi_window = vmx_enable_nmi_window,
7784         .enable_irq_window = vmx_enable_irq_window,
7785         .update_cr8_intercept = vmx_update_cr8_intercept,
7786         .set_virtual_apic_mode = vmx_set_virtual_apic_mode,
7787         .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
7788         .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
7789         .load_eoi_exitmap = vmx_load_eoi_exitmap,
7790         .apicv_post_state_restore = vmx_apicv_post_state_restore,
7791         .check_apicv_inhibit_reasons = vmx_check_apicv_inhibit_reasons,
7792         .hwapic_irr_update = vmx_hwapic_irr_update,
7793         .hwapic_isr_update = vmx_hwapic_isr_update,
7794         .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt,
7795         .sync_pir_to_irr = vmx_sync_pir_to_irr,
7796         .deliver_interrupt = vmx_deliver_interrupt,
7797         .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt,
7798
7799         .set_tss_addr = vmx_set_tss_addr,
7800         .set_identity_map_addr = vmx_set_identity_map_addr,
7801         .get_mt_mask = vmx_get_mt_mask,
7802
7803         .get_exit_info = vmx_get_exit_info,
7804
7805         .vcpu_after_set_cpuid = vmx_vcpu_after_set_cpuid,
7806
7807         .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
7808
7809         .get_l2_tsc_offset = vmx_get_l2_tsc_offset,
7810         .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier,
7811         .write_tsc_offset = vmx_write_tsc_offset,
7812         .write_tsc_multiplier = vmx_write_tsc_multiplier,
7813
7814         .load_mmu_pgd = vmx_load_mmu_pgd,
7815
7816         .check_intercept = vmx_check_intercept,
7817         .handle_exit_irqoff = vmx_handle_exit_irqoff,
7818
7819         .request_immediate_exit = vmx_request_immediate_exit,
7820
7821         .sched_in = vmx_sched_in,
7822
7823         .cpu_dirty_log_size = PML_ENTITY_NUM,
7824         .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging,
7825
7826         .pmu_ops = &intel_pmu_ops,
7827         .nested_ops = &vmx_nested_ops,
7828
7829         .pi_update_irte = vmx_pi_update_irte,
7830         .pi_start_assignment = vmx_pi_start_assignment,
7831
7832 #ifdef CONFIG_X86_64
7833         .set_hv_timer = vmx_set_hv_timer,
7834         .cancel_hv_timer = vmx_cancel_hv_timer,
7835 #endif
7836
7837         .setup_mce = vmx_setup_mce,
7838
7839         .smi_allowed = vmx_smi_allowed,
7840         .enter_smm = vmx_enter_smm,
7841         .leave_smm = vmx_leave_smm,
7842         .enable_smi_window = vmx_enable_smi_window,
7843
7844         .can_emulate_instruction = vmx_can_emulate_instruction,
7845         .apic_init_signal_blocked = vmx_apic_init_signal_blocked,
7846         .migrate_timers = vmx_migrate_timers,
7847
7848         .msr_filter_changed = vmx_msr_filter_changed,
7849         .complete_emulated_msr = kvm_complete_insn_gp,
7850
7851         .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector,
7852 };
7853
7854 static unsigned int vmx_handle_intel_pt_intr(void)
7855 {
7856         struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
7857
7858         /* '0' on failure so that the !PT case can use a RET0 static call. */
7859         if (!kvm_arch_pmi_in_guest(vcpu))
7860                 return 0;
7861
7862         kvm_make_request(KVM_REQ_PMI, vcpu);
7863         __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
7864                   (unsigned long *)&vcpu->arch.pmu.global_status);
7865         return 1;
7866 }
7867
7868 static __init void vmx_setup_user_return_msrs(void)
7869 {
7870
7871         /*
7872          * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm
7873          * will emulate SYSCALL in legacy mode if the vendor string in guest
7874          * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To
7875          * support this emulation, MSR_STAR is included in the list for i386,
7876          * but is never loaded into hardware.  MSR_CSTAR is also never loaded
7877          * into hardware and is here purely for emulation purposes.
7878          */
7879         const u32 vmx_uret_msrs_list[] = {
7880         #ifdef CONFIG_X86_64
7881                 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
7882         #endif
7883                 MSR_EFER, MSR_TSC_AUX, MSR_STAR,
7884                 MSR_IA32_TSX_CTRL,
7885         };
7886         int i;
7887
7888         BUILD_BUG_ON(ARRAY_SIZE(vmx_uret_msrs_list) != MAX_NR_USER_RETURN_MSRS);
7889
7890         for (i = 0; i < ARRAY_SIZE(vmx_uret_msrs_list); ++i)
7891                 kvm_add_user_return_msr(vmx_uret_msrs_list[i]);
7892 }
7893
7894 static struct kvm_x86_init_ops vmx_init_ops __initdata;
7895
7896 static __init int hardware_setup(void)
7897 {
7898         unsigned long host_bndcfgs;
7899         struct desc_ptr dt;
7900         int r;
7901
7902         store_idt(&dt);
7903         host_idt_base = dt.address;
7904
7905         vmx_setup_user_return_msrs();
7906
7907         if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0)
7908                 return -EIO;
7909
7910         if (boot_cpu_has(X86_FEATURE_NX))
7911                 kvm_enable_efer_bits(EFER_NX);
7912
7913         if (boot_cpu_has(X86_FEATURE_MPX)) {
7914                 rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs);
7915                 WARN_ONCE(host_bndcfgs, "KVM: BNDCFGS in host will be lost");
7916         }
7917
7918         if (!cpu_has_vmx_mpx())
7919                 supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS |
7920                                     XFEATURE_MASK_BNDCSR);
7921
7922         if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() ||
7923             !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global()))
7924                 enable_vpid = 0;
7925
7926         if (!cpu_has_vmx_ept() ||
7927             !cpu_has_vmx_ept_4levels() ||
7928             !cpu_has_vmx_ept_mt_wb() ||
7929             !cpu_has_vmx_invept_global())
7930                 enable_ept = 0;
7931
7932         /* NX support is required for shadow paging. */
7933         if (!enable_ept && !boot_cpu_has(X86_FEATURE_NX)) {
7934                 pr_err_ratelimited("kvm: NX (Execute Disable) not supported\n");
7935                 return -EOPNOTSUPP;
7936         }
7937
7938         if (!cpu_has_vmx_ept_ad_bits() || !enable_ept)
7939                 enable_ept_ad_bits = 0;
7940
7941         if (!cpu_has_vmx_unrestricted_guest() || !enable_ept)
7942                 enable_unrestricted_guest = 0;
7943
7944         if (!cpu_has_vmx_flexpriority())
7945                 flexpriority_enabled = 0;
7946
7947         if (!cpu_has_virtual_nmis())
7948                 enable_vnmi = 0;
7949
7950         /*
7951          * set_apic_access_page_addr() is used to reload apic access
7952          * page upon invalidation.  No need to do anything if not
7953          * using the APIC_ACCESS_ADDR VMCS field.
7954          */
7955         if (!flexpriority_enabled)
7956                 vmx_x86_ops.set_apic_access_page_addr = NULL;
7957
7958         if (!cpu_has_vmx_tpr_shadow())
7959                 vmx_x86_ops.update_cr8_intercept = NULL;
7960
7961 #if IS_ENABLED(CONFIG_HYPERV)
7962         if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH
7963             && enable_ept) {
7964                 vmx_x86_ops.tlb_remote_flush = hv_remote_flush_tlb;
7965                 vmx_x86_ops.tlb_remote_flush_with_range =
7966                                 hv_remote_flush_tlb_with_range;
7967         }
7968 #endif
7969
7970         if (!cpu_has_vmx_ple()) {
7971                 ple_gap = 0;
7972                 ple_window = 0;
7973                 ple_window_grow = 0;
7974                 ple_window_max = 0;
7975                 ple_window_shrink = 0;
7976         }
7977
7978         if (!cpu_has_vmx_apicv())
7979                 enable_apicv = 0;
7980         if (!enable_apicv)
7981                 vmx_x86_ops.sync_pir_to_irr = NULL;
7982
7983         if (cpu_has_vmx_tsc_scaling())
7984                 kvm_has_tsc_control = true;
7985
7986         kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX;
7987         kvm_tsc_scaling_ratio_frac_bits = 48;
7988         kvm_has_bus_lock_exit = cpu_has_vmx_bus_lock_detection();
7989
7990         set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
7991
7992         if (enable_ept)
7993                 kvm_mmu_set_ept_masks(enable_ept_ad_bits,
7994                                       cpu_has_vmx_ept_execute_only());
7995
7996         kvm_configure_mmu(enable_ept, 0, vmx_get_max_tdp_level(),
7997                           ept_caps_to_lpage_level(vmx_capability.ept));
7998
7999         /*
8000          * Only enable PML when hardware supports PML feature, and both EPT
8001          * and EPT A/D bit features are enabled -- PML depends on them to work.
8002          */
8003         if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml())
8004                 enable_pml = 0;
8005
8006         if (!enable_pml)
8007                 vmx_x86_ops.cpu_dirty_log_size = 0;
8008
8009         if (!cpu_has_vmx_preemption_timer())
8010                 enable_preemption_timer = false;
8011
8012         if (enable_preemption_timer) {
8013                 u64 use_timer_freq = 5000ULL * 1000 * 1000;
8014                 u64 vmx_msr;
8015
8016                 rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
8017                 cpu_preemption_timer_multi =
8018                         vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK;
8019
8020                 if (tsc_khz)
8021                         use_timer_freq = (u64)tsc_khz * 1000;
8022                 use_timer_freq >>= cpu_preemption_timer_multi;
8023
8024                 /*
8025                  * KVM "disables" the preemption timer by setting it to its max
8026                  * value.  Don't use the timer if it might cause spurious exits
8027                  * at a rate faster than 0.1 Hz (of uninterrupted guest time).
8028                  */
8029                 if (use_timer_freq > 0xffffffffu / 10)
8030                         enable_preemption_timer = false;
8031         }
8032
8033         if (!enable_preemption_timer) {
8034                 vmx_x86_ops.set_hv_timer = NULL;
8035                 vmx_x86_ops.cancel_hv_timer = NULL;
8036                 vmx_x86_ops.request_immediate_exit = __kvm_request_immediate_exit;
8037         }
8038
8039         kvm_mce_cap_supported |= MCG_LMCE_P;
8040
8041         if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST)
8042                 return -EINVAL;
8043         if (!enable_ept || !cpu_has_vmx_intel_pt())
8044                 pt_mode = PT_MODE_SYSTEM;
8045         if (pt_mode == PT_MODE_HOST_GUEST)
8046                 vmx_init_ops.handle_intel_pt_intr = vmx_handle_intel_pt_intr;
8047         else
8048                 vmx_init_ops.handle_intel_pt_intr = NULL;
8049
8050         setup_default_sgx_lepubkeyhash();
8051
8052         if (nested) {
8053                 nested_vmx_setup_ctls_msrs(&vmcs_config.nested,
8054                                            vmx_capability.ept);
8055
8056                 r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers);
8057                 if (r)
8058                         return r;
8059         }
8060
8061         vmx_set_cpu_caps();
8062
8063         r = alloc_kvm_area();
8064         if (r && nested)
8065                 nested_vmx_hardware_unsetup();
8066
8067         kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler);
8068
8069         return r;
8070 }
8071
8072 static struct kvm_x86_init_ops vmx_init_ops __initdata = {
8073         .cpu_has_kvm_support = cpu_has_kvm_support,
8074         .disabled_by_bios = vmx_disabled_by_bios,
8075         .check_processor_compatibility = vmx_check_processor_compat,
8076         .hardware_setup = hardware_setup,
8077         .handle_intel_pt_intr = NULL,
8078
8079         .runtime_ops = &vmx_x86_ops,
8080 };
8081
8082 static void vmx_cleanup_l1d_flush(void)
8083 {
8084         if (vmx_l1d_flush_pages) {
8085                 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER);
8086                 vmx_l1d_flush_pages = NULL;
8087         }
8088         /* Restore state so sysfs ignores VMX */
8089         l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
8090 }
8091
8092 static void vmx_exit(void)
8093 {
8094 #ifdef CONFIG_KEXEC_CORE
8095         RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
8096         synchronize_rcu();
8097 #endif
8098
8099         kvm_exit();
8100
8101 #if IS_ENABLED(CONFIG_HYPERV)
8102         if (static_branch_unlikely(&enable_evmcs)) {
8103                 int cpu;
8104                 struct hv_vp_assist_page *vp_ap;
8105                 /*
8106                  * Reset everything to support using non-enlightened VMCS
8107                  * access later (e.g. when we reload the module with
8108                  * enlightened_vmcs=0)
8109                  */
8110                 for_each_online_cpu(cpu) {
8111                         vp_ap = hv_get_vp_assist_page(cpu);
8112
8113                         if (!vp_ap)
8114                                 continue;
8115
8116                         vp_ap->nested_control.features.directhypercall = 0;
8117                         vp_ap->current_nested_vmcs = 0;
8118                         vp_ap->enlighten_vmentry = 0;
8119                 }
8120
8121                 static_branch_disable(&enable_evmcs);
8122         }
8123 #endif
8124         vmx_cleanup_l1d_flush();
8125
8126         allow_smaller_maxphyaddr = false;
8127 }
8128 module_exit(vmx_exit);
8129
8130 static int __init vmx_init(void)
8131 {
8132         int r, cpu;
8133
8134 #if IS_ENABLED(CONFIG_HYPERV)
8135         /*
8136          * Enlightened VMCS usage should be recommended and the host needs
8137          * to support eVMCS v1 or above. We can also disable eVMCS support
8138          * with module parameter.
8139          */
8140         if (enlightened_vmcs &&
8141             ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED &&
8142             (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >=
8143             KVM_EVMCS_VERSION) {
8144
8145                 /* Check that we have assist pages on all online CPUs */
8146                 for_each_online_cpu(cpu) {
8147                         if (!hv_get_vp_assist_page(cpu)) {
8148                                 enlightened_vmcs = false;
8149                                 break;
8150                         }
8151                 }
8152
8153                 if (enlightened_vmcs) {
8154                         pr_info("KVM: vmx: using Hyper-V Enlightened VMCS\n");
8155                         static_branch_enable(&enable_evmcs);
8156                 }
8157
8158                 if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH)
8159                         vmx_x86_ops.enable_direct_tlbflush
8160                                 = hv_enable_direct_tlbflush;
8161
8162         } else {
8163                 enlightened_vmcs = false;
8164         }
8165 #endif
8166
8167         r = kvm_init(&vmx_init_ops, sizeof(struct vcpu_vmx),
8168                      __alignof__(struct vcpu_vmx), THIS_MODULE);
8169         if (r)
8170                 return r;
8171
8172         /*
8173          * Must be called after kvm_init() so enable_ept is properly set
8174          * up. Hand the parameter mitigation value in which was stored in
8175          * the pre module init parser. If no parameter was given, it will
8176          * contain 'auto' which will be turned into the default 'cond'
8177          * mitigation mode.
8178          */
8179         r = vmx_setup_l1d_flush(vmentry_l1d_flush_param);
8180         if (r) {
8181                 vmx_exit();
8182                 return r;
8183         }
8184
8185         for_each_possible_cpu(cpu) {
8186                 INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
8187
8188                 pi_init_cpu(cpu);
8189         }
8190
8191 #ifdef CONFIG_KEXEC_CORE
8192         rcu_assign_pointer(crash_vmclear_loaded_vmcss,
8193                            crash_vmclear_local_loaded_vmcss);
8194 #endif
8195         vmx_check_vmcs12_offsets();
8196
8197         /*
8198          * Shadow paging doesn't have a (further) performance penalty
8199          * from GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable it
8200          * by default
8201          */
8202         if (!enable_ept)
8203                 allow_smaller_maxphyaddr = true;
8204
8205         return 0;
8206 }
8207 module_init(vmx_init);