arch/x86/kvm/vmx/hyperv.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #define pr_fmt(fmt) "kvm/hyper-v: " fmt
   4
   5 #include <linux/errno.h>
   6 #include <linux/smp.h>
   7
   8 #include "../cpuid.h"
   9 #include "hyperv.h"
  10 #include "nested.h"
  11 #include "vmcs.h"
  12 #include "vmx.h"
  13 #include "trace.h"
  14
  15 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
  16
  17 DEFINE_STATIC_KEY_FALSE(enable_evmcs);
  18
  19 #define EVMCS1_OFFSET(x) offsetof(struct hv_enlightened_vmcs, x)
  20 #define EVMCS1_FIELD(number, name, clean_field)[ROL16(number, 6)] = \
  21                 {EVMCS1_OFFSET(name), clean_field}
  22
  23 const struct evmcs_field vmcs_field_to_evmcs_1[] = {
  24         /* 64 bit rw */
  25         EVMCS1_FIELD(GUEST_RIP, guest_rip,
  26                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
  27         EVMCS1_FIELD(GUEST_RSP, guest_rsp,
  28                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC),
  29         EVMCS1_FIELD(GUEST_RFLAGS, guest_rflags,
  30                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC),
  31         EVMCS1_FIELD(HOST_IA32_PAT, host_ia32_pat,
  32                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
  33         EVMCS1_FIELD(HOST_IA32_EFER, host_ia32_efer,
  34                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
  35         EVMCS1_FIELD(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl,
  36                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
  37         EVMCS1_FIELD(HOST_CR0, host_cr0,
  38                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
  39         EVMCS1_FIELD(HOST_CR3, host_cr3,
  40                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
  41         EVMCS1_FIELD(HOST_CR4, host_cr4,
  42                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
  43         EVMCS1_FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp,
  44                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
  45         EVMCS1_FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip,
  46                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
  47         EVMCS1_FIELD(HOST_RIP, host_rip,
  48                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
  49         EVMCS1_FIELD(IO_BITMAP_A, io_bitmap_a,
  50                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP),
  51         EVMCS1_FIELD(IO_BITMAP_B, io_bitmap_b,
  52                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP),
  53         EVMCS1_FIELD(MSR_BITMAP, msr_bitmap,
  54                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP),
  55         EVMCS1_FIELD(GUEST_ES_BASE, guest_es_base,
  56                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
  57         EVMCS1_FIELD(GUEST_CS_BASE, guest_cs_base,
  58                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
  59         EVMCS1_FIELD(GUEST_SS_BASE, guest_ss_base,
  60                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
  61         EVMCS1_FIELD(GUEST_DS_BASE, guest_ds_base,
  62                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
  63         EVMCS1_FIELD(GUEST_FS_BASE, guest_fs_base,
  64                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
  65         EVMCS1_FIELD(GUEST_GS_BASE, guest_gs_base,
  66                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
  67         EVMCS1_FIELD(GUEST_LDTR_BASE, guest_ldtr_base,
  68                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
  69         EVMCS1_FIELD(GUEST_TR_BASE, guest_tr_base,
  70                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
  71         EVMCS1_FIELD(GUEST_GDTR_BASE, guest_gdtr_base,
  72                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
  73         EVMCS1_FIELD(GUEST_IDTR_BASE, guest_idtr_base,
  74                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
  75         EVMCS1_FIELD(TSC_OFFSET, tsc_offset,
  76                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2),
  77         EVMCS1_FIELD(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr,
  78                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2),
  79         EVMCS1_FIELD(VMCS_LINK_POINTER, vmcs_link_pointer,
  80                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
  81         EVMCS1_FIELD(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl,
  82                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
  83         EVMCS1_FIELD(GUEST_IA32_PAT, guest_ia32_pat,
  84                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
  85         EVMCS1_FIELD(GUEST_IA32_EFER, guest_ia32_efer,
  86                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
  87         EVMCS1_FIELD(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl,
  88                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
  89         EVMCS1_FIELD(GUEST_PDPTR0, guest_pdptr0,
  90                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
  91         EVMCS1_FIELD(GUEST_PDPTR1, guest_pdptr1,
  92                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
  93         EVMCS1_FIELD(GUEST_PDPTR2, guest_pdptr2,
  94                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
  95         EVMCS1_FIELD(GUEST_PDPTR3, guest_pdptr3,
  96                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
  97         EVMCS1_FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions,
  98                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
  99         EVMCS1_FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp,
 100                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 101         EVMCS1_FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip,
 102                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 103         EVMCS1_FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask,
 104                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 105         EVMCS1_FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask,
 106                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 107         EVMCS1_FIELD(CR0_READ_SHADOW, cr0_read_shadow,
 108                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 109         EVMCS1_FIELD(CR4_READ_SHADOW, cr4_read_shadow,
 110                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 111         EVMCS1_FIELD(GUEST_CR0, guest_cr0,
 112                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 113         EVMCS1_FIELD(GUEST_CR3, guest_cr3,
 114                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 115         EVMCS1_FIELD(GUEST_CR4, guest_cr4,
 116                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 117         EVMCS1_FIELD(GUEST_DR7, guest_dr7,
 118                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 119         EVMCS1_FIELD(HOST_FS_BASE, host_fs_base,
 120                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 121         EVMCS1_FIELD(HOST_GS_BASE, host_gs_base,
 122                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 123         EVMCS1_FIELD(HOST_TR_BASE, host_tr_base,
 124                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 125         EVMCS1_FIELD(HOST_GDTR_BASE, host_gdtr_base,
 126                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 127         EVMCS1_FIELD(HOST_IDTR_BASE, host_idtr_base,
 128                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 129         EVMCS1_FIELD(HOST_RSP, host_rsp,
 130                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 131         EVMCS1_FIELD(EPT_POINTER, ept_pointer,
 132                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT),
 133         EVMCS1_FIELD(GUEST_BNDCFGS, guest_bndcfgs,
 134                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 135         EVMCS1_FIELD(XSS_EXIT_BITMAP, xss_exit_bitmap,
 136                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2),
 137         EVMCS1_FIELD(ENCLS_EXITING_BITMAP, encls_exiting_bitmap,
 138                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2),
 139         EVMCS1_FIELD(TSC_MULTIPLIER, tsc_multiplier,
 140                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2),
 141         /*
 142          * Not used by KVM:
 143          *
 144          * EVMCS1_FIELD(0x00006828, guest_ia32_s_cet,
 145          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 146          * EVMCS1_FIELD(0x0000682A, guest_ssp,
 147          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC),
 148          * EVMCS1_FIELD(0x0000682C, guest_ia32_int_ssp_table_addr,
 149          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 150          * EVMCS1_FIELD(0x00002816, guest_ia32_lbr_ctl,
 151          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 152          * EVMCS1_FIELD(0x00006C18, host_ia32_s_cet,
 153          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 154          * EVMCS1_FIELD(0x00006C1A, host_ssp,
 155          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 156          * EVMCS1_FIELD(0x00006C1C, host_ia32_int_ssp_table_addr,
 157          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 158          */
 159
 160         /* 64 bit read only */
 161         EVMCS1_FIELD(GUEST_PHYSICAL_ADDRESS, guest_physical_address,
 162                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 163         EVMCS1_FIELD(EXIT_QUALIFICATION, exit_qualification,
 164                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 165         /*
 166          * Not defined in KVM:
 167          *
 168          * EVMCS1_FIELD(0x00006402, exit_io_instruction_ecx,
 169          *              HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE);
 170          * EVMCS1_FIELD(0x00006404, exit_io_instruction_esi,
 171          *              HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE);
 172          * EVMCS1_FIELD(0x00006406, exit_io_instruction_esi,
 173          *              HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE);
 174          * EVMCS1_FIELD(0x00006408, exit_io_instruction_eip,
 175          *              HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE);
 176          */
 177         EVMCS1_FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address,
 178                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 179
 180         /*
 181          * No mask defined in the spec as Hyper-V doesn't currently support
 182          * these. Future proof by resetting the whole clean field mask on
 183          * access.
 184          */
 185         EVMCS1_FIELD(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr,
 186                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 187         EVMCS1_FIELD(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr,
 188                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 189         EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr,
 190                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 191
 192         /* 32 bit rw */
 193         EVMCS1_FIELD(TPR_THRESHOLD, tpr_threshold,
 194                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 195         EVMCS1_FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info,
 196                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC),
 197         EVMCS1_FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control,
 198                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC),
 199         EVMCS1_FIELD(EXCEPTION_BITMAP, exception_bitmap,
 200                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN),
 201         EVMCS1_FIELD(VM_ENTRY_CONTROLS, vm_entry_controls,
 202                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY),
 203         EVMCS1_FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field,
 204                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT),
 205         EVMCS1_FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE,
 206                      vm_entry_exception_error_code,
 207                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT),
 208         EVMCS1_FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len,
 209                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT),
 210         EVMCS1_FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs,
 211                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 212         EVMCS1_FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control,
 213                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1),
 214         EVMCS1_FIELD(VM_EXIT_CONTROLS, vm_exit_controls,
 215                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1),
 216         EVMCS1_FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control,
 217                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1),
 218         EVMCS1_FIELD(GUEST_ES_LIMIT, guest_es_limit,
 219                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 220         EVMCS1_FIELD(GUEST_CS_LIMIT, guest_cs_limit,
 221                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 222         EVMCS1_FIELD(GUEST_SS_LIMIT, guest_ss_limit,
 223                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 224         EVMCS1_FIELD(GUEST_DS_LIMIT, guest_ds_limit,
 225                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 226         EVMCS1_FIELD(GUEST_FS_LIMIT, guest_fs_limit,
 227                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 228         EVMCS1_FIELD(GUEST_GS_LIMIT, guest_gs_limit,
 229                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 230         EVMCS1_FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit,
 231                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 232         EVMCS1_FIELD(GUEST_TR_LIMIT, guest_tr_limit,
 233                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 234         EVMCS1_FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit,
 235                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 236         EVMCS1_FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit,
 237                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 238         EVMCS1_FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes,
 239                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 240         EVMCS1_FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes,
 241                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 242         EVMCS1_FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes,
 243                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 244         EVMCS1_FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes,
 245                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 246         EVMCS1_FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes,
 247                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 248         EVMCS1_FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes,
 249                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 250         EVMCS1_FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes,
 251                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 252         EVMCS1_FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes,
 253                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 254         EVMCS1_FIELD(GUEST_ACTIVITY_STATE, guest_activity_state,
 255                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 256         EVMCS1_FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs,
 257                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 258
 259         /* 32 bit read only */
 260         EVMCS1_FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error,
 261                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 262         EVMCS1_FIELD(VM_EXIT_REASON, vm_exit_reason,
 263                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 264         EVMCS1_FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info,
 265                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 266         EVMCS1_FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code,
 267                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 268         EVMCS1_FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field,
 269                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 270         EVMCS1_FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code,
 271                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 272         EVMCS1_FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len,
 273                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 274         EVMCS1_FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info,
 275                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 276
 277         /* No mask defined in the spec (not used) */
 278         EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask,
 279                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 280         EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match,
 281                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 282         EVMCS1_FIELD(CR3_TARGET_COUNT, cr3_target_count,
 283                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 284         EVMCS1_FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count,
 285                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 286         EVMCS1_FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count,
 287                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 288         EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count,
 289                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 290
 291         /* 16 bit rw */
 292         EVMCS1_FIELD(HOST_ES_SELECTOR, host_es_selector,
 293                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 294         EVMCS1_FIELD(HOST_CS_SELECTOR, host_cs_selector,
 295                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 296         EVMCS1_FIELD(HOST_SS_SELECTOR, host_ss_selector,
 297                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 298         EVMCS1_FIELD(HOST_DS_SELECTOR, host_ds_selector,
 299                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 300         EVMCS1_FIELD(HOST_FS_SELECTOR, host_fs_selector,
 301                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 302         EVMCS1_FIELD(HOST_GS_SELECTOR, host_gs_selector,
 303                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 304         EVMCS1_FIELD(HOST_TR_SELECTOR, host_tr_selector,
 305                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 306         EVMCS1_FIELD(GUEST_ES_SELECTOR, guest_es_selector,
 307                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 308         EVMCS1_FIELD(GUEST_CS_SELECTOR, guest_cs_selector,
 309                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 310         EVMCS1_FIELD(GUEST_SS_SELECTOR, guest_ss_selector,
 311                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 312         EVMCS1_FIELD(GUEST_DS_SELECTOR, guest_ds_selector,
 313                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 314         EVMCS1_FIELD(GUEST_FS_SELECTOR, guest_fs_selector,
 315                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 316         EVMCS1_FIELD(GUEST_GS_SELECTOR, guest_gs_selector,
 317                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 318         EVMCS1_FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector,
 319                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 320         EVMCS1_FIELD(GUEST_TR_SELECTOR, guest_tr_selector,
 321                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 322         EVMCS1_FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id,
 323                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT),
 324 };
 325 const unsigned int nr_evmcs_1_fields = ARRAY_SIZE(vmcs_field_to_evmcs_1);
 326
 327 u64 nested_get_evmptr(struct kvm_vcpu *vcpu)
 328 {
 329         struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
 330
 331         if (unlikely(kvm_hv_get_assist_page(vcpu)))
 332                 return EVMPTR_INVALID;
 333
 334         if (unlikely(!hv_vcpu->vp_assist_page.enlighten_vmentry))
 335                 return EVMPTR_INVALID;
 336
 337         return hv_vcpu->vp_assist_page.current_nested_vmcs;
 338 }
 339
 340 uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu)
 341 {
 342         /*
 343          * vmcs_version represents the range of supported Enlightened VMCS
 344          * versions: lower 8 bits is the minimal version, higher 8 bits is the
 345          * maximum supported version. KVM supports versions from 1 to
 346          * KVM_EVMCS_VERSION.
 347          *
 348          * Note, do not check the Hyper-V is fully enabled in guest CPUID, this
 349          * helper is used to _get_ the vCPU's supported CPUID.
 350          */
 351         if (kvm_cpu_cap_get(X86_FEATURE_VMX) &&
 352             (!vcpu || to_vmx(vcpu)->nested.enlightened_vmcs_enabled))
 353                 return (KVM_EVMCS_VERSION << 8) | 1;
 354
 355         return 0;
 356 }
 357
 358 enum evmcs_revision {
 359         EVMCSv1_LEGACY,
 360         NR_EVMCS_REVISIONS,
 361 };
 362
 363 enum evmcs_ctrl_type {
 364         EVMCS_EXIT_CTRLS,
 365         EVMCS_ENTRY_CTRLS,
 366         EVMCS_EXEC_CTRL,
 367         EVMCS_2NDEXEC,
 368         EVMCS_3RDEXEC,
 369         EVMCS_PINCTRL,
 370         EVMCS_VMFUNC,
 371         NR_EVMCS_CTRLS,
 372 };
 373
 374 static const u32 evmcs_supported_ctrls[NR_EVMCS_CTRLS][NR_EVMCS_REVISIONS] = {
 375         [EVMCS_EXIT_CTRLS] = {
 376                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMEXIT_CTRL,
 377         },
 378         [EVMCS_ENTRY_CTRLS] = {
 379                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMENTRY_CTRL,
 380         },
 381         [EVMCS_EXEC_CTRL] = {
 382                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_EXEC_CTRL,
 383         },
 384         [EVMCS_2NDEXEC] = {
 385                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_2NDEXEC & ~SECONDARY_EXEC_TSC_SCALING,
 386         },
 387         [EVMCS_3RDEXEC] = {
 388                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_3RDEXEC,
 389         },
 390         [EVMCS_PINCTRL] = {
 391                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_PINCTRL,
 392         },
 393         [EVMCS_VMFUNC] = {
 394                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMFUNC,
 395         },
 396 };
 397
 398 static u32 evmcs_get_supported_ctls(enum evmcs_ctrl_type ctrl_type)
 399 {
 400         enum evmcs_revision evmcs_rev = EVMCSv1_LEGACY;
 401
 402         return evmcs_supported_ctrls[ctrl_type][evmcs_rev];
 403 }
 404
 405 static bool evmcs_has_perf_global_ctrl(struct kvm_vcpu *vcpu)
 406 {
 407         struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
 408
 409         /*
 410          * PERF_GLOBAL_CTRL has a quirk where some Windows guests may fail to
 411          * boot if a PV CPUID feature flag is not also set.  Treat the fields
 412          * as unsupported if the flag is not set in guest CPUID.  This should
 413          * be called only for guest accesses, and all guest accesses should be
 414          * gated on Hyper-V being enabled and initialized.
 415          */
 416         if (WARN_ON_ONCE(!hv_vcpu))
 417                 return false;
 418
 419         return hv_vcpu->cpuid_cache.nested_ebx & HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL;
 420 }
 421
 422 void nested_evmcs_filter_control_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
 423 {
 424         u32 ctl_low = (u32)*pdata;
 425         u32 ctl_high = (u32)(*pdata >> 32);
 426         u32 supported_ctrls;
 427
 428         /*
 429          * Hyper-V 2016 and 2019 try using these features even when eVMCS
 430          * is enabled but there are no corresponding fields.
 431          */
 432         switch (msr_index) {
 433         case MSR_IA32_VMX_EXIT_CTLS:
 434         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
 435                 supported_ctrls = evmcs_get_supported_ctls(EVMCS_EXIT_CTRLS);
 436                 if (!evmcs_has_perf_global_ctrl(vcpu))
 437                         supported_ctrls &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
 438                 ctl_high &= supported_ctrls;
 439                 break;
 440         case MSR_IA32_VMX_ENTRY_CTLS:
 441         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
 442                 supported_ctrls = evmcs_get_supported_ctls(EVMCS_ENTRY_CTRLS);
 443                 if (!evmcs_has_perf_global_ctrl(vcpu))
 444                         supported_ctrls &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
 445                 ctl_high &= supported_ctrls;
 446                 break;
 447         case MSR_IA32_VMX_PROCBASED_CTLS:
 448         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
 449                 ctl_high &= evmcs_get_supported_ctls(EVMCS_EXEC_CTRL);
 450                 break;
 451         case MSR_IA32_VMX_PROCBASED_CTLS2:
 452                 ctl_high &= evmcs_get_supported_ctls(EVMCS_2NDEXEC);
 453                 break;
 454         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
 455         case MSR_IA32_VMX_PINBASED_CTLS:
 456                 ctl_high &= evmcs_get_supported_ctls(EVMCS_PINCTRL);
 457                 break;
 458         case MSR_IA32_VMX_VMFUNC:
 459                 ctl_low &= evmcs_get_supported_ctls(EVMCS_VMFUNC);
 460                 break;
 461         }
 462
 463         *pdata = ctl_low | ((u64)ctl_high << 32);
 464 }
 465
 466 static bool nested_evmcs_is_valid_controls(enum evmcs_ctrl_type ctrl_type,
 467                                            u32 val)
 468 {
 469         return !(val & ~evmcs_get_supported_ctls(ctrl_type));
 470 }
 471
 472 int nested_evmcs_check_controls(struct vmcs12 *vmcs12)
 473 {
 474         if (CC(!nested_evmcs_is_valid_controls(EVMCS_PINCTRL,
 475                                                vmcs12->pin_based_vm_exec_control)))
 476                 return -EINVAL;
 477
 478         if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXEC_CTRL,
 479                                                vmcs12->cpu_based_vm_exec_control)))
 480                 return -EINVAL;
 481
 482         if (CC(!nested_evmcs_is_valid_controls(EVMCS_2NDEXEC,
 483                                                vmcs12->secondary_vm_exec_control)))
 484                 return -EINVAL;
 485
 486         if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXIT_CTRLS,
 487                                                vmcs12->vm_exit_controls)))
 488                 return -EINVAL;
 489
 490         if (CC(!nested_evmcs_is_valid_controls(EVMCS_ENTRY_CTRLS,
 491                                                vmcs12->vm_entry_controls)))
 492                 return -EINVAL;
 493
 494         /*
 495          * VM-Func controls are 64-bit, but KVM currently doesn't support any
 496          * controls in bits 63:32, i.e. dropping those bits on the consistency
 497          * check is intentional.
 498          */
 499         if (WARN_ON_ONCE(vmcs12->vm_function_control >> 32))
 500                 return -EINVAL;
 501
 502         if (CC(!nested_evmcs_is_valid_controls(EVMCS_VMFUNC,
 503                                                vmcs12->vm_function_control)))
 504                 return -EINVAL;
 505
 506         return 0;
 507 }
 508
 509 #if IS_ENABLED(CONFIG_HYPERV)
 510 /*
 511  * KVM on Hyper-V always uses the latest known eVMCSv1 revision, the assumption
 512  * is: in case a feature has corresponding fields in eVMCS described and it was
 513  * exposed in VMX feature MSRs, KVM is free to use it. Warn if KVM meets a
 514  * feature which has no corresponding eVMCS field, this likely means that KVM
 515  * needs to be updated.
 516  */
 517 #define evmcs_check_vmcs_conf(field, ctrl)                                      \
 518         do {                                                                    \
 519                 typeof(vmcs_conf->field) unsupported;                           \
 520                                                                                 \
 521                 unsupported = vmcs_conf->field & ~EVMCS1_SUPPORTED_ ## ctrl;    \
 522                 if (unsupported) {                                              \
 523                         pr_warn_once(#field " unsupported with eVMCS: 0x%llx\n",\
 524                                      (u64)unsupported);                         \
 525                         vmcs_conf->field &= EVMCS1_SUPPORTED_ ## ctrl;          \
 526                 }                                                               \
 527         }                                                                       \
 528         while (0)
 529
 530 __init void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf)
 531 {
 532         evmcs_check_vmcs_conf(cpu_based_exec_ctrl, EXEC_CTRL);
 533         evmcs_check_vmcs_conf(pin_based_exec_ctrl, PINCTRL);
 534         evmcs_check_vmcs_conf(cpu_based_2nd_exec_ctrl, 2NDEXEC);
 535         evmcs_check_vmcs_conf(cpu_based_3rd_exec_ctrl, 3RDEXEC);
 536         evmcs_check_vmcs_conf(vmentry_ctrl, VMENTRY_CTRL);
 537         evmcs_check_vmcs_conf(vmexit_ctrl, VMEXIT_CTRL);
 538 }
 539 #endif
 540
 541 int nested_enable_evmcs(struct kvm_vcpu *vcpu,
 542                         uint16_t *vmcs_version)
 543 {
 544         struct vcpu_vmx *vmx = to_vmx(vcpu);
 545
 546         vmx->nested.enlightened_vmcs_enabled = true;
 547
 548         if (vmcs_version)
 549                 *vmcs_version = nested_get_evmcs_version(vcpu);
 550
 551         return 0;
 552 }
 553
 554 bool nested_evmcs_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu)
 555 {
 556         struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
 557         struct vcpu_vmx *vmx = to_vmx(vcpu);
 558         struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
 559
 560         if (!hv_vcpu || !evmcs)
 561                 return false;
 562
 563         if (!evmcs->hv_enlightenments_control.nested_flush_hypercall)
 564                 return false;
 565
 566         return hv_vcpu->vp_assist_page.nested_control.features.directhypercall;
 567 }
 568
 569 void vmx_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu)
 570 {
 571         nested_vmx_vmexit(vcpu, HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH, 0, 0);
 572 }