KVM: TDX: Implement TDX vcpu enter/exit path

Implement callbacks to enter/exit a TDX VCPU by calling tdh_vp_enter().
Ensure the TDX VCPU is in a correct state to run.

Do not pass arguments from/to vcpu->arch.regs[] unconditionally. Instead,
marshall state to/from the appropriate x86 registers only when needed,
i.e., to handle some TDVMCALL sub-leaves following KVM's ABI to leverage
the existing code.

Signed-off-by: Isaku Yamahata <isaku.yamahata@intel.com>
Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
Message-ID: <20250129095902.16391-6-adrian.hunter@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index 36b6343..037590f 100644 (file)
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -145,6 +145,22 @@ static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
        vmx_update_cpu_dirty_logging(vcpu);
 }
 
+static int vt_vcpu_pre_run(struct kvm_vcpu *vcpu)
+{
+       if (is_td_vcpu(vcpu))
+               return tdx_vcpu_pre_run(vcpu);
+
+       return vmx_vcpu_pre_run(vcpu);
+}
+
+static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit)
+{
+       if (is_td_vcpu(vcpu))
+               return tdx_vcpu_run(vcpu, force_immediate_exit);
+
+       return vmx_vcpu_run(vcpu, force_immediate_exit);
+}
+
 static void vt_flush_tlb_all(struct kvm_vcpu *vcpu)
 {
        if (is_td_vcpu(vcpu)) {
@@ -285,8 +301,8 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
        .flush_tlb_gva = vt_flush_tlb_gva,
        .flush_tlb_guest = vt_flush_tlb_guest,
 
-       .vcpu_pre_run = vmx_vcpu_pre_run,
-       .vcpu_run = vmx_vcpu_run,
+       .vcpu_pre_run = vt_vcpu_pre_run,
+       .vcpu_run = vt_vcpu_run,
        .handle_exit = vmx_handle_exit,
        .skip_emulated_instruction = vmx_skip_emulated_instruction,
        .update_emulated_instruction = vmx_update_emulated_instruction,

diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index 277f5de..bd29960 100644 (file)
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -12,6 +12,8 @@
 #include "vmx.h"
 #include "mmu/spte.h"
 #include "common.h"
+#include <trace/events/kvm.h>
+#include "trace.h"
 
 #pragma GCC poison to_vmx
 
@@ -660,6 +662,66 @@ void tdx_vcpu_free(struct kvm_vcpu *vcpu)
        tdx->state = VCPU_TD_STATE_UNINITIALIZED;
 }
 
+int tdx_vcpu_pre_run(struct kvm_vcpu *vcpu)
+{
+       if (unlikely(to_tdx(vcpu)->state != VCPU_TD_STATE_INITIALIZED ||
+                    to_kvm_tdx(vcpu->kvm)->state != TD_STATE_RUNNABLE))
+               return -EINVAL;
+
+       return 1;
+}
+
+static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu)
+{
+       struct vcpu_tdx *tdx = to_tdx(vcpu);
+
+       guest_state_enter_irqoff();
+
+       tdx->vp_enter_ret = tdh_vp_enter(&tdx->vp, &tdx->vp_enter_args);
+
+       guest_state_exit_irqoff();
+}
+
+#define TDX_REGS_AVAIL_SET     (BIT_ULL(VCPU_EXREG_EXIT_INFO_1) | \
+                                BIT_ULL(VCPU_EXREG_EXIT_INFO_2) | \
+                                BIT_ULL(VCPU_REGS_RAX) | \
+                                BIT_ULL(VCPU_REGS_RBX) | \
+                                BIT_ULL(VCPU_REGS_RCX) | \
+                                BIT_ULL(VCPU_REGS_RDX) | \
+                                BIT_ULL(VCPU_REGS_RBP) | \
+                                BIT_ULL(VCPU_REGS_RSI) | \
+                                BIT_ULL(VCPU_REGS_RDI) | \
+                                BIT_ULL(VCPU_REGS_R8) | \
+                                BIT_ULL(VCPU_REGS_R9) | \
+                                BIT_ULL(VCPU_REGS_R10) | \
+                                BIT_ULL(VCPU_REGS_R11) | \
+                                BIT_ULL(VCPU_REGS_R12) | \
+                                BIT_ULL(VCPU_REGS_R13) | \
+                                BIT_ULL(VCPU_REGS_R14) | \
+                                BIT_ULL(VCPU_REGS_R15))
+
+fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit)
+{
+       /*
+        * force_immediate_exit requires vCPU entering for events injection with
+        * an immediately exit followed. But The TDX module doesn't guarantee
+        * entry, it's already possible for KVM to _think_ it completely entry
+        * to the guest without actually having done so.
+        * Since KVM never needs to force an immediate exit for TDX, and can't
+        * do direct injection, just warn on force_immediate_exit.
+        */
+       WARN_ON_ONCE(force_immediate_exit);
+
+       trace_kvm_entry(vcpu, force_immediate_exit);
+
+       tdx_vcpu_enter_exit(vcpu);
+
+       vcpu->arch.regs_avail &= TDX_REGS_AVAIL_SET;
+
+       trace_kvm_exit(vcpu, KVM_ISA_VMX);
+
+       return EXIT_FASTPATH_NONE;
+}
 
 void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level)
 {

diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h
index db6aa02..59a35e4 100644 (file)
--- a/arch/x86/kvm/vmx/tdx.h
+++ b/arch/x86/kvm/vmx/tdx.h
@@ -48,11 +48,14 @@ enum vcpu_tdx_state {
 struct vcpu_tdx {
        struct kvm_vcpu vcpu;
        struct vcpu_vt vt;
+       struct tdx_module_args vp_enter_args;
 
        struct tdx_vp vp;
 
        struct list_head cpu_list;
 
+       u64 vp_enter_ret;
+
        enum vcpu_tdx_state state;
 };
 

diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h
index f47d739..578c26d 100644 (file)
--- a/arch/x86/kvm/vmx/x86_ops.h
+++ b/arch/x86/kvm/vmx/x86_ops.h
@@ -131,6 +131,8 @@ int tdx_vm_ioctl(struct kvm *kvm, void __user *argp);
 int tdx_vcpu_create(struct kvm_vcpu *vcpu);
 void tdx_vcpu_free(struct kvm_vcpu *vcpu);
 void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
+int tdx_vcpu_pre_run(struct kvm_vcpu *vcpu);
+fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit);
 
 int tdx_vcpu_ioctl(struct kvm_vcpu *vcpu, void __user *argp);
 
@@ -157,6 +159,11 @@ static inline int tdx_vm_ioctl(struct kvm *kvm, void __user *argp) { return -EOP
 static inline int tdx_vcpu_create(struct kvm_vcpu *vcpu) { return -EOPNOTSUPP; }
 static inline void tdx_vcpu_free(struct kvm_vcpu *vcpu) {}
 static inline void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) {}
+static inline int tdx_vcpu_pre_run(struct kvm_vcpu *vcpu) { return -EOPNOTSUPP; }
+static inline fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit)
+{
+       return EXIT_FASTPATH_NONE;
+}
 
 static inline int tdx_vcpu_ioctl(struct kvm_vcpu *vcpu, void __user *argp) { return -EOPNOTSUPP; }