KVM: arm64: Make kvm_skip_instr() and co private to HYP

In an effort to remove the vcpu PC manipulations from EL1 on nVHE
systems, move kvm_skip_instr() to be HYP-specific. EL1's intent
to increment PC post emulation is now signalled via a flag in the
vcpu structure.

Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 0864f42..6d2b5d1 100644 (file)
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -472,32 +472,9 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
        return data;            /* Leave LE untouched */
 }
 
-static __always_inline void kvm_skip_instr(struct kvm_vcpu *vcpu)
+static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)
 {
-       if (vcpu_mode_is_32bit(vcpu)) {
-               kvm_skip_instr32(vcpu);
-       } else {
-               *vcpu_pc(vcpu) += 4;
-               *vcpu_cpsr(vcpu) &= ~PSR_BTYPE_MASK;
-       }
-
-       /* advance the singlestep state machine */
-       *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
-}
-
-/*
- * Skip an instruction which has been emulated at hyp while most guest sysregs
- * are live.
- */
-static __always_inline void __kvm_skip_instr(struct kvm_vcpu *vcpu)
-{
-       *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
-       vcpu_gp_regs(vcpu)->pstate = read_sysreg_el2(SYS_SPSR);
-
-       kvm_skip_instr(vcpu);
-
-       write_sysreg_el2(vcpu_gp_regs(vcpu)->pstate, SYS_SPSR);
-       write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
+       vcpu->arch.flags |= KVM_ARM64_INCREMENT_PC;
 }
 
 #endif /* __ARM64_KVM_EMULATE_H__ */

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 781d029..7991a1c 100644 (file)
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -407,6 +407,7 @@ struct kvm_vcpu_arch {
 #define KVM_ARM64_GUEST_HAS_SVE                (1 << 5) /* SVE exposed to guest */
 #define KVM_ARM64_VCPU_SVE_FINALIZED   (1 << 6) /* SVE config completed */
 #define KVM_ARM64_GUEST_HAS_PTRAUTH    (1 << 7) /* PTRAUTH exposed to guest */
+#define KVM_ARM64_INCREMENT_PC         (1 << 8) /* Increment PC */
 
 #define vcpu_has_sve(vcpu) (system_supports_sve() && \
                            ((vcpu)->arch.flags & KVM_ARM64_GUEST_HAS_SVE))

diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index 30bf8e2..d4e00a8 100644 (file)
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -61,7 +61,7 @@ static int handle_smc(struct kvm_vcpu *vcpu)
         * otherwise return to the same address...
         */
        vcpu_set_reg(vcpu, 0, ~0UL);
-       kvm_skip_instr(vcpu);
+       kvm_incr_pc(vcpu);
        return 1;
 }
 
@@ -100,7 +100,7 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu)
                kvm_clear_request(KVM_REQ_UNHALT, vcpu);
        }
 
-       kvm_skip_instr(vcpu);
+       kvm_incr_pc(vcpu);
 
        return 1;
 }
@@ -221,7 +221,7 @@ static int handle_trap_exceptions(struct kvm_vcpu *vcpu)
         * that fail their condition code check"
         */
        if (!kvm_condition_valid(vcpu)) {
-               kvm_skip_instr(vcpu);
+               kvm_incr_pc(vcpu);
                handled = 1;
        } else {
                exit_handle_fn exit_handler;

diff --git a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h
new file mode 100644 (file)
index 0000000..d3043b0
--- /dev/null
+++ b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Guest PC manipulation helpers
+ *
+ * Copyright (C) 2012,2013 - ARM Ltd
+ * Copyright (C) 2020 - Google LLC
+ * Author: Marc Zyngier <maz@kernel.org>
+ */
+
+#ifndef __ARM64_KVM_HYP_ADJUST_PC_H__
+#define __ARM64_KVM_HYP_ADJUST_PC_H__
+
+#include <asm/kvm_emulate.h>
+#include <asm/kvm_host.h>
+
+static inline void kvm_skip_instr(struct kvm_vcpu *vcpu)
+{
+       if (vcpu_mode_is_32bit(vcpu)) {
+               kvm_skip_instr32(vcpu);
+       } else {
+               *vcpu_pc(vcpu) += 4;
+               *vcpu_cpsr(vcpu) &= ~PSR_BTYPE_MASK;
+       }
+
+       /* advance the singlestep state machine */
+       *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
+}
+
+/*
+ * Skip an instruction which has been emulated at hyp while most guest sysregs
+ * are live.
+ */
+static inline void __kvm_skip_instr(struct kvm_vcpu *vcpu)
+{
+       *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
+       vcpu_gp_regs(vcpu)->pstate = read_sysreg_el2(SYS_SPSR);
+
+       kvm_skip_instr(vcpu);
+
+       write_sysreg_el2(vcpu_gp_regs(vcpu)->pstate, SYS_SPSR);
+       write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
+}
+
+/*
+ * Adjust the guest PC on entry, depending on flags provided by EL1
+ * for the purpose of emulation (MMIO, sysreg).
+ */
+static inline void __adjust_pc(struct kvm_vcpu *vcpu)
+{
+       if (vcpu->arch.flags & KVM_ARM64_INCREMENT_PC) {
+               kvm_skip_instr(vcpu);
+               vcpu->arch.flags &= ~KVM_ARM64_INCREMENT_PC;
+       }
+}
+
+#endif

diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 1f875a8..8b2328f 100644 (file)
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -7,6 +7,8 @@
 #ifndef __ARM64_KVM_HYP_SWITCH_H__
 #define __ARM64_KVM_HYP_SWITCH_H__
 
+#include <hyp/adjust_pc.h>
+
 #include <linux/arm-smccc.h>
 #include <linux/kvm_host.h>
 #include <linux/types.h>

diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 8ae8160..3e50ff3 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -4,6 +4,7 @@
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  */
 
+#include <hyp/adjust_pc.h>
 #include <hyp/switch.h>
 #include <hyp/sysreg-sr.h>
 
@@ -189,6 +190,8 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
 
        __sysreg_save_state_nvhe(host_ctxt);
 
+       __adjust_pc(vcpu);
+
        /*
         * We must restore the 32-bit state before the sysregs, thanks
         * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).

diff --git a/arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c b/arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c
index bd1bab5..8f05856 100644 (file)
--- a/arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c
+++ b/arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c
@@ -4,6 +4,8 @@
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  */
 
+#include <hyp/adjust_pc.h>
+
 #include <linux/compiler.h>
 #include <linux/irqchip/arm-gic.h>
 #include <linux/kvm_host.h>

diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c
index 452f4ca..80406f4 100644 (file)
--- a/arch/arm64/kvm/hyp/vgic-v3-sr.c
+++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c
@@ -4,6 +4,8 @@
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  */
 
+#include <hyp/adjust_pc.h>
+
 #include <linux/compiler.h>
 #include <linux/irqchip/arm-gic-v3.h>
 #include <linux/kvm_host.h>

diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index 62546e2..af8e940 100644 (file)
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -4,6 +4,7 @@
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  */
 
+#include <hyp/adjust_pc.h>
 #include <hyp/switch.h>
 
 #include <linux/arm-smccc.h>
@@ -133,6 +134,8 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
        __load_guest_stage2(vcpu->arch.hw_mmu);
        __activate_traps(vcpu);
 
+       __adjust_pc(vcpu);
+
        sysreg_restore_guest_state_vhe(guest_ctxt);
        __debug_switch_to_guest(vcpu);
 

diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c
index 7e8eb32..3e2d8ba 100644 (file)
--- a/arch/arm64/kvm/mmio.c
+++ b/arch/arm64/kvm/mmio.c
@@ -115,7 +115,7 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu)
         * The MMIO instruction is emulated and should not be re-executed
         * in the guest.
         */
-       kvm_skip_instr(vcpu);
+       kvm_incr_pc(vcpu);
 
        return 0;
 }

diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 0bec07c..2ecd0c5 100644 (file)
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1014,7 +1014,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
                 * cautious, and skip the instruction.
                 */
                if (kvm_is_error_hva(hva) && kvm_vcpu_dabt_is_cm(vcpu)) {
-                       kvm_skip_instr(vcpu);
+                       kvm_incr_pc(vcpu);
                        ret = 1;
                        goto out_unlock;
                }

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 1232a81..6a6f062 100644 (file)
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -2199,7 +2199,7 @@ static void perform_access(struct kvm_vcpu *vcpu,
 
        /* Skip instruction if instructed so */
        if (likely(r->access(vcpu, params, r)))
-               kvm_skip_instr(vcpu);
+               kvm_incr_pc(vcpu);
 }
 
 /*