arch/arm64/kvm/hyp/nvhe/hyp-main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020 - Google Inc
   4  * Author: Andrew Scull <ascull@google.com>
   5  */
   6
   7 #include <hyp/switch.h>
   8
   9 #include <asm/pgtable-types.h>
  10 #include <asm/kvm_asm.h>
  11 #include <asm/kvm_emulate.h>
  12 #include <asm/kvm_host.h>
  13 #include <asm/kvm_hyp.h>
  14 #include <asm/kvm_mmu.h>
  15
  16 #include <nvhe/mem_protect.h>
  17 #include <nvhe/mm.h>
  18 #include <nvhe/trap_handler.h>
  19
  20 DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
  21
  22 void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt);
  23
  24 static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
  25 {
  26         DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1);
  27
  28         cpu_reg(host_ctxt, 1) =  __kvm_vcpu_run(kern_hyp_va(vcpu));
  29 }
  30
  31 static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt)
  32 {
  33         DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1);
  34
  35         __kvm_adjust_pc(kern_hyp_va(vcpu));
  36 }
  37
  38 static void handle___kvm_flush_vm_context(struct kvm_cpu_context *host_ctxt)
  39 {
  40         __kvm_flush_vm_context();
  41 }
  42
  43 static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt)
  44 {
  45         DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
  46         DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2);
  47         DECLARE_REG(int, level, host_ctxt, 3);
  48
  49         __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level);
  50 }
  51
  52 static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt)
  53 {
  54         DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
  55
  56         __kvm_tlb_flush_vmid(kern_hyp_va(mmu));
  57 }
  58
  59 static void handle___kvm_flush_cpu_context(struct kvm_cpu_context *host_ctxt)
  60 {
  61         DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
  62
  63         __kvm_flush_cpu_context(kern_hyp_va(mmu));
  64 }
  65
  66 static void handle___kvm_timer_set_cntvoff(struct kvm_cpu_context *host_ctxt)
  67 {
  68         __kvm_timer_set_cntvoff(cpu_reg(host_ctxt, 1));
  69 }
  70
  71 static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt)
  72 {
  73         u64 tmp;
  74
  75         tmp = read_sysreg_el2(SYS_SCTLR);
  76         tmp |= SCTLR_ELx_DSSBS;
  77         write_sysreg_el2(tmp, SYS_SCTLR);
  78 }
  79
  80 static void handle___vgic_v3_get_gic_config(struct kvm_cpu_context *host_ctxt)
  81 {
  82         cpu_reg(host_ctxt, 1) = __vgic_v3_get_gic_config();
  83 }
  84
  85 static void handle___vgic_v3_read_vmcr(struct kvm_cpu_context *host_ctxt)
  86 {
  87         cpu_reg(host_ctxt, 1) = __vgic_v3_read_vmcr();
  88 }
  89
  90 static void handle___vgic_v3_write_vmcr(struct kvm_cpu_context *host_ctxt)
  91 {
  92         __vgic_v3_write_vmcr(cpu_reg(host_ctxt, 1));
  93 }
  94
  95 static void handle___vgic_v3_init_lrs(struct kvm_cpu_context *host_ctxt)
  96 {
  97         __vgic_v3_init_lrs();
  98 }
  99
 100 static void handle___kvm_get_mdcr_el2(struct kvm_cpu_context *host_ctxt)
 101 {
 102         cpu_reg(host_ctxt, 1) = __kvm_get_mdcr_el2();
 103 }
 104
 105 static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt)
 106 {
 107         DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1);
 108
 109         __vgic_v3_save_aprs(kern_hyp_va(cpu_if));
 110 }
 111
 112 static void handle___vgic_v3_restore_aprs(struct kvm_cpu_context *host_ctxt)
 113 {
 114         DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1);
 115
 116         __vgic_v3_restore_aprs(kern_hyp_va(cpu_if));
 117 }
 118
 119 static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt)
 120 {
 121         DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
 122         DECLARE_REG(unsigned long, size, host_ctxt, 2);
 123         DECLARE_REG(unsigned long, nr_cpus, host_ctxt, 3);
 124         DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 4);
 125         DECLARE_REG(u32, hyp_va_bits, host_ctxt, 5);
 126
 127         /*
 128          * __pkvm_init() will return only if an error occurred, otherwise it
 129          * will tail-call in __pkvm_init_finalise() which will have to deal
 130          * with the host context directly.
 131          */
 132         cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, nr_cpus, per_cpu_base,
 133                                             hyp_va_bits);
 134 }
 135
 136 static void handle___pkvm_cpu_set_vector(struct kvm_cpu_context *host_ctxt)
 137 {
 138         DECLARE_REG(enum arm64_hyp_spectre_vector, slot, host_ctxt, 1);
 139
 140         cpu_reg(host_ctxt, 1) = pkvm_cpu_set_vector(slot);
 141 }
 142
 143 static void handle___pkvm_create_mappings(struct kvm_cpu_context *host_ctxt)
 144 {
 145         DECLARE_REG(unsigned long, start, host_ctxt, 1);
 146         DECLARE_REG(unsigned long, size, host_ctxt, 2);
 147         DECLARE_REG(unsigned long, phys, host_ctxt, 3);
 148         DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 4);
 149
 150         cpu_reg(host_ctxt, 1) = __pkvm_create_mappings(start, size, phys, prot);
 151 }
 152
 153 static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt)
 154 {
 155         DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
 156         DECLARE_REG(size_t, size, host_ctxt, 2);
 157         DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 3);
 158
 159         cpu_reg(host_ctxt, 1) = __pkvm_create_private_mapping(phys, size, prot);
 160 }
 161
 162 static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt)
 163 {
 164         cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize();
 165 }
 166
 167 static void handle___pkvm_mark_hyp(struct kvm_cpu_context *host_ctxt)
 168 {
 169         DECLARE_REG(phys_addr_t, start, host_ctxt, 1);
 170         DECLARE_REG(phys_addr_t, end, host_ctxt, 2);
 171
 172         cpu_reg(host_ctxt, 1) = __pkvm_mark_hyp(start, end);
 173 }
 174 typedef void (*hcall_t)(struct kvm_cpu_context *);
 175
 176 #define HANDLE_FUNC(x)  [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
 177
 178 static const hcall_t host_hcall[] = {
 179         HANDLE_FUNC(__kvm_vcpu_run),
 180         HANDLE_FUNC(__kvm_adjust_pc),
 181         HANDLE_FUNC(__kvm_flush_vm_context),
 182         HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa),
 183         HANDLE_FUNC(__kvm_tlb_flush_vmid),
 184         HANDLE_FUNC(__kvm_flush_cpu_context),
 185         HANDLE_FUNC(__kvm_timer_set_cntvoff),
 186         HANDLE_FUNC(__kvm_enable_ssbs),
 187         HANDLE_FUNC(__vgic_v3_get_gic_config),
 188         HANDLE_FUNC(__vgic_v3_read_vmcr),
 189         HANDLE_FUNC(__vgic_v3_write_vmcr),
 190         HANDLE_FUNC(__vgic_v3_init_lrs),
 191         HANDLE_FUNC(__kvm_get_mdcr_el2),
 192         HANDLE_FUNC(__vgic_v3_save_aprs),
 193         HANDLE_FUNC(__vgic_v3_restore_aprs),
 194         HANDLE_FUNC(__pkvm_init),
 195         HANDLE_FUNC(__pkvm_cpu_set_vector),
 196         HANDLE_FUNC(__pkvm_create_mappings),
 197         HANDLE_FUNC(__pkvm_create_private_mapping),
 198         HANDLE_FUNC(__pkvm_prot_finalize),
 199         HANDLE_FUNC(__pkvm_mark_hyp),
 200 };
 201
 202 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
 203 {
 204         DECLARE_REG(unsigned long, id, host_ctxt, 0);
 205         hcall_t hfn;
 206
 207         id -= KVM_HOST_SMCCC_ID(0);
 208
 209         if (unlikely(id >= ARRAY_SIZE(host_hcall)))
 210                 goto inval;
 211
 212         hfn = host_hcall[id];
 213         if (unlikely(!hfn))
 214                 goto inval;
 215
 216         cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS;
 217         hfn(host_ctxt);
 218
 219         return;
 220 inval:
 221         cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED;
 222 }
 223
 224 static void default_host_smc_handler(struct kvm_cpu_context *host_ctxt)
 225 {
 226         __kvm_hyp_host_forward_smc(host_ctxt);
 227 }
 228
 229 static void handle_host_smc(struct kvm_cpu_context *host_ctxt)
 230 {
 231         bool handled;
 232
 233         handled = kvm_host_psci_handler(host_ctxt);
 234         if (!handled)
 235                 default_host_smc_handler(host_ctxt);
 236
 237         /* SMC was trapped, move ELR past the current PC. */
 238         kvm_skip_host_instr();
 239 }
 240
 241 void handle_trap(struct kvm_cpu_context *host_ctxt)
 242 {
 243         u64 esr = read_sysreg_el2(SYS_ESR);
 244
 245         switch (ESR_ELx_EC(esr)) {
 246         case ESR_ELx_EC_HVC64:
 247                 handle_host_hcall(host_ctxt);
 248                 break;
 249         case ESR_ELx_EC_SMC64:
 250                 handle_host_smc(host_ctxt);
 251                 break;
 252         case ESR_ELx_EC_SVE:
 253                 sysreg_clear_set(cptr_el2, CPTR_EL2_TZ, 0);
 254                 isb();
 255                 sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
 256                 break;
 257         case ESR_ELx_EC_IABT_LOW:
 258         case ESR_ELx_EC_DABT_LOW:
 259                 handle_host_mem_abort(host_ctxt);
 260                 break;
 261         default:
 262                 BUG();
 263         }
 264 }