arch/arm64/kvm/psci.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2012 - ARM Ltd
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/arm-smccc.h>
   8 #include <linux/preempt.h>
   9 #include <linux/kvm_host.h>
  10 #include <linux/uaccess.h>
  11 #include <linux/wait.h>
  12
  13 #include <asm/cputype.h>
  14 #include <asm/kvm_emulate.h>
  15
  16 #include <kvm/arm_psci.h>
  17 #include <kvm/arm_hypercalls.h>
  18
  19 /*
  20  * This is an implementation of the Power State Coordination Interface
  21  * as described in ARM document number ARM DEN 0022A.
  22  */
  23
  24 #define AFFINITY_MASK(level)    ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1)
  25
  26 static unsigned long psci_affinity_mask(unsigned long affinity_level)
  27 {
  28         if (affinity_level <= 3)
  29                 return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level);
  30
  31         return 0;
  32 }
  33
  34 static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu)
  35 {
  36         /*
  37          * NOTE: For simplicity, we make VCPU suspend emulation to be
  38          * same-as WFI (Wait-for-interrupt) emulation.
  39          *
  40          * This means for KVM the wakeup events are interrupts and
  41          * this is consistent with intended use of StateID as described
  42          * in section 5.4.1 of PSCI v0.2 specification (ARM DEN 0022A).
  43          *
  44          * Further, we also treat power-down request to be same as
  45          * stand-by request as-per section 5.4.2 clause 3 of PSCI v0.2
  46          * specification (ARM DEN 0022A). This means all suspend states
  47          * for KVM will preserve the register state.
  48          */
  49         kvm_vcpu_wfi(vcpu);
  50
  51         return PSCI_RET_SUCCESS;
  52 }
  53
  54 static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu,
  55                                            unsigned long affinity)
  56 {
  57         return !(affinity & ~MPIDR_HWID_BITMASK);
  58 }
  59
  60 static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
  61 {
  62         struct vcpu_reset_state *reset_state;
  63         struct kvm *kvm = source_vcpu->kvm;
  64         struct kvm_vcpu *vcpu = NULL;
  65         unsigned long cpu_id;
  66
  67         cpu_id = smccc_get_arg1(source_vcpu);
  68         if (!kvm_psci_valid_affinity(source_vcpu, cpu_id))
  69                 return PSCI_RET_INVALID_PARAMS;
  70
  71         vcpu = kvm_mpidr_to_vcpu(kvm, cpu_id);
  72
  73         /*
  74          * Make sure the caller requested a valid CPU and that the CPU is
  75          * turned off.
  76          */
  77         if (!vcpu)
  78                 return PSCI_RET_INVALID_PARAMS;
  79         if (!kvm_arm_vcpu_stopped(vcpu)) {
  80                 if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1)
  81                         return PSCI_RET_ALREADY_ON;
  82                 else
  83                         return PSCI_RET_INVALID_PARAMS;
  84         }
  85
  86         reset_state = &vcpu->arch.reset_state;
  87
  88         reset_state->pc = smccc_get_arg2(source_vcpu);
  89
  90         /* Propagate caller endianness */
  91         reset_state->be = kvm_vcpu_is_be(source_vcpu);
  92
  93         /*
  94          * NOTE: We always update r0 (or x0) because for PSCI v0.1
  95          * the general purpose registers are undefined upon CPU_ON.
  96          */
  97         reset_state->r0 = smccc_get_arg3(source_vcpu);
  98
  99         WRITE_ONCE(reset_state->reset, true);
 100         kvm_make_request(KVM_REQ_VCPU_RESET, vcpu);
 101
 102         /*
 103          * Make sure the reset request is observed if the RUNNABLE mp_state is
 104          * observed.
 105          */
 106         smp_wmb();
 107
 108         vcpu->arch.mp_state.mp_state = KVM_MP_STATE_RUNNABLE;
 109         kvm_vcpu_wake_up(vcpu);
 110
 111         return PSCI_RET_SUCCESS;
 112 }
 113
 114 static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu)
 115 {
 116         int matching_cpus = 0;
 117         unsigned long i, mpidr;
 118         unsigned long target_affinity;
 119         unsigned long target_affinity_mask;
 120         unsigned long lowest_affinity_level;
 121         struct kvm *kvm = vcpu->kvm;
 122         struct kvm_vcpu *tmp;
 123
 124         target_affinity = smccc_get_arg1(vcpu);
 125         lowest_affinity_level = smccc_get_arg2(vcpu);
 126
 127         if (!kvm_psci_valid_affinity(vcpu, target_affinity))
 128                 return PSCI_RET_INVALID_PARAMS;
 129
 130         /* Determine target affinity mask */
 131         target_affinity_mask = psci_affinity_mask(lowest_affinity_level);
 132         if (!target_affinity_mask)
 133                 return PSCI_RET_INVALID_PARAMS;
 134
 135         /* Ignore other bits of target affinity */
 136         target_affinity &= target_affinity_mask;
 137
 138         /*
 139          * If one or more VCPU matching target affinity are running
 140          * then ON else OFF
 141          */
 142         kvm_for_each_vcpu(i, tmp, kvm) {
 143                 mpidr = kvm_vcpu_get_mpidr_aff(tmp);
 144                 if ((mpidr & target_affinity_mask) == target_affinity) {
 145                         matching_cpus++;
 146                         if (!kvm_arm_vcpu_stopped(tmp))
 147                                 return PSCI_0_2_AFFINITY_LEVEL_ON;
 148                 }
 149         }
 150
 151         if (!matching_cpus)
 152                 return PSCI_RET_INVALID_PARAMS;
 153
 154         return PSCI_0_2_AFFINITY_LEVEL_OFF;
 155 }
 156
 157 static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags)
 158 {
 159         unsigned long i;
 160         struct kvm_vcpu *tmp;
 161
 162         /*
 163          * The KVM ABI specifies that a system event exit may call KVM_RUN
 164          * again and may perform shutdown/reboot at a later time that when the
 165          * actual request is made.  Since we are implementing PSCI and a
 166          * caller of PSCI reboot and shutdown expects that the system shuts
 167          * down or reboots immediately, let's make sure that VCPUs are not run
 168          * after this call is handled and before the VCPUs have been
 169          * re-initialized.
 170          */
 171         kvm_for_each_vcpu(i, tmp, vcpu->kvm)
 172                 tmp->arch.mp_state.mp_state = KVM_MP_STATE_STOPPED;
 173         kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP);
 174
 175         memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
 176         vcpu->run->system_event.type = type;
 177         vcpu->run->system_event.ndata = 1;
 178         vcpu->run->system_event.data[0] = flags;
 179         vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
 180 }
 181
 182 static void kvm_psci_system_off(struct kvm_vcpu *vcpu)
 183 {
 184         kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_SHUTDOWN, 0);
 185 }
 186
 187 static void kvm_psci_system_reset(struct kvm_vcpu *vcpu)
 188 {
 189         kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, 0);
 190 }
 191
 192 static void kvm_psci_system_reset2(struct kvm_vcpu *vcpu)
 193 {
 194         kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET,
 195                                  KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2);
 196 }
 197
 198 static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu)
 199 {
 200         struct kvm_run *run = vcpu->run;
 201
 202         memset(&run->system_event, 0, sizeof(vcpu->run->system_event));
 203         run->system_event.type = KVM_SYSTEM_EVENT_SUSPEND;
 204         run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
 205 }
 206
 207 static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu)
 208 {
 209         int i;
 210
 211         /*
 212          * Zero the input registers' upper 32 bits. They will be fully
 213          * zeroed on exit, so we're fine changing them in place.
 214          */
 215         for (i = 1; i < 4; i++)
 216                 vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i)));
 217 }
 218
 219 static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn)
 220 {
 221         /*
 222          * Prevent 32 bit guests from calling 64 bit PSCI functions.
 223          */
 224         if ((fn & PSCI_0_2_64BIT) && vcpu_mode_is_32bit(vcpu))
 225                 return PSCI_RET_NOT_SUPPORTED;
 226
 227         return 0;
 228 }
 229
 230 static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu)
 231 {
 232         struct kvm *kvm = vcpu->kvm;
 233         u32 psci_fn = smccc_get_function(vcpu);
 234         unsigned long val;
 235         int ret = 1;
 236
 237         switch (psci_fn) {
 238         case PSCI_0_2_FN_PSCI_VERSION:
 239                 /*
 240                  * Bits[31:16] = Major Version = 0
 241                  * Bits[15:0] = Minor Version = 2
 242                  */
 243                 val = KVM_ARM_PSCI_0_2;
 244                 break;
 245         case PSCI_0_2_FN_CPU_SUSPEND:
 246         case PSCI_0_2_FN64_CPU_SUSPEND:
 247                 val = kvm_psci_vcpu_suspend(vcpu);
 248                 break;
 249         case PSCI_0_2_FN_CPU_OFF:
 250                 kvm_arm_vcpu_power_off(vcpu);
 251                 val = PSCI_RET_SUCCESS;
 252                 break;
 253         case PSCI_0_2_FN_CPU_ON:
 254                 kvm_psci_narrow_to_32bit(vcpu);
 255                 fallthrough;
 256         case PSCI_0_2_FN64_CPU_ON:
 257                 mutex_lock(&kvm->lock);
 258                 val = kvm_psci_vcpu_on(vcpu);
 259                 mutex_unlock(&kvm->lock);
 260                 break;
 261         case PSCI_0_2_FN_AFFINITY_INFO:
 262                 kvm_psci_narrow_to_32bit(vcpu);
 263                 fallthrough;
 264         case PSCI_0_2_FN64_AFFINITY_INFO:
 265                 val = kvm_psci_vcpu_affinity_info(vcpu);
 266                 break;
 267         case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
 268                 /*
 269                  * Trusted OS is MP hence does not require migration
 270                  * or
 271                  * Trusted OS is not present
 272                  */
 273                 val = PSCI_0_2_TOS_MP;
 274                 break;
 275         case PSCI_0_2_FN_SYSTEM_OFF:
 276                 kvm_psci_system_off(vcpu);
 277                 /*
 278                  * We shouldn't be going back to guest VCPU after
 279                  * receiving SYSTEM_OFF request.
 280                  *
 281                  * If user space accidentally/deliberately resumes
 282                  * guest VCPU after SYSTEM_OFF request then guest
 283                  * VCPU should see internal failure from PSCI return
 284                  * value. To achieve this, we preload r0 (or x0) with
 285                  * PSCI return value INTERNAL_FAILURE.
 286                  */
 287                 val = PSCI_RET_INTERNAL_FAILURE;
 288                 ret = 0;
 289                 break;
 290         case PSCI_0_2_FN_SYSTEM_RESET:
 291                 kvm_psci_system_reset(vcpu);
 292                 /*
 293                  * Same reason as SYSTEM_OFF for preloading r0 (or x0)
 294                  * with PSCI return value INTERNAL_FAILURE.
 295                  */
 296                 val = PSCI_RET_INTERNAL_FAILURE;
 297                 ret = 0;
 298                 break;
 299         default:
 300                 val = PSCI_RET_NOT_SUPPORTED;
 301                 break;
 302         }
 303
 304         smccc_set_retval(vcpu, val, 0, 0, 0);
 305         return ret;
 306 }
 307
 308 static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor)
 309 {
 310         unsigned long val = PSCI_RET_NOT_SUPPORTED;
 311         u32 psci_fn = smccc_get_function(vcpu);
 312         struct kvm *kvm = vcpu->kvm;
 313         u32 arg;
 314         int ret = 1;
 315
 316         switch(psci_fn) {
 317         case PSCI_0_2_FN_PSCI_VERSION:
 318                 val = minor == 0 ? KVM_ARM_PSCI_1_0 : KVM_ARM_PSCI_1_1;
 319                 break;
 320         case PSCI_1_0_FN_PSCI_FEATURES:
 321                 arg = smccc_get_arg1(vcpu);
 322                 val = kvm_psci_check_allowed_function(vcpu, arg);
 323                 if (val)
 324                         break;
 325
 326                 val = PSCI_RET_NOT_SUPPORTED;
 327
 328                 switch(arg) {
 329                 case PSCI_0_2_FN_PSCI_VERSION:
 330                 case PSCI_0_2_FN_CPU_SUSPEND:
 331                 case PSCI_0_2_FN64_CPU_SUSPEND:
 332                 case PSCI_0_2_FN_CPU_OFF:
 333                 case PSCI_0_2_FN_CPU_ON:
 334                 case PSCI_0_2_FN64_CPU_ON:
 335                 case PSCI_0_2_FN_AFFINITY_INFO:
 336                 case PSCI_0_2_FN64_AFFINITY_INFO:
 337                 case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
 338                 case PSCI_0_2_FN_SYSTEM_OFF:
 339                 case PSCI_0_2_FN_SYSTEM_RESET:
 340                 case PSCI_1_0_FN_PSCI_FEATURES:
 341                 case ARM_SMCCC_VERSION_FUNC_ID:
 342                         val = 0;
 343                         break;
 344                 case PSCI_1_0_FN_SYSTEM_SUSPEND:
 345                 case PSCI_1_0_FN64_SYSTEM_SUSPEND:
 346                         if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags))
 347                                 val = 0;
 348                         break;
 349                 case PSCI_1_1_FN_SYSTEM_RESET2:
 350                 case PSCI_1_1_FN64_SYSTEM_RESET2:
 351                         if (minor >= 1)
 352                                 val = 0;
 353                         break;
 354                 }
 355                 break;
 356         case PSCI_1_0_FN_SYSTEM_SUSPEND:
 357                 kvm_psci_narrow_to_32bit(vcpu);
 358                 fallthrough;
 359         case PSCI_1_0_FN64_SYSTEM_SUSPEND:
 360                 /*
 361                  * Return directly to userspace without changing the vCPU's
 362                  * registers. Userspace depends on reading the SMCCC parameters
 363                  * to implement SYSTEM_SUSPEND.
 364                  */
 365                 if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) {
 366                         kvm_psci_system_suspend(vcpu);
 367                         return 0;
 368                 }
 369                 break;
 370         case PSCI_1_1_FN_SYSTEM_RESET2:
 371                 kvm_psci_narrow_to_32bit(vcpu);
 372                 fallthrough;
 373         case PSCI_1_1_FN64_SYSTEM_RESET2:
 374                 if (minor >= 1) {
 375                         arg = smccc_get_arg1(vcpu);
 376
 377                         if (arg <= PSCI_1_1_RESET_TYPE_SYSTEM_WARM_RESET ||
 378                             arg >= PSCI_1_1_RESET_TYPE_VENDOR_START) {
 379                                 kvm_psci_system_reset2(vcpu);
 380                                 vcpu_set_reg(vcpu, 0, PSCI_RET_INTERNAL_FAILURE);
 381                                 return 0;
 382                         }
 383
 384                         val = PSCI_RET_INVALID_PARAMS;
 385                         break;
 386                 }
 387                 break;
 388         default:
 389                 return kvm_psci_0_2_call(vcpu);
 390         }
 391
 392         smccc_set_retval(vcpu, val, 0, 0, 0);
 393         return ret;
 394 }
 395
 396 static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu)
 397 {
 398         struct kvm *kvm = vcpu->kvm;
 399         u32 psci_fn = smccc_get_function(vcpu);
 400         unsigned long val;
 401
 402         switch (psci_fn) {
 403         case KVM_PSCI_FN_CPU_OFF:
 404                 kvm_arm_vcpu_power_off(vcpu);
 405                 val = PSCI_RET_SUCCESS;
 406                 break;
 407         case KVM_PSCI_FN_CPU_ON:
 408                 mutex_lock(&kvm->lock);
 409                 val = kvm_psci_vcpu_on(vcpu);
 410                 mutex_unlock(&kvm->lock);
 411                 break;
 412         default:
 413                 val = PSCI_RET_NOT_SUPPORTED;
 414                 break;
 415         }
 416
 417         smccc_set_retval(vcpu, val, 0, 0, 0);
 418         return 1;
 419 }
 420
 421 /**
 422  * kvm_psci_call - handle PSCI call if r0 value is in range
 423  * @vcpu: Pointer to the VCPU struct
 424  *
 425  * Handle PSCI calls from guests through traps from HVC instructions.
 426  * The calling convention is similar to SMC calls to the secure world
 427  * where the function number is placed in r0.
 428  *
 429  * This function returns: > 0 (success), 0 (success but exit to user
 430  * space), and < 0 (errors)
 431  *
 432  * Errors:
 433  * -EINVAL: Unrecognized PSCI function
 434  */
 435 int kvm_psci_call(struct kvm_vcpu *vcpu)
 436 {
 437         u32 psci_fn = smccc_get_function(vcpu);
 438         unsigned long val;
 439
 440         val = kvm_psci_check_allowed_function(vcpu, psci_fn);
 441         if (val) {
 442                 smccc_set_retval(vcpu, val, 0, 0, 0);
 443                 return 1;
 444         }
 445
 446         switch (kvm_psci_version(vcpu)) {
 447         case KVM_ARM_PSCI_1_1:
 448                 return kvm_psci_1_x_call(vcpu, 1);
 449         case KVM_ARM_PSCI_1_0:
 450                 return kvm_psci_1_x_call(vcpu, 0);
 451         case KVM_ARM_PSCI_0_2:
 452                 return kvm_psci_0_2_call(vcpu);
 453         case KVM_ARM_PSCI_0_1:
 454                 return kvm_psci_0_1_call(vcpu);
 455         default:
 456                 return -EINVAL;
 457         }
 458 }