virt/kvm/arm/vgic/vgic-init.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2015, 2016 ARM Ltd.
   4  */
   5
   6 #include <linux/uaccess.h>
   7 #include <linux/interrupt.h>
   8 #include <linux/cpu.h>
   9 #include <linux/kvm_host.h>
  10 #include <kvm/arm_vgic.h>
  11 #include <asm/kvm_mmu.h>
  12 #include "vgic.h"
  13
  14 /*
  15  * Initialization rules: there are multiple stages to the vgic
  16  * initialization, both for the distributor and the CPU interfaces.  The basic
  17  * idea is that even though the VGIC is not functional or not requested from
  18  * user space, the critical path of the run loop can still call VGIC functions
  19  * that just won't do anything, without them having to check additional
  20  * initialization flags to ensure they don't look at uninitialized data
  21  * structures.
  22  *
  23  * Distributor:
  24  *
  25  * - kvm_vgic_early_init(): initialization of static data that doesn't
  26  *   depend on any sizing information or emulation type. No allocation
  27  *   is allowed there.
  28  *
  29  * - vgic_init(): allocation and initialization of the generic data
  30  *   structures that depend on sizing information (number of CPUs,
  31  *   number of interrupts). Also initializes the vcpu specific data
  32  *   structures. Can be executed lazily for GICv2.
  33  *
  34  * CPU Interface:
  35  *
  36  * - kvm_vgic_vcpu_init(): initialization of static data that
  37  *   doesn't depend on any sizing information or emulation type. No
  38  *   allocation is allowed there.
  39  */
  40
  41 /* EARLY INIT */
  42
  43 /**
  44  * kvm_vgic_early_init() - Initialize static VGIC VCPU data structures
  45  * @kvm: The VM whose VGIC districutor should be initialized
  46  *
  47  * Only do initialization of static structures that don't require any
  48  * allocation or sizing information from userspace.  vgic_init() called
  49  * kvm_vgic_dist_init() which takes care of the rest.
  50  */
  51 void kvm_vgic_early_init(struct kvm *kvm)
  52 {
  53         struct vgic_dist *dist = &kvm->arch.vgic;
  54
  55         INIT_LIST_HEAD(&dist->lpi_list_head);
  56         INIT_LIST_HEAD(&dist->lpi_translation_cache);
  57         raw_spin_lock_init(&dist->lpi_list_lock);
  58 }
  59
  60 /* CREATION */
  61
  62 /**
  63  * kvm_vgic_create: triggered by the instantiation of the VGIC device by
  64  * user space, either through the legacy KVM_CREATE_IRQCHIP ioctl (v2 only)
  65  * or through the generic KVM_CREATE_DEVICE API ioctl.
  66  * irqchip_in_kernel() tells you if this function succeeded or not.
  67  * @kvm: kvm struct pointer
  68  * @type: KVM_DEV_TYPE_ARM_VGIC_V[23]
  69  */
  70 int kvm_vgic_create(struct kvm *kvm, u32 type)
  71 {
  72         int i, vcpu_lock_idx = -1, ret;
  73         struct kvm_vcpu *vcpu;
  74
  75         if (irqchip_in_kernel(kvm))
  76                 return -EEXIST;
  77
  78         /*
  79          * This function is also called by the KVM_CREATE_IRQCHIP handler,
  80          * which had no chance yet to check the availability of the GICv2
  81          * emulation. So check this here again. KVM_CREATE_DEVICE does
  82          * the proper checks already.
  83          */
  84         if (type == KVM_DEV_TYPE_ARM_VGIC_V2 &&
  85                 !kvm_vgic_global_state.can_emulate_gicv2)
  86                 return -ENODEV;
  87
  88         /*
  89          * Any time a vcpu is run, vcpu_load is called which tries to grab the
  90          * vcpu->mutex.  By grabbing the vcpu->mutex of all VCPUs we ensure
  91          * that no other VCPUs are run while we create the vgic.
  92          */
  93         ret = -EBUSY;
  94         kvm_for_each_vcpu(i, vcpu, kvm) {
  95                 if (!mutex_trylock(&vcpu->mutex))
  96                         goto out_unlock;
  97                 vcpu_lock_idx = i;
  98         }
  99
 100         kvm_for_each_vcpu(i, vcpu, kvm) {
 101                 if (vcpu->arch.has_run_once)
 102                         goto out_unlock;
 103         }
 104         ret = 0;
 105
 106         if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
 107                 kvm->arch.max_vcpus = VGIC_V2_MAX_CPUS;
 108         else
 109                 kvm->arch.max_vcpus = VGIC_V3_MAX_CPUS;
 110
 111         if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus) {
 112                 ret = -E2BIG;
 113                 goto out_unlock;
 114         }
 115
 116         kvm->arch.vgic.in_kernel = true;
 117         kvm->arch.vgic.vgic_model = type;
 118
 119         kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
 120
 121         if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
 122                 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
 123         else
 124                 INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
 125
 126 out_unlock:
 127         for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
 128                 vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
 129                 mutex_unlock(&vcpu->mutex);
 130         }
 131         return ret;
 132 }
 133
 134 /* INIT/DESTROY */
 135
 136 /**
 137  * kvm_vgic_dist_init: initialize the dist data structures
 138  * @kvm: kvm struct pointer
 139  * @nr_spis: number of spis, frozen by caller
 140  */
 141 static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
 142 {
 143         struct vgic_dist *dist = &kvm->arch.vgic;
 144         struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
 145         int i;
 146
 147         dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL);
 148         if (!dist->spis)
 149                 return  -ENOMEM;
 150
 151         /*
 152          * In the following code we do not take the irq struct lock since
 153          * no other action on irq structs can happen while the VGIC is
 154          * not initialized yet:
 155          * If someone wants to inject an interrupt or does a MMIO access, we
 156          * require prior initialization in case of a virtual GICv3 or trigger
 157          * initialization when using a virtual GICv2.
 158          */
 159         for (i = 0; i < nr_spis; i++) {
 160                 struct vgic_irq *irq = &dist->spis[i];
 161
 162                 irq->intid = i + VGIC_NR_PRIVATE_IRQS;
 163                 INIT_LIST_HEAD(&irq->ap_list);
 164                 raw_spin_lock_init(&irq->irq_lock);
 165                 irq->vcpu = NULL;
 166                 irq->target_vcpu = vcpu0;
 167                 kref_init(&irq->refcount);
 168                 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2) {
 169                         irq->targets = 0;
 170                         irq->group = 0;
 171                 } else {
 172                         irq->mpidr = 0;
 173                         irq->group = 1;
 174                 }
 175         }
 176         return 0;
 177 }
 178
 179 /**
 180  * kvm_vgic_vcpu_init() - Initialize static VGIC VCPU data
 181  * structures and register VCPU-specific KVM iodevs
 182  *
 183  * @vcpu: pointer to the VCPU being created and initialized
 184  *
 185  * Only do initialization, but do not actually enable the
 186  * VGIC CPU interface
 187  */
 188 int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
 189 {
 190         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 191         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 192         int ret = 0;
 193         int i;
 194
 195         vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
 196         vgic_cpu->sgi_iodev.base_addr = VGIC_ADDR_UNDEF;
 197
 198         INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
 199         raw_spin_lock_init(&vgic_cpu->ap_list_lock);
 200
 201         /*
 202          * Enable and configure all SGIs to be edge-triggered and
 203          * configure all PPIs as level-triggered.
 204          */
 205         for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
 206                 struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
 207
 208                 INIT_LIST_HEAD(&irq->ap_list);
 209                 raw_spin_lock_init(&irq->irq_lock);
 210                 irq->intid = i;
 211                 irq->vcpu = NULL;
 212                 irq->target_vcpu = vcpu;
 213                 irq->targets = 1U << vcpu->vcpu_id;
 214                 kref_init(&irq->refcount);
 215                 if (vgic_irq_is_sgi(i)) {
 216                         /* SGIs */
 217                         irq->enabled = 1;
 218                         irq->config = VGIC_CONFIG_EDGE;
 219                 } else {
 220                         /* PPIs */
 221                         irq->config = VGIC_CONFIG_LEVEL;
 222                 }
 223
 224                 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
 225                         irq->group = 1;
 226                 else
 227                         irq->group = 0;
 228         }
 229
 230         if (!irqchip_in_kernel(vcpu->kvm))
 231                 return 0;
 232
 233         /*
 234          * If we are creating a VCPU with a GICv3 we must also register the
 235          * KVM io device for the redistributor that belongs to this VCPU.
 236          */
 237         if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
 238                 mutex_lock(&vcpu->kvm->lock);
 239                 ret = vgic_register_redist_iodev(vcpu);
 240                 mutex_unlock(&vcpu->kvm->lock);
 241         }
 242         return ret;
 243 }
 244
 245 static void kvm_vgic_vcpu_enable(struct kvm_vcpu *vcpu)
 246 {
 247         if (kvm_vgic_global_state.type == VGIC_V2)
 248                 vgic_v2_enable(vcpu);
 249         else
 250                 vgic_v3_enable(vcpu);
 251 }
 252
 253 /*
 254  * vgic_init: allocates and initializes dist and vcpu data structures
 255  * depending on two dimensioning parameters:
 256  * - the number of spis
 257  * - the number of vcpus
 258  * The function is generally called when nr_spis has been explicitly set
 259  * by the guest through the KVM DEVICE API. If not nr_spis is set to 256.
 260  * vgic_initialized() returns true when this function has succeeded.
 261  * Must be called with kvm->lock held!
 262  */
 263 int vgic_init(struct kvm *kvm)
 264 {
 265         struct vgic_dist *dist = &kvm->arch.vgic;
 266         struct kvm_vcpu *vcpu;
 267         int ret = 0, i, idx;
 268
 269         if (vgic_initialized(kvm))
 270                 return 0;
 271
 272         /* Are we also in the middle of creating a VCPU? */
 273         if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
 274                 return -EBUSY;
 275
 276         /* freeze the number of spis */
 277         if (!dist->nr_spis)
 278                 dist->nr_spis = VGIC_NR_IRQS_LEGACY - VGIC_NR_PRIVATE_IRQS;
 279
 280         ret = kvm_vgic_dist_init(kvm, dist->nr_spis);
 281         if (ret)
 282                 goto out;
 283
 284         /* Initialize groups on CPUs created before the VGIC type was known */
 285         kvm_for_each_vcpu(idx, vcpu, kvm) {
 286                 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 287
 288                 for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
 289                         struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
 290                         if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
 291                                 irq->group = 1;
 292                         else
 293                                 irq->group = 0;
 294                 }
 295         }
 296
 297         if (vgic_has_its(kvm)) {
 298                 vgic_lpi_translation_cache_init(kvm);
 299                 ret = vgic_v4_init(kvm);
 300                 if (ret)
 301                         goto out;
 302         }
 303
 304         kvm_for_each_vcpu(i, vcpu, kvm)
 305                 kvm_vgic_vcpu_enable(vcpu);
 306
 307         ret = kvm_vgic_setup_default_irq_routing(kvm);
 308         if (ret)
 309                 goto out;
 310
 311         vgic_debug_init(kvm);
 312
 313         dist->implementation_rev = 2;
 314         dist->initialized = true;
 315
 316 out:
 317         return ret;
 318 }
 319
 320 static void kvm_vgic_dist_destroy(struct kvm *kvm)
 321 {
 322         struct vgic_dist *dist = &kvm->arch.vgic;
 323         struct vgic_redist_region *rdreg, *next;
 324
 325         dist->ready = false;
 326         dist->initialized = false;
 327
 328         kfree(dist->spis);
 329         dist->spis = NULL;
 330         dist->nr_spis = 0;
 331
 332         if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
 333                 list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list) {
 334                         list_del(&rdreg->list);
 335                         kfree(rdreg);
 336                 }
 337                 INIT_LIST_HEAD(&dist->rd_regions);
 338         }
 339
 340         if (vgic_has_its(kvm))
 341                 vgic_lpi_translation_cache_destroy(kvm);
 342
 343         if (vgic_supports_direct_msis(kvm))
 344                 vgic_v4_teardown(kvm);
 345 }
 346
 347 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
 348 {
 349         struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 350
 351         INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
 352 }
 353
 354 /* To be called with kvm->lock held */
 355 static void __kvm_vgic_destroy(struct kvm *kvm)
 356 {
 357         struct kvm_vcpu *vcpu;
 358         int i;
 359
 360         vgic_debug_destroy(kvm);
 361
 362         kvm_vgic_dist_destroy(kvm);
 363
 364         kvm_for_each_vcpu(i, vcpu, kvm)
 365                 kvm_vgic_vcpu_destroy(vcpu);
 366 }
 367
 368 void kvm_vgic_destroy(struct kvm *kvm)
 369 {
 370         mutex_lock(&kvm->lock);
 371         __kvm_vgic_destroy(kvm);
 372         mutex_unlock(&kvm->lock);
 373 }
 374
 375 /**
 376  * vgic_lazy_init: Lazy init is only allowed if the GIC exposed to the guest
 377  * is a GICv2. A GICv3 must be explicitly initialized by the guest using the
 378  * KVM_DEV_ARM_VGIC_GRP_CTRL KVM_DEVICE group.
 379  * @kvm: kvm struct pointer
 380  */
 381 int vgic_lazy_init(struct kvm *kvm)
 382 {
 383         int ret = 0;
 384
 385         if (unlikely(!vgic_initialized(kvm))) {
 386                 /*
 387                  * We only provide the automatic initialization of the VGIC
 388                  * for the legacy case of a GICv2. Any other type must
 389                  * be explicitly initialized once setup with the respective
 390                  * KVM device call.
 391                  */
 392                 if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)
 393                         return -EBUSY;
 394
 395                 mutex_lock(&kvm->lock);
 396                 ret = vgic_init(kvm);
 397                 mutex_unlock(&kvm->lock);
 398         }
 399
 400         return ret;
 401 }
 402
 403 /* RESOURCE MAPPING */
 404
 405 /**
 406  * Map the MMIO regions depending on the VGIC model exposed to the guest
 407  * called on the first VCPU run.
 408  * Also map the virtual CPU interface into the VM.
 409  * v2/v3 derivatives call vgic_init if not already done.
 410  * vgic_ready() returns true if this function has succeeded.
 411  * @kvm: kvm struct pointer
 412  */
 413 int kvm_vgic_map_resources(struct kvm *kvm)
 414 {
 415         struct vgic_dist *dist = &kvm->arch.vgic;
 416         int ret = 0;
 417
 418         mutex_lock(&kvm->lock);
 419         if (!irqchip_in_kernel(kvm))
 420                 goto out;
 421
 422         if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2)
 423                 ret = vgic_v2_map_resources(kvm);
 424         else
 425                 ret = vgic_v3_map_resources(kvm);
 426
 427         if (ret)
 428                 __kvm_vgic_destroy(kvm);
 429
 430 out:
 431         mutex_unlock(&kvm->lock);
 432         return ret;
 433 }
 434
 435 /* GENERIC PROBE */
 436
 437 static int vgic_init_cpu_starting(unsigned int cpu)
 438 {
 439         enable_percpu_irq(kvm_vgic_global_state.maint_irq, 0);
 440         return 0;
 441 }
 442
 443
 444 static int vgic_init_cpu_dying(unsigned int cpu)
 445 {
 446         disable_percpu_irq(kvm_vgic_global_state.maint_irq);
 447         return 0;
 448 }
 449
 450 static irqreturn_t vgic_maintenance_handler(int irq, void *data)
 451 {
 452         /*
 453          * We cannot rely on the vgic maintenance interrupt to be
 454          * delivered synchronously. This means we can only use it to
 455          * exit the VM, and we perform the handling of EOIed
 456          * interrupts on the exit path (see vgic_fold_lr_state).
 457          */
 458         return IRQ_HANDLED;
 459 }
 460
 461 /**
 462  * kvm_vgic_init_cpu_hardware - initialize the GIC VE hardware
 463  *
 464  * For a specific CPU, initialize the GIC VE hardware.
 465  */
 466 void kvm_vgic_init_cpu_hardware(void)
 467 {
 468         BUG_ON(preemptible());
 469
 470         /*
 471          * We want to make sure the list registers start out clear so that we
 472          * only have the program the used registers.
 473          */
 474         if (kvm_vgic_global_state.type == VGIC_V2)
 475                 vgic_v2_init_lrs();
 476         else
 477                 kvm_call_hyp(__vgic_v3_init_lrs);
 478 }
 479
 480 /**
 481  * kvm_vgic_hyp_init: populates the kvm_vgic_global_state variable
 482  * according to the host GIC model. Accordingly calls either
 483  * vgic_v2/v3_probe which registers the KVM_DEVICE that can be
 484  * instantiated by a guest later on .
 485  */
 486 int kvm_vgic_hyp_init(void)
 487 {
 488         const struct gic_kvm_info *gic_kvm_info;
 489         int ret;
 490
 491         gic_kvm_info = gic_get_kvm_info();
 492         if (!gic_kvm_info)
 493                 return -ENODEV;
 494
 495         if (!gic_kvm_info->maint_irq) {
 496                 kvm_err("No vgic maintenance irq\n");
 497                 return -ENXIO;
 498         }
 499
 500         switch (gic_kvm_info->type) {
 501         case GIC_V2:
 502                 ret = vgic_v2_probe(gic_kvm_info);
 503                 break;
 504         case GIC_V3:
 505                 ret = vgic_v3_probe(gic_kvm_info);
 506                 if (!ret) {
 507                         static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
 508                         kvm_info("GIC system register CPU interface enabled\n");
 509                 }
 510                 break;
 511         default:
 512                 ret = -ENODEV;
 513         }
 514
 515         if (ret)
 516                 return ret;
 517
 518         kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq;
 519         ret = request_percpu_irq(kvm_vgic_global_state.maint_irq,
 520                                  vgic_maintenance_handler,
 521                                  "vgic", kvm_get_running_vcpus());
 522         if (ret) {
 523                 kvm_err("Cannot register interrupt %d\n",
 524                         kvm_vgic_global_state.maint_irq);
 525                 return ret;
 526         }
 527
 528         ret = cpuhp_setup_state(CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING,
 529                                 "kvm/arm/vgic:starting",
 530                                 vgic_init_cpu_starting, vgic_init_cpu_dying);
 531         if (ret) {
 532                 kvm_err("Cannot register vgic CPU notifier\n");
 533                 goto out_free_irq;
 534         }
 535
 536         kvm_info("vgic interrupt IRQ%d\n", kvm_vgic_global_state.maint_irq);
 537         return 0;
 538
 539 out_free_irq:
 540         free_percpu_irq(kvm_vgic_global_state.maint_irq,
 541                         kvm_get_running_vcpus());
 542         return ret;
 543 }