arch/arm64/kvm/vgic/vgic-its.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * GICv3 ITS emulation
   4  *
   5  * Copyright (C) 2015,2016 ARM Ltd.
   6  * Author: Andre Przywara <andre.przywara@arm.com>
   7  */
   8
   9 #include <linux/cpu.h>
  10 #include <linux/kvm.h>
  11 #include <linux/kvm_host.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/list.h>
  14 #include <linux/uaccess.h>
  15 #include <linux/list_sort.h>
  16
  17 #include <linux/irqchip/arm-gic-v3.h>
  18
  19 #include <asm/kvm_emulate.h>
  20 #include <asm/kvm_arm.h>
  21 #include <asm/kvm_mmu.h>
  22
  23 #include "vgic.h"
  24 #include "vgic-mmio.h"
  25
  26 static int vgic_its_save_tables_v0(struct vgic_its *its);
  27 static int vgic_its_restore_tables_v0(struct vgic_its *its);
  28 static int vgic_its_commit_v0(struct vgic_its *its);
  29 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
  30                              struct kvm_vcpu *filter_vcpu, bool needs_inv);
  31
  32 /*
  33  * Creates a new (reference to a) struct vgic_irq for a given LPI.
  34  * If this LPI is already mapped on another ITS, we increase its refcount
  35  * and return a pointer to the existing structure.
  36  * If this is a "new" LPI, we allocate and initialize a new struct vgic_irq.
  37  * This function returns a pointer to the _unlocked_ structure.
  38  */
  39 static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
  40                                      struct kvm_vcpu *vcpu)
  41 {
  42         struct vgic_dist *dist = &kvm->arch.vgic;
  43         struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq;
  44         unsigned long flags;
  45         int ret;
  46
  47         /* In this case there is no put, since we keep the reference. */
  48         if (irq)
  49                 return irq;
  50
  51         irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL_ACCOUNT);
  52         if (!irq)
  53                 return ERR_PTR(-ENOMEM);
  54
  55         INIT_LIST_HEAD(&irq->lpi_list);
  56         INIT_LIST_HEAD(&irq->ap_list);
  57         raw_spin_lock_init(&irq->irq_lock);
  58
  59         irq->config = VGIC_CONFIG_EDGE;
  60         kref_init(&irq->refcount);
  61         irq->intid = intid;
  62         irq->target_vcpu = vcpu;
  63         irq->group = 1;
  64
  65         raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
  66
  67         /*
  68          * There could be a race with another vgic_add_lpi(), so we need to
  69          * check that we don't add a second list entry with the same LPI.
  70          */
  71         list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) {
  72                 if (oldirq->intid != intid)
  73                         continue;
  74
  75                 /* Someone was faster with adding this LPI, lets use that. */
  76                 kfree(irq);
  77                 irq = oldirq;
  78
  79                 /*
  80                  * This increases the refcount, the caller is expected to
  81                  * call vgic_put_irq() on the returned pointer once it's
  82                  * finished with the IRQ.
  83                  */
  84                 vgic_get_irq_kref(irq);
  85
  86                 goto out_unlock;
  87         }
  88
  89         list_add_tail(&irq->lpi_list, &dist->lpi_list_head);
  90         dist->lpi_list_count++;
  91
  92 out_unlock:
  93         raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
  94
  95         /*
  96          * We "cache" the configuration table entries in our struct vgic_irq's.
  97          * However we only have those structs for mapped IRQs, so we read in
  98          * the respective config data from memory here upon mapping the LPI.
  99          *
 100          * Should any of these fail, behave as if we couldn't create the LPI
 101          * by dropping the refcount and returning the error.
 102          */
 103         ret = update_lpi_config(kvm, irq, NULL, false);
 104         if (ret) {
 105                 vgic_put_irq(kvm, irq);
 106                 return ERR_PTR(ret);
 107         }
 108
 109         ret = vgic_v3_lpi_sync_pending_status(kvm, irq);
 110         if (ret) {
 111                 vgic_put_irq(kvm, irq);
 112                 return ERR_PTR(ret);
 113         }
 114
 115         return irq;
 116 }
 117
 118 struct its_device {
 119         struct list_head dev_list;
 120
 121         /* the head for the list of ITTEs */
 122         struct list_head itt_head;
 123         u32 num_eventid_bits;
 124         gpa_t itt_addr;
 125         u32 device_id;
 126 };
 127
 128 #define COLLECTION_NOT_MAPPED ((u32)~0)
 129
 130 struct its_collection {
 131         struct list_head coll_list;
 132
 133         u32 collection_id;
 134         u32 target_addr;
 135 };
 136
 137 #define its_is_collection_mapped(coll) ((coll) && \
 138                                 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
 139
 140 struct its_ite {
 141         struct list_head ite_list;
 142
 143         struct vgic_irq *irq;
 144         struct its_collection *collection;
 145         u32 event_id;
 146 };
 147
 148 struct vgic_translation_cache_entry {
 149         struct list_head        entry;
 150         phys_addr_t             db;
 151         u32                     devid;
 152         u32                     eventid;
 153         struct vgic_irq         *irq;
 154 };
 155
 156 /**
 157  * struct vgic_its_abi - ITS abi ops and settings
 158  * @cte_esz: collection table entry size
 159  * @dte_esz: device table entry size
 160  * @ite_esz: interrupt translation table entry size
 161  * @save tables: save the ITS tables into guest RAM
 162  * @restore_tables: restore the ITS internal structs from tables
 163  *  stored in guest RAM
 164  * @commit: initialize the registers which expose the ABI settings,
 165  *  especially the entry sizes
 166  */
 167 struct vgic_its_abi {
 168         int cte_esz;
 169         int dte_esz;
 170         int ite_esz;
 171         int (*save_tables)(struct vgic_its *its);
 172         int (*restore_tables)(struct vgic_its *its);
 173         int (*commit)(struct vgic_its *its);
 174 };
 175
 176 #define ABI_0_ESZ       8
 177 #define ESZ_MAX         ABI_0_ESZ
 178
 179 static const struct vgic_its_abi its_table_abi_versions[] = {
 180         [0] = {
 181          .cte_esz = ABI_0_ESZ,
 182          .dte_esz = ABI_0_ESZ,
 183          .ite_esz = ABI_0_ESZ,
 184          .save_tables = vgic_its_save_tables_v0,
 185          .restore_tables = vgic_its_restore_tables_v0,
 186          .commit = vgic_its_commit_v0,
 187         },
 188 };
 189
 190 #define NR_ITS_ABIS     ARRAY_SIZE(its_table_abi_versions)
 191
 192 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
 193 {
 194         return &its_table_abi_versions[its->abi_rev];
 195 }
 196
 197 static int vgic_its_set_abi(struct vgic_its *its, u32 rev)
 198 {
 199         const struct vgic_its_abi *abi;
 200
 201         its->abi_rev = rev;
 202         abi = vgic_its_get_abi(its);
 203         return abi->commit(its);
 204 }
 205
 206 /*
 207  * Find and returns a device in the device table for an ITS.
 208  * Must be called with the its_lock mutex held.
 209  */
 210 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
 211 {
 212         struct its_device *device;
 213
 214         list_for_each_entry(device, &its->device_list, dev_list)
 215                 if (device_id == device->device_id)
 216                         return device;
 217
 218         return NULL;
 219 }
 220
 221 /*
 222  * Find and returns an interrupt translation table entry (ITTE) for a given
 223  * Device ID/Event ID pair on an ITS.
 224  * Must be called with the its_lock mutex held.
 225  */
 226 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
 227                                   u32 event_id)
 228 {
 229         struct its_device *device;
 230         struct its_ite *ite;
 231
 232         device = find_its_device(its, device_id);
 233         if (device == NULL)
 234                 return NULL;
 235
 236         list_for_each_entry(ite, &device->itt_head, ite_list)
 237                 if (ite->event_id == event_id)
 238                         return ite;
 239
 240         return NULL;
 241 }
 242
 243 /* To be used as an iterator this macro misses the enclosing parentheses */
 244 #define for_each_lpi_its(dev, ite, its) \
 245         list_for_each_entry(dev, &(its)->device_list, dev_list) \
 246                 list_for_each_entry(ite, &(dev)->itt_head, ite_list)
 247
 248 #define GIC_LPI_OFFSET 8192
 249
 250 #define VITS_TYPER_IDBITS 16
 251 #define VITS_TYPER_DEVBITS 16
 252 #define VITS_DTE_MAX_DEVID_OFFSET       (BIT(14) - 1)
 253 #define VITS_ITE_MAX_EVENTID_OFFSET     (BIT(16) - 1)
 254
 255 /*
 256  * Finds and returns a collection in the ITS collection table.
 257  * Must be called with the its_lock mutex held.
 258  */
 259 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
 260 {
 261         struct its_collection *collection;
 262
 263         list_for_each_entry(collection, &its->collection_list, coll_list) {
 264                 if (coll_id == collection->collection_id)
 265                         return collection;
 266         }
 267
 268         return NULL;
 269 }
 270
 271 #define LPI_PROP_ENABLE_BIT(p)  ((p) & LPI_PROP_ENABLED)
 272 #define LPI_PROP_PRIORITY(p)    ((p) & 0xfc)
 273
 274 /*
 275  * Reads the configuration data for a given LPI from guest memory and
 276  * updates the fields in struct vgic_irq.
 277  * If filter_vcpu is not NULL, applies only if the IRQ is targeting this
 278  * VCPU. Unconditionally applies if filter_vcpu is NULL.
 279  */
 280 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
 281                              struct kvm_vcpu *filter_vcpu, bool needs_inv)
 282 {
 283         u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
 284         u8 prop;
 285         int ret;
 286         unsigned long flags;
 287
 288         ret = kvm_read_guest_lock(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
 289                                   &prop, 1);
 290
 291         if (ret)
 292                 return ret;
 293
 294         raw_spin_lock_irqsave(&irq->irq_lock, flags);
 295
 296         if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
 297                 irq->priority = LPI_PROP_PRIORITY(prop);
 298                 irq->enabled = LPI_PROP_ENABLE_BIT(prop);
 299
 300                 if (!irq->hw) {
 301                         vgic_queue_irq_unlock(kvm, irq, flags);
 302                         return 0;
 303                 }
 304         }
 305
 306         raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 307
 308         if (irq->hw)
 309                 return its_prop_update_vlpi(irq->host_irq, prop, needs_inv);
 310
 311         return 0;
 312 }
 313
 314 /*
 315  * Create a snapshot of the current LPIs targeting @vcpu, so that we can
 316  * enumerate those LPIs without holding any lock.
 317  * Returns their number and puts the kmalloc'ed array into intid_ptr.
 318  */
 319 int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr)
 320 {
 321         struct vgic_dist *dist = &kvm->arch.vgic;
 322         struct vgic_irq *irq;
 323         unsigned long flags;
 324         u32 *intids;
 325         int irq_count, i = 0;
 326
 327         /*
 328          * There is an obvious race between allocating the array and LPIs
 329          * being mapped/unmapped. If we ended up here as a result of a
 330          * command, we're safe (locks are held, preventing another
 331          * command). If coming from another path (such as enabling LPIs),
 332          * we must be careful not to overrun the array.
 333          */
 334         irq_count = READ_ONCE(dist->lpi_list_count);
 335         intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL_ACCOUNT);
 336         if (!intids)
 337                 return -ENOMEM;
 338
 339         raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
 340         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
 341                 if (i == irq_count)
 342                         break;
 343                 /* We don't need to "get" the IRQ, as we hold the list lock. */
 344                 if (vcpu && irq->target_vcpu != vcpu)
 345                         continue;
 346                 intids[i++] = irq->intid;
 347         }
 348         raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
 349
 350         *intid_ptr = intids;
 351         return i;
 352 }
 353
 354 static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu)
 355 {
 356         int ret = 0;
 357         unsigned long flags;
 358
 359         raw_spin_lock_irqsave(&irq->irq_lock, flags);
 360         irq->target_vcpu = vcpu;
 361         raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 362
 363         if (irq->hw) {
 364                 struct its_vlpi_map map;
 365
 366                 ret = its_get_vlpi(irq->host_irq, &map);
 367                 if (ret)
 368                         return ret;
 369
 370                 if (map.vpe)
 371                         atomic_dec(&map.vpe->vlpi_count);
 372                 map.vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
 373                 atomic_inc(&map.vpe->vlpi_count);
 374
 375                 ret = its_map_vlpi(irq->host_irq, &map);
 376         }
 377
 378         return ret;
 379 }
 380
 381 /*
 382  * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
 383  * is targeting) to the VGIC's view, which deals with target VCPUs.
 384  * Needs to be called whenever either the collection for a LPIs has
 385  * changed or the collection itself got retargeted.
 386  */
 387 static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
 388 {
 389         struct kvm_vcpu *vcpu;
 390
 391         if (!its_is_collection_mapped(ite->collection))
 392                 return;
 393
 394         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
 395         update_affinity(ite->irq, vcpu);
 396 }
 397
 398 /*
 399  * Updates the target VCPU for every LPI targeting this collection.
 400  * Must be called with the its_lock mutex held.
 401  */
 402 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
 403                                        struct its_collection *coll)
 404 {
 405         struct its_device *device;
 406         struct its_ite *ite;
 407
 408         for_each_lpi_its(device, ite, its) {
 409                 if (!ite->collection || coll != ite->collection)
 410                         continue;
 411
 412                 update_affinity_ite(kvm, ite);
 413         }
 414 }
 415
 416 static u32 max_lpis_propbaser(u64 propbaser)
 417 {
 418         int nr_idbits = (propbaser & 0x1f) + 1;
 419
 420         return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
 421 }
 422
 423 /*
 424  * Sync the pending table pending bit of LPIs targeting @vcpu
 425  * with our own data structures. This relies on the LPI being
 426  * mapped before.
 427  */
 428 static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
 429 {
 430         gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
 431         struct vgic_irq *irq;
 432         int last_byte_offset = -1;
 433         int ret = 0;
 434         u32 *intids;
 435         int nr_irqs, i;
 436         unsigned long flags;
 437         u8 pendmask;
 438
 439         nr_irqs = vgic_copy_lpi_list(vcpu->kvm, vcpu, &intids);
 440         if (nr_irqs < 0)
 441                 return nr_irqs;
 442
 443         for (i = 0; i < nr_irqs; i++) {
 444                 int byte_offset, bit_nr;
 445
 446                 byte_offset = intids[i] / BITS_PER_BYTE;
 447                 bit_nr = intids[i] % BITS_PER_BYTE;
 448
 449                 /*
 450                  * For contiguously allocated LPIs chances are we just read
 451                  * this very same byte in the last iteration. Reuse that.
 452                  */
 453                 if (byte_offset != last_byte_offset) {
 454                         ret = kvm_read_guest_lock(vcpu->kvm,
 455                                                   pendbase + byte_offset,
 456                                                   &pendmask, 1);
 457                         if (ret) {
 458                                 kfree(intids);
 459                                 return ret;
 460                         }
 461                         last_byte_offset = byte_offset;
 462                 }
 463
 464                 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);
 465                 raw_spin_lock_irqsave(&irq->irq_lock, flags);
 466                 irq->pending_latch = pendmask & (1U << bit_nr);
 467                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 468                 vgic_put_irq(vcpu->kvm, irq);
 469         }
 470
 471         kfree(intids);
 472
 473         return ret;
 474 }
 475
 476 static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
 477                                               struct vgic_its *its,
 478                                               gpa_t addr, unsigned int len)
 479 {
 480         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
 481         u64 reg = GITS_TYPER_PLPIS;
 482
 483         /*
 484          * We use linear CPU numbers for redistributor addressing,
 485          * so GITS_TYPER.PTA is 0.
 486          * Also we force all PROPBASER registers to be the same, so
 487          * CommonLPIAff is 0 as well.
 488          * To avoid memory waste in the guest, we keep the number of IDBits and
 489          * DevBits low - as least for the time being.
 490          */
 491         reg |= GIC_ENCODE_SZ(VITS_TYPER_DEVBITS, 5) << GITS_TYPER_DEVBITS_SHIFT;
 492         reg |= GIC_ENCODE_SZ(VITS_TYPER_IDBITS, 5) << GITS_TYPER_IDBITS_SHIFT;
 493         reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT;
 494
 495         return extract_bytes(reg, addr & 7, len);
 496 }
 497
 498 static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
 499                                              struct vgic_its *its,
 500                                              gpa_t addr, unsigned int len)
 501 {
 502         u32 val;
 503
 504         val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
 505         val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM;
 506         return val;
 507 }
 508
 509 static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
 510                                             struct vgic_its *its,
 511                                             gpa_t addr, unsigned int len,
 512                                             unsigned long val)
 513 {
 514         u32 rev = GITS_IIDR_REV(val);
 515
 516         if (rev >= NR_ITS_ABIS)
 517                 return -EINVAL;
 518         return vgic_its_set_abi(its, rev);
 519 }
 520
 521 static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
 522                                                struct vgic_its *its,
 523                                                gpa_t addr, unsigned int len)
 524 {
 525         switch (addr & 0xffff) {
 526         case GITS_PIDR0:
 527                 return 0x92;    /* part number, bits[7:0] */
 528         case GITS_PIDR1:
 529                 return 0xb4;    /* part number, bits[11:8] */
 530         case GITS_PIDR2:
 531                 return GIC_PIDR2_ARCH_GICv3 | 0x0b;
 532         case GITS_PIDR4:
 533                 return 0x40;    /* This is a 64K software visible page */
 534         /* The following are the ID registers for (any) GIC. */
 535         case GITS_CIDR0:
 536                 return 0x0d;
 537         case GITS_CIDR1:
 538                 return 0xf0;
 539         case GITS_CIDR2:
 540                 return 0x05;
 541         case GITS_CIDR3:
 542                 return 0xb1;
 543         }
 544
 545         return 0;
 546 }
 547
 548 static struct vgic_irq *__vgic_its_check_cache(struct vgic_dist *dist,
 549                                                phys_addr_t db,
 550                                                u32 devid, u32 eventid)
 551 {
 552         struct vgic_translation_cache_entry *cte;
 553
 554         list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
 555                 /*
 556                  * If we hit a NULL entry, there is nothing after this
 557                  * point.
 558                  */
 559                 if (!cte->irq)
 560                         break;
 561
 562                 if (cte->db != db || cte->devid != devid ||
 563                     cte->eventid != eventid)
 564                         continue;
 565
 566                 /*
 567                  * Move this entry to the head, as it is the most
 568                  * recently used.
 569                  */
 570                 if (!list_is_first(&cte->entry, &dist->lpi_translation_cache))
 571                         list_move(&cte->entry, &dist->lpi_translation_cache);
 572
 573                 return cte->irq;
 574         }
 575
 576         return NULL;
 577 }
 578
 579 static struct vgic_irq *vgic_its_check_cache(struct kvm *kvm, phys_addr_t db,
 580                                              u32 devid, u32 eventid)
 581 {
 582         struct vgic_dist *dist = &kvm->arch.vgic;
 583         struct vgic_irq *irq;
 584         unsigned long flags;
 585
 586         raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
 587         irq = __vgic_its_check_cache(dist, db, devid, eventid);
 588         raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
 589
 590         return irq;
 591 }
 592
 593 static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,
 594                                        u32 devid, u32 eventid,
 595                                        struct vgic_irq *irq)
 596 {
 597         struct vgic_dist *dist = &kvm->arch.vgic;
 598         struct vgic_translation_cache_entry *cte;
 599         unsigned long flags;
 600         phys_addr_t db;
 601
 602         /* Do not cache a directly injected interrupt */
 603         if (irq->hw)
 604                 return;
 605
 606         raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
 607
 608         if (unlikely(list_empty(&dist->lpi_translation_cache)))
 609                 goto out;
 610
 611         /*
 612          * We could have raced with another CPU caching the same
 613          * translation behind our back, so let's check it is not in
 614          * already
 615          */
 616         db = its->vgic_its_base + GITS_TRANSLATER;
 617         if (__vgic_its_check_cache(dist, db, devid, eventid))
 618                 goto out;
 619
 620         /* Always reuse the last entry (LRU policy) */
 621         cte = list_last_entry(&dist->lpi_translation_cache,
 622                               typeof(*cte), entry);
 623
 624         /*
 625          * Caching the translation implies having an extra reference
 626          * to the interrupt, so drop the potential reference on what
 627          * was in the cache, and increment it on the new interrupt.
 628          */
 629         if (cte->irq)
 630                 __vgic_put_lpi_locked(kvm, cte->irq);
 631
 632         vgic_get_irq_kref(irq);
 633
 634         cte->db         = db;
 635         cte->devid      = devid;
 636         cte->eventid    = eventid;
 637         cte->irq        = irq;
 638
 639         /* Move the new translation to the head of the list */
 640         list_move(&cte->entry, &dist->lpi_translation_cache);
 641
 642 out:
 643         raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
 644 }
 645
 646 void vgic_its_invalidate_cache(struct kvm *kvm)
 647 {
 648         struct vgic_dist *dist = &kvm->arch.vgic;
 649         struct vgic_translation_cache_entry *cte;
 650         unsigned long flags;
 651
 652         raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
 653
 654         list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
 655                 /*
 656                  * If we hit a NULL entry, there is nothing after this
 657                  * point.
 658                  */
 659                 if (!cte->irq)
 660                         break;
 661
 662                 __vgic_put_lpi_locked(kvm, cte->irq);
 663                 cte->irq = NULL;
 664         }
 665
 666         raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
 667 }
 668
 669 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
 670                          u32 devid, u32 eventid, struct vgic_irq **irq)
 671 {
 672         struct kvm_vcpu *vcpu;
 673         struct its_ite *ite;
 674
 675         if (!its->enabled)
 676                 return -EBUSY;
 677
 678         ite = find_ite(its, devid, eventid);
 679         if (!ite || !its_is_collection_mapped(ite->collection))
 680                 return E_ITS_INT_UNMAPPED_INTERRUPT;
 681
 682         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
 683         if (!vcpu)
 684                 return E_ITS_INT_UNMAPPED_INTERRUPT;
 685
 686         if (!vcpu->arch.vgic_cpu.lpis_enabled)
 687                 return -EBUSY;
 688
 689         vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
 690
 691         *irq = ite->irq;
 692         return 0;
 693 }
 694
 695 struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi)
 696 {
 697         u64 address;
 698         struct kvm_io_device *kvm_io_dev;
 699         struct vgic_io_device *iodev;
 700
 701         if (!vgic_has_its(kvm))
 702                 return ERR_PTR(-ENODEV);
 703
 704         if (!(msi->flags & KVM_MSI_VALID_DEVID))
 705                 return ERR_PTR(-EINVAL);
 706
 707         address = (u64)msi->address_hi << 32 | msi->address_lo;
 708
 709         kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address);
 710         if (!kvm_io_dev)
 711                 return ERR_PTR(-EINVAL);
 712
 713         if (kvm_io_dev->ops != &kvm_io_gic_ops)
 714                 return ERR_PTR(-EINVAL);
 715
 716         iodev = container_of(kvm_io_dev, struct vgic_io_device, dev);
 717         if (iodev->iodev_type != IODEV_ITS)
 718                 return ERR_PTR(-EINVAL);
 719
 720         return iodev->its;
 721 }
 722
 723 /*
 724  * Find the target VCPU and the LPI number for a given devid/eventid pair
 725  * and make this IRQ pending, possibly injecting it.
 726  * Must be called with the its_lock mutex held.
 727  * Returns 0 on success, a positive error value for any ITS mapping
 728  * related errors and negative error values for generic errors.
 729  */
 730 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
 731                                 u32 devid, u32 eventid)
 732 {
 733         struct vgic_irq *irq = NULL;
 734         unsigned long flags;
 735         int err;
 736
 737         err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);
 738         if (err)
 739                 return err;
 740
 741         if (irq->hw)
 742                 return irq_set_irqchip_state(irq->host_irq,
 743                                              IRQCHIP_STATE_PENDING, true);
 744
 745         raw_spin_lock_irqsave(&irq->irq_lock, flags);
 746         irq->pending_latch = true;
 747         vgic_queue_irq_unlock(kvm, irq, flags);
 748
 749         return 0;
 750 }
 751
 752 int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi)
 753 {
 754         struct vgic_irq *irq;
 755         unsigned long flags;
 756         phys_addr_t db;
 757
 758         db = (u64)msi->address_hi << 32 | msi->address_lo;
 759         irq = vgic_its_check_cache(kvm, db, msi->devid, msi->data);
 760         if (!irq)
 761                 return -EWOULDBLOCK;
 762
 763         raw_spin_lock_irqsave(&irq->irq_lock, flags);
 764         irq->pending_latch = true;
 765         vgic_queue_irq_unlock(kvm, irq, flags);
 766
 767         return 0;
 768 }
 769
 770 /*
 771  * Queries the KVM IO bus framework to get the ITS pointer from the given
 772  * doorbell address.
 773  * We then call vgic_its_trigger_msi() with the decoded data.
 774  * According to the KVM_SIGNAL_MSI API description returns 1 on success.
 775  */
 776 int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
 777 {
 778         struct vgic_its *its;
 779         int ret;
 780
 781         if (!vgic_its_inject_cached_translation(kvm, msi))
 782                 return 1;
 783
 784         its = vgic_msi_to_its(kvm, msi);
 785         if (IS_ERR(its))
 786                 return PTR_ERR(its);
 787
 788         mutex_lock(&its->its_lock);
 789         ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);
 790         mutex_unlock(&its->its_lock);
 791
 792         if (ret < 0)
 793                 return ret;
 794
 795         /*
 796          * KVM_SIGNAL_MSI demands a return value > 0 for success and 0
 797          * if the guest has blocked the MSI. So we map any LPI mapping
 798          * related error to that.
 799          */
 800         if (ret)
 801                 return 0;
 802         else
 803                 return 1;
 804 }
 805
 806 /* Requires the its_lock to be held. */
 807 static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
 808 {
 809         list_del(&ite->ite_list);
 810
 811         /* This put matches the get in vgic_add_lpi. */
 812         if (ite->irq) {
 813                 if (ite->irq->hw)
 814                         WARN_ON(its_unmap_vlpi(ite->irq->host_irq));
 815
 816                 vgic_put_irq(kvm, ite->irq);
 817         }
 818
 819         kfree(ite);
 820 }
 821
 822 static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
 823 {
 824         return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
 825 }
 826
 827 #define its_cmd_get_command(cmd)        its_cmd_mask_field(cmd, 0,  0,  8)
 828 #define its_cmd_get_deviceid(cmd)       its_cmd_mask_field(cmd, 0, 32, 32)
 829 #define its_cmd_get_size(cmd)           (its_cmd_mask_field(cmd, 1,  0,  5) + 1)
 830 #define its_cmd_get_id(cmd)             its_cmd_mask_field(cmd, 1,  0, 32)
 831 #define its_cmd_get_physical_id(cmd)    its_cmd_mask_field(cmd, 1, 32, 32)
 832 #define its_cmd_get_collection(cmd)     its_cmd_mask_field(cmd, 2,  0, 16)
 833 #define its_cmd_get_ittaddr(cmd)        (its_cmd_mask_field(cmd, 2,  8, 44) << 8)
 834 #define its_cmd_get_target_addr(cmd)    its_cmd_mask_field(cmd, 2, 16, 32)
 835 #define its_cmd_get_validbit(cmd)       its_cmd_mask_field(cmd, 2, 63,  1)
 836
 837 /*
 838  * The DISCARD command frees an Interrupt Translation Table Entry (ITTE).
 839  * Must be called with the its_lock mutex held.
 840  */
 841 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
 842                                        u64 *its_cmd)
 843 {
 844         u32 device_id = its_cmd_get_deviceid(its_cmd);
 845         u32 event_id = its_cmd_get_id(its_cmd);
 846         struct its_ite *ite;
 847
 848         ite = find_ite(its, device_id, event_id);
 849         if (ite && its_is_collection_mapped(ite->collection)) {
 850                 /*
 851                  * Though the spec talks about removing the pending state, we
 852                  * don't bother here since we clear the ITTE anyway and the
 853                  * pending state is a property of the ITTE struct.
 854                  */
 855                 vgic_its_invalidate_cache(kvm);
 856
 857                 its_free_ite(kvm, ite);
 858                 return 0;
 859         }
 860
 861         return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
 862 }
 863
 864 /*
 865  * The MOVI command moves an ITTE to a different collection.
 866  * Must be called with the its_lock mutex held.
 867  */
 868 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
 869                                     u64 *its_cmd)
 870 {
 871         u32 device_id = its_cmd_get_deviceid(its_cmd);
 872         u32 event_id = its_cmd_get_id(its_cmd);
 873         u32 coll_id = its_cmd_get_collection(its_cmd);
 874         struct kvm_vcpu *vcpu;
 875         struct its_ite *ite;
 876         struct its_collection *collection;
 877
 878         ite = find_ite(its, device_id, event_id);
 879         if (!ite)
 880                 return E_ITS_MOVI_UNMAPPED_INTERRUPT;
 881
 882         if (!its_is_collection_mapped(ite->collection))
 883                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
 884
 885         collection = find_collection(its, coll_id);
 886         if (!its_is_collection_mapped(collection))
 887                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
 888
 889         ite->collection = collection;
 890         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
 891
 892         vgic_its_invalidate_cache(kvm);
 893
 894         return update_affinity(ite->irq, vcpu);
 895 }
 896
 897 /*
 898  * Check whether an ID can be stored into the corresponding guest table.
 899  * For a direct table this is pretty easy, but gets a bit nasty for
 900  * indirect tables. We check whether the resulting guest physical address
 901  * is actually valid (covered by a memslot and guest accessible).
 902  * For this we have to read the respective first level entry.
 903  */
 904 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
 905                               gpa_t *eaddr)
 906 {
 907         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
 908         u64 indirect_ptr, type = GITS_BASER_TYPE(baser);
 909         phys_addr_t base = GITS_BASER_ADDR_48_to_52(baser);
 910         int esz = GITS_BASER_ENTRY_SIZE(baser);
 911         int index, idx;
 912         gfn_t gfn;
 913         bool ret;
 914
 915         switch (type) {
 916         case GITS_BASER_TYPE_DEVICE:
 917                 if (id >= BIT_ULL(VITS_TYPER_DEVBITS))
 918                         return false;
 919                 break;
 920         case GITS_BASER_TYPE_COLLECTION:
 921                 /* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */
 922                 if (id >= BIT_ULL(16))
 923                         return false;
 924                 break;
 925         default:
 926                 return false;
 927         }
 928
 929         if (!(baser & GITS_BASER_INDIRECT)) {
 930                 phys_addr_t addr;
 931
 932                 if (id >= (l1_tbl_size / esz))
 933                         return false;
 934
 935                 addr = base + id * esz;
 936                 gfn = addr >> PAGE_SHIFT;
 937
 938                 if (eaddr)
 939                         *eaddr = addr;
 940
 941                 goto out;
 942         }
 943
 944         /* calculate and check the index into the 1st level */
 945         index = id / (SZ_64K / esz);
 946         if (index >= (l1_tbl_size / sizeof(u64)))
 947                 return false;
 948
 949         /* Each 1st level entry is represented by a 64-bit value. */
 950         if (kvm_read_guest_lock(its->dev->kvm,
 951                            base + index * sizeof(indirect_ptr),
 952                            &indirect_ptr, sizeof(indirect_ptr)))
 953                 return false;
 954
 955         indirect_ptr = le64_to_cpu(indirect_ptr);
 956
 957         /* check the valid bit of the first level entry */
 958         if (!(indirect_ptr & BIT_ULL(63)))
 959                 return false;
 960
 961         /* Mask the guest physical address and calculate the frame number. */
 962         indirect_ptr &= GENMASK_ULL(51, 16);
 963
 964         /* Find the address of the actual entry */
 965         index = id % (SZ_64K / esz);
 966         indirect_ptr += index * esz;
 967         gfn = indirect_ptr >> PAGE_SHIFT;
 968
 969         if (eaddr)
 970                 *eaddr = indirect_ptr;
 971
 972 out:
 973         idx = srcu_read_lock(&its->dev->kvm->srcu);
 974         ret = kvm_is_visible_gfn(its->dev->kvm, gfn);
 975         srcu_read_unlock(&its->dev->kvm->srcu, idx);
 976         return ret;
 977 }
 978
 979 static int vgic_its_alloc_collection(struct vgic_its *its,
 980                                      struct its_collection **colp,
 981                                      u32 coll_id)
 982 {
 983         struct its_collection *collection;
 984
 985         if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
 986                 return E_ITS_MAPC_COLLECTION_OOR;
 987
 988         collection = kzalloc(sizeof(*collection), GFP_KERNEL_ACCOUNT);
 989         if (!collection)
 990                 return -ENOMEM;
 991
 992         collection->collection_id = coll_id;
 993         collection->target_addr = COLLECTION_NOT_MAPPED;
 994
 995         list_add_tail(&collection->coll_list, &its->collection_list);
 996         *colp = collection;
 997
 998         return 0;
 999 }
1000
1001 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
1002 {
1003         struct its_collection *collection;
1004         struct its_device *device;
1005         struct its_ite *ite;
1006
1007         /*
1008          * Clearing the mapping for that collection ID removes the
1009          * entry from the list. If there wasn't any before, we can
1010          * go home early.
1011          */
1012         collection = find_collection(its, coll_id);
1013         if (!collection)
1014                 return;
1015
1016         for_each_lpi_its(device, ite, its)
1017                 if (ite->collection &&
1018                     ite->collection->collection_id == coll_id)
1019                         ite->collection = NULL;
1020
1021         list_del(&collection->coll_list);
1022         kfree(collection);
1023 }
1024
1025 /* Must be called with its_lock mutex held */
1026 static struct its_ite *vgic_its_alloc_ite(struct its_device *device,
1027                                           struct its_collection *collection,
1028                                           u32 event_id)
1029 {
1030         struct its_ite *ite;
1031
1032         ite = kzalloc(sizeof(*ite), GFP_KERNEL_ACCOUNT);
1033         if (!ite)
1034                 return ERR_PTR(-ENOMEM);
1035
1036         ite->event_id   = event_id;
1037         ite->collection = collection;
1038
1039         list_add_tail(&ite->ite_list, &device->itt_head);
1040         return ite;
1041 }
1042
1043 /*
1044  * The MAPTI and MAPI commands map LPIs to ITTEs.
1045  * Must be called with its_lock mutex held.
1046  */
1047 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
1048                                     u64 *its_cmd)
1049 {
1050         u32 device_id = its_cmd_get_deviceid(its_cmd);
1051         u32 event_id = its_cmd_get_id(its_cmd);
1052         u32 coll_id = its_cmd_get_collection(its_cmd);
1053         struct its_ite *ite;
1054         struct kvm_vcpu *vcpu = NULL;
1055         struct its_device *device;
1056         struct its_collection *collection, *new_coll = NULL;
1057         struct vgic_irq *irq;
1058         int lpi_nr;
1059
1060         device = find_its_device(its, device_id);
1061         if (!device)
1062                 return E_ITS_MAPTI_UNMAPPED_DEVICE;
1063
1064         if (event_id >= BIT_ULL(device->num_eventid_bits))
1065                 return E_ITS_MAPTI_ID_OOR;
1066
1067         if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
1068                 lpi_nr = its_cmd_get_physical_id(its_cmd);
1069         else
1070                 lpi_nr = event_id;
1071         if (lpi_nr < GIC_LPI_OFFSET ||
1072             lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
1073                 return E_ITS_MAPTI_PHYSICALID_OOR;
1074
1075         /* If there is an existing mapping, behavior is UNPREDICTABLE. */
1076         if (find_ite(its, device_id, event_id))
1077                 return 0;
1078
1079         collection = find_collection(its, coll_id);
1080         if (!collection) {
1081                 int ret = vgic_its_alloc_collection(its, &collection, coll_id);
1082                 if (ret)
1083                         return ret;
1084                 new_coll = collection;
1085         }
1086
1087         ite = vgic_its_alloc_ite(device, collection, event_id);
1088         if (IS_ERR(ite)) {
1089                 if (new_coll)
1090                         vgic_its_free_collection(its, coll_id);
1091                 return PTR_ERR(ite);
1092         }
1093
1094         if (its_is_collection_mapped(collection))
1095                 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1096
1097         irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
1098         if (IS_ERR(irq)) {
1099                 if (new_coll)
1100                         vgic_its_free_collection(its, coll_id);
1101                 its_free_ite(kvm, ite);
1102                 return PTR_ERR(irq);
1103         }
1104         ite->irq = irq;
1105
1106         return 0;
1107 }
1108
1109 /* Requires the its_lock to be held. */
1110 static void vgic_its_free_device(struct kvm *kvm, struct its_device *device)
1111 {
1112         struct its_ite *ite, *temp;
1113
1114         /*
1115          * The spec says that unmapping a device with still valid
1116          * ITTEs associated is UNPREDICTABLE. We remove all ITTEs,
1117          * since we cannot leave the memory unreferenced.
1118          */
1119         list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
1120                 its_free_ite(kvm, ite);
1121
1122         vgic_its_invalidate_cache(kvm);
1123
1124         list_del(&device->dev_list);
1125         kfree(device);
1126 }
1127
1128 /* its lock must be held */
1129 static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
1130 {
1131         struct its_device *cur, *temp;
1132
1133         list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
1134                 vgic_its_free_device(kvm, cur);
1135 }
1136
1137 /* its lock must be held */
1138 static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
1139 {
1140         struct its_collection *cur, *temp;
1141
1142         list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
1143                 vgic_its_free_collection(its, cur->collection_id);
1144 }
1145
1146 /* Must be called with its_lock mutex held */
1147 static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
1148                                                 u32 device_id, gpa_t itt_addr,
1149                                                 u8 num_eventid_bits)
1150 {
1151         struct its_device *device;
1152
1153         device = kzalloc(sizeof(*device), GFP_KERNEL_ACCOUNT);
1154         if (!device)
1155                 return ERR_PTR(-ENOMEM);
1156
1157         device->device_id = device_id;
1158         device->itt_addr = itt_addr;
1159         device->num_eventid_bits = num_eventid_bits;
1160         INIT_LIST_HEAD(&device->itt_head);
1161
1162         list_add_tail(&device->dev_list, &its->device_list);
1163         return device;
1164 }
1165
1166 /*
1167  * MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs).
1168  * Must be called with the its_lock mutex held.
1169  */
1170 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
1171                                     u64 *its_cmd)
1172 {
1173         u32 device_id = its_cmd_get_deviceid(its_cmd);
1174         bool valid = its_cmd_get_validbit(its_cmd);
1175         u8 num_eventid_bits = its_cmd_get_size(its_cmd);
1176         gpa_t itt_addr = its_cmd_get_ittaddr(its_cmd);
1177         struct its_device *device;
1178
1179         if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))
1180                 return E_ITS_MAPD_DEVICE_OOR;
1181
1182         if (valid && num_eventid_bits > VITS_TYPER_IDBITS)
1183                 return E_ITS_MAPD_ITTSIZE_OOR;
1184
1185         device = find_its_device(its, device_id);
1186
1187         /*
1188          * The spec says that calling MAPD on an already mapped device
1189          * invalidates all cached data for this device. We implement this
1190          * by removing the mapping and re-establishing it.
1191          */
1192         if (device)
1193                 vgic_its_free_device(kvm, device);
1194
1195         /*
1196          * The spec does not say whether unmapping a not-mapped device
1197          * is an error, so we are done in any case.
1198          */
1199         if (!valid)
1200                 return 0;
1201
1202         device = vgic_its_alloc_device(its, device_id, itt_addr,
1203                                        num_eventid_bits);
1204
1205         return PTR_ERR_OR_ZERO(device);
1206 }
1207
1208 /*
1209  * The MAPC command maps collection IDs to redistributors.
1210  * Must be called with the its_lock mutex held.
1211  */
1212 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1213                                     u64 *its_cmd)
1214 {
1215         u16 coll_id;
1216         u32 target_addr;
1217         struct its_collection *collection;
1218         bool valid;
1219
1220         valid = its_cmd_get_validbit(its_cmd);
1221         coll_id = its_cmd_get_collection(its_cmd);
1222         target_addr = its_cmd_get_target_addr(its_cmd);
1223
1224         if (target_addr >= atomic_read(&kvm->online_vcpus))
1225                 return E_ITS_MAPC_PROCNUM_OOR;
1226
1227         if (!valid) {
1228                 vgic_its_free_collection(its, coll_id);
1229                 vgic_its_invalidate_cache(kvm);
1230         } else {
1231                 collection = find_collection(its, coll_id);
1232
1233                 if (!collection) {
1234                         int ret;
1235
1236                         ret = vgic_its_alloc_collection(its, &collection,
1237                                                         coll_id);
1238                         if (ret)
1239                                 return ret;
1240                         collection->target_addr = target_addr;
1241                 } else {
1242                         collection->target_addr = target_addr;
1243                         update_affinity_collection(kvm, its, collection);
1244                 }
1245         }
1246
1247         return 0;
1248 }
1249
1250 /*
1251  * The CLEAR command removes the pending state for a particular LPI.
1252  * Must be called with the its_lock mutex held.
1253  */
1254 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1255                                      u64 *its_cmd)
1256 {
1257         u32 device_id = its_cmd_get_deviceid(its_cmd);
1258         u32 event_id = its_cmd_get_id(its_cmd);
1259         struct its_ite *ite;
1260
1261
1262         ite = find_ite(its, device_id, event_id);
1263         if (!ite)
1264                 return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
1265
1266         ite->irq->pending_latch = false;
1267
1268         if (ite->irq->hw)
1269                 return irq_set_irqchip_state(ite->irq->host_irq,
1270                                              IRQCHIP_STATE_PENDING, false);
1271
1272         return 0;
1273 }
1274
1275 /*
1276  * The INV command syncs the configuration bits from the memory table.
1277  * Must be called with the its_lock mutex held.
1278  */
1279 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1280                                    u64 *its_cmd)
1281 {
1282         u32 device_id = its_cmd_get_deviceid(its_cmd);
1283         u32 event_id = its_cmd_get_id(its_cmd);
1284         struct its_ite *ite;
1285
1286
1287         ite = find_ite(its, device_id, event_id);
1288         if (!ite)
1289                 return E_ITS_INV_UNMAPPED_INTERRUPT;
1290
1291         return update_lpi_config(kvm, ite->irq, NULL, true);
1292 }
1293
1294 /*
1295  * The INVALL command requests flushing of all IRQ data in this collection.
1296  * Find the VCPU mapped to that collection, then iterate over the VM's list
1297  * of mapped LPIs and update the configuration for each IRQ which targets
1298  * the specified vcpu. The configuration will be read from the in-memory
1299  * configuration table.
1300  * Must be called with the its_lock mutex held.
1301  */
1302 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1303                                       u64 *its_cmd)
1304 {
1305         u32 coll_id = its_cmd_get_collection(its_cmd);
1306         struct its_collection *collection;
1307         struct kvm_vcpu *vcpu;
1308         struct vgic_irq *irq;
1309         u32 *intids;
1310         int irq_count, i;
1311
1312         collection = find_collection(its, coll_id);
1313         if (!its_is_collection_mapped(collection))
1314                 return E_ITS_INVALL_UNMAPPED_COLLECTION;
1315
1316         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1317
1318         irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids);
1319         if (irq_count < 0)
1320                 return irq_count;
1321
1322         for (i = 0; i < irq_count; i++) {
1323                 irq = vgic_get_irq(kvm, NULL, intids[i]);
1324                 if (!irq)
1325                         continue;
1326                 update_lpi_config(kvm, irq, vcpu, false);
1327                 vgic_put_irq(kvm, irq);
1328         }
1329
1330         kfree(intids);
1331
1332         if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
1333                 its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
1334
1335         return 0;
1336 }
1337
1338 /*
1339  * The MOVALL command moves the pending state of all IRQs targeting one
1340  * redistributor to another. We don't hold the pending state in the VCPUs,
1341  * but in the IRQs instead, so there is really not much to do for us here.
1342  * However the spec says that no IRQ must target the old redistributor
1343  * afterwards, so we make sure that no LPI is using the associated target_vcpu.
1344  * This command affects all LPIs in the system that target that redistributor.
1345  */
1346 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1347                                       u64 *its_cmd)
1348 {
1349         u32 target1_addr = its_cmd_get_target_addr(its_cmd);
1350         u32 target2_addr = its_cmd_mask_field(its_cmd, 3, 16, 32);
1351         struct kvm_vcpu *vcpu1, *vcpu2;
1352         struct vgic_irq *irq;
1353         u32 *intids;
1354         int irq_count, i;
1355
1356         if (target1_addr >= atomic_read(&kvm->online_vcpus) ||
1357             target2_addr >= atomic_read(&kvm->online_vcpus))
1358                 return E_ITS_MOVALL_PROCNUM_OOR;
1359
1360         if (target1_addr == target2_addr)
1361                 return 0;
1362
1363         vcpu1 = kvm_get_vcpu(kvm, target1_addr);
1364         vcpu2 = kvm_get_vcpu(kvm, target2_addr);
1365
1366         irq_count = vgic_copy_lpi_list(kvm, vcpu1, &intids);
1367         if (irq_count < 0)
1368                 return irq_count;
1369
1370         for (i = 0; i < irq_count; i++) {
1371                 irq = vgic_get_irq(kvm, NULL, intids[i]);
1372
1373                 update_affinity(irq, vcpu2);
1374
1375                 vgic_put_irq(kvm, irq);
1376         }
1377
1378         vgic_its_invalidate_cache(kvm);
1379
1380         kfree(intids);
1381         return 0;
1382 }
1383
1384 /*
1385  * The INT command injects the LPI associated with that DevID/EvID pair.
1386  * Must be called with the its_lock mutex held.
1387  */
1388 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1389                                    u64 *its_cmd)
1390 {
1391         u32 msi_data = its_cmd_get_id(its_cmd);
1392         u64 msi_devid = its_cmd_get_deviceid(its_cmd);
1393
1394         return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1395 }
1396
1397 /*
1398  * This function is called with the its_cmd lock held, but the ITS data
1399  * structure lock dropped.
1400  */
1401 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1402                                    u64 *its_cmd)
1403 {
1404         int ret = -ENODEV;
1405
1406         mutex_lock(&its->its_lock);
1407         switch (its_cmd_get_command(its_cmd)) {
1408         case GITS_CMD_MAPD:
1409                 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1410                 break;
1411         case GITS_CMD_MAPC:
1412                 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1413                 break;
1414         case GITS_CMD_MAPI:
1415                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1416                 break;
1417         case GITS_CMD_MAPTI:
1418                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1419                 break;
1420         case GITS_CMD_MOVI:
1421                 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1422                 break;
1423         case GITS_CMD_DISCARD:
1424                 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1425                 break;
1426         case GITS_CMD_CLEAR:
1427                 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1428                 break;
1429         case GITS_CMD_MOVALL:
1430                 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1431                 break;
1432         case GITS_CMD_INT:
1433                 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1434                 break;
1435         case GITS_CMD_INV:
1436                 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1437                 break;
1438         case GITS_CMD_INVALL:
1439                 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1440                 break;
1441         case GITS_CMD_SYNC:
1442                 /* we ignore this command: we are in sync all of the time */
1443                 ret = 0;
1444                 break;
1445         }
1446         mutex_unlock(&its->its_lock);
1447
1448         return ret;
1449 }
1450
1451 static u64 vgic_sanitise_its_baser(u64 reg)
1452 {
1453         reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1454                                   GITS_BASER_SHAREABILITY_SHIFT,
1455                                   vgic_sanitise_shareability);
1456         reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1457                                   GITS_BASER_INNER_CACHEABILITY_SHIFT,
1458                                   vgic_sanitise_inner_cacheability);
1459         reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1460                                   GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1461                                   vgic_sanitise_outer_cacheability);
1462
1463         /* We support only one (ITS) page size: 64K */
1464         reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1465
1466         return reg;
1467 }
1468
1469 static u64 vgic_sanitise_its_cbaser(u64 reg)
1470 {
1471         reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1472                                   GITS_CBASER_SHAREABILITY_SHIFT,
1473                                   vgic_sanitise_shareability);
1474         reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1475                                   GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1476                                   vgic_sanitise_inner_cacheability);
1477         reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1478                                   GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1479                                   vgic_sanitise_outer_cacheability);
1480
1481         /* Sanitise the physical address to be 64k aligned. */
1482         reg &= ~GENMASK_ULL(15, 12);
1483
1484         return reg;
1485 }
1486
1487 static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1488                                                struct vgic_its *its,
1489                                                gpa_t addr, unsigned int len)
1490 {
1491         return extract_bytes(its->cbaser, addr & 7, len);
1492 }
1493
1494 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1495                                        gpa_t addr, unsigned int len,
1496                                        unsigned long val)
1497 {
1498         /* When GITS_CTLR.Enable is 1, this register is RO. */
1499         if (its->enabled)
1500                 return;
1501
1502         mutex_lock(&its->cmd_lock);
1503         its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1504         its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1505         its->creadr = 0;
1506         /*
1507          * CWRITER is architecturally UNKNOWN on reset, but we need to reset
1508          * it to CREADR to make sure we start with an empty command buffer.
1509          */
1510         its->cwriter = its->creadr;
1511         mutex_unlock(&its->cmd_lock);
1512 }
1513
1514 #define ITS_CMD_BUFFER_SIZE(baser)      ((((baser) & 0xff) + 1) << 12)
1515 #define ITS_CMD_SIZE                    32
1516 #define ITS_CMD_OFFSET(reg)             ((reg) & GENMASK(19, 5))
1517
1518 /* Must be called with the cmd_lock held. */
1519 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1520 {
1521         gpa_t cbaser;
1522         u64 cmd_buf[4];
1523
1524         /* Commands are only processed when the ITS is enabled. */
1525         if (!its->enabled)
1526                 return;
1527
1528         cbaser = GITS_CBASER_ADDRESS(its->cbaser);
1529
1530         while (its->cwriter != its->creadr) {
1531                 int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,
1532                                               cmd_buf, ITS_CMD_SIZE);
1533                 /*
1534                  * If kvm_read_guest() fails, this could be due to the guest
1535                  * programming a bogus value in CBASER or something else going
1536                  * wrong from which we cannot easily recover.
1537                  * According to section 6.3.2 in the GICv3 spec we can just
1538                  * ignore that command then.
1539                  */
1540                 if (!ret)
1541                         vgic_its_handle_command(kvm, its, cmd_buf);
1542
1543                 its->creadr += ITS_CMD_SIZE;
1544                 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1545                         its->creadr = 0;
1546         }
1547 }
1548
1549 /*
1550  * By writing to CWRITER the guest announces new commands to be processed.
1551  * To avoid any races in the first place, we take the its_cmd lock, which
1552  * protects our ring buffer variables, so that there is only one user
1553  * per ITS handling commands at a given time.
1554  */
1555 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1556                                         gpa_t addr, unsigned int len,
1557                                         unsigned long val)
1558 {
1559         u64 reg;
1560
1561         if (!its)
1562                 return;
1563
1564         mutex_lock(&its->cmd_lock);
1565
1566         reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1567         reg = ITS_CMD_OFFSET(reg);
1568         if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1569                 mutex_unlock(&its->cmd_lock);
1570                 return;
1571         }
1572         its->cwriter = reg;
1573
1574         vgic_its_process_commands(kvm, its);
1575
1576         mutex_unlock(&its->cmd_lock);
1577 }
1578
1579 static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1580                                                 struct vgic_its *its,
1581                                                 gpa_t addr, unsigned int len)
1582 {
1583         return extract_bytes(its->cwriter, addr & 0x7, len);
1584 }
1585
1586 static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1587                                                struct vgic_its *its,
1588                                                gpa_t addr, unsigned int len)
1589 {
1590         return extract_bytes(its->creadr, addr & 0x7, len);
1591 }
1592
1593 static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
1594                                               struct vgic_its *its,
1595                                               gpa_t addr, unsigned int len,
1596                                               unsigned long val)
1597 {
1598         u32 cmd_offset;
1599         int ret = 0;
1600
1601         mutex_lock(&its->cmd_lock);
1602
1603         if (its->enabled) {
1604                 ret = -EBUSY;
1605                 goto out;
1606         }
1607
1608         cmd_offset = ITS_CMD_OFFSET(val);
1609         if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1610                 ret = -EINVAL;
1611                 goto out;
1612         }
1613
1614         its->creadr = cmd_offset;
1615 out:
1616         mutex_unlock(&its->cmd_lock);
1617         return ret;
1618 }
1619
1620 #define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1621 static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1622                                               struct vgic_its *its,
1623                                               gpa_t addr, unsigned int len)
1624 {
1625         u64 reg;
1626
1627         switch (BASER_INDEX(addr)) {
1628         case 0:
1629                 reg = its->baser_device_table;
1630                 break;
1631         case 1:
1632                 reg = its->baser_coll_table;
1633                 break;
1634         default:
1635                 reg = 0;
1636                 break;
1637         }
1638
1639         return extract_bytes(reg, addr & 7, len);
1640 }
1641
1642 #define GITS_BASER_RO_MASK      (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1643 static void vgic_mmio_write_its_baser(struct kvm *kvm,
1644                                       struct vgic_its *its,
1645                                       gpa_t addr, unsigned int len,
1646                                       unsigned long val)
1647 {
1648         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1649         u64 entry_size, table_type;
1650         u64 reg, *regptr, clearbits = 0;
1651
1652         /* When GITS_CTLR.Enable is 1, we ignore write accesses. */
1653         if (its->enabled)
1654                 return;
1655
1656         switch (BASER_INDEX(addr)) {
1657         case 0:
1658                 regptr = &its->baser_device_table;
1659                 entry_size = abi->dte_esz;
1660                 table_type = GITS_BASER_TYPE_DEVICE;
1661                 break;
1662         case 1:
1663                 regptr = &its->baser_coll_table;
1664                 entry_size = abi->cte_esz;
1665                 table_type = GITS_BASER_TYPE_COLLECTION;
1666                 clearbits = GITS_BASER_INDIRECT;
1667                 break;
1668         default:
1669                 return;
1670         }
1671
1672         reg = update_64bit_reg(*regptr, addr & 7, len, val);
1673         reg &= ~GITS_BASER_RO_MASK;
1674         reg &= ~clearbits;
1675
1676         reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1677         reg |= table_type << GITS_BASER_TYPE_SHIFT;
1678         reg = vgic_sanitise_its_baser(reg);
1679
1680         *regptr = reg;
1681
1682         if (!(reg & GITS_BASER_VALID)) {
1683                 /* Take the its_lock to prevent a race with a save/restore */
1684                 mutex_lock(&its->its_lock);
1685                 switch (table_type) {
1686                 case GITS_BASER_TYPE_DEVICE:
1687                         vgic_its_free_device_list(kvm, its);
1688                         break;
1689                 case GITS_BASER_TYPE_COLLECTION:
1690                         vgic_its_free_collection_list(kvm, its);
1691                         break;
1692                 }
1693                 mutex_unlock(&its->its_lock);
1694         }
1695 }
1696
1697 static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
1698                                              struct vgic_its *its,
1699                                              gpa_t addr, unsigned int len)
1700 {
1701         u32 reg = 0;
1702
1703         mutex_lock(&its->cmd_lock);
1704         if (its->creadr == its->cwriter)
1705                 reg |= GITS_CTLR_QUIESCENT;
1706         if (its->enabled)
1707                 reg |= GITS_CTLR_ENABLE;
1708         mutex_unlock(&its->cmd_lock);
1709
1710         return reg;
1711 }
1712
1713 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1714                                      gpa_t addr, unsigned int len,
1715                                      unsigned long val)
1716 {
1717         mutex_lock(&its->cmd_lock);
1718
1719         /*
1720          * It is UNPREDICTABLE to enable the ITS if any of the CBASER or
1721          * device/collection BASER are invalid
1722          */
1723         if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1724                 (!(its->baser_device_table & GITS_BASER_VALID) ||
1725                  !(its->baser_coll_table & GITS_BASER_VALID) ||
1726                  !(its->cbaser & GITS_CBASER_VALID)))
1727                 goto out;
1728
1729         its->enabled = !!(val & GITS_CTLR_ENABLE);
1730         if (!its->enabled)
1731                 vgic_its_invalidate_cache(kvm);
1732
1733         /*
1734          * Try to process any pending commands. This function bails out early
1735          * if the ITS is disabled or no commands have been queued.
1736          */
1737         vgic_its_process_commands(kvm, its);
1738
1739 out:
1740         mutex_unlock(&its->cmd_lock);
1741 }
1742
1743 #define REGISTER_ITS_DESC(off, rd, wr, length, acc)             \
1744 {                                                               \
1745         .reg_offset = off,                                      \
1746         .len = length,                                          \
1747         .access_flags = acc,                                    \
1748         .its_read = rd,                                         \
1749         .its_write = wr,                                        \
1750 }
1751
1752 #define REGISTER_ITS_DESC_UACCESS(off, rd, wr, uwr, length, acc)\
1753 {                                                               \
1754         .reg_offset = off,                                      \
1755         .len = length,                                          \
1756         .access_flags = acc,                                    \
1757         .its_read = rd,                                         \
1758         .its_write = wr,                                        \
1759         .uaccess_its_write = uwr,                               \
1760 }
1761
1762 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1763                               gpa_t addr, unsigned int len, unsigned long val)
1764 {
1765         /* Ignore */
1766 }
1767
1768 static struct vgic_register_region its_registers[] = {
1769         REGISTER_ITS_DESC(GITS_CTLR,
1770                 vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1771                 VGIC_ACCESS_32bit),
1772         REGISTER_ITS_DESC_UACCESS(GITS_IIDR,
1773                 vgic_mmio_read_its_iidr, its_mmio_write_wi,
1774                 vgic_mmio_uaccess_write_its_iidr, 4,
1775                 VGIC_ACCESS_32bit),
1776         REGISTER_ITS_DESC(GITS_TYPER,
1777                 vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1778                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1779         REGISTER_ITS_DESC(GITS_CBASER,
1780                 vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1781                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1782         REGISTER_ITS_DESC(GITS_CWRITER,
1783                 vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1784                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1785         REGISTER_ITS_DESC_UACCESS(GITS_CREADR,
1786                 vgic_mmio_read_its_creadr, its_mmio_write_wi,
1787                 vgic_mmio_uaccess_write_its_creadr, 8,
1788                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1789         REGISTER_ITS_DESC(GITS_BASER,
1790                 vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1791                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1792         REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1793                 vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1794                 VGIC_ACCESS_32bit),
1795 };
1796
1797 /* This is called on setting the LPI enable bit in the redistributor. */
1798 void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1799 {
1800         if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1801                 its_sync_lpi_pending_table(vcpu);
1802 }
1803
1804 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1805                                    u64 addr)
1806 {
1807         struct vgic_io_device *iodev = &its->iodev;
1808         int ret;
1809
1810         mutex_lock(&kvm->slots_lock);
1811         if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1812                 ret = -EBUSY;
1813                 goto out;
1814         }
1815
1816         its->vgic_its_base = addr;
1817         iodev->regions = its_registers;
1818         iodev->nr_regions = ARRAY_SIZE(its_registers);
1819         kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1820
1821         iodev->base_addr = its->vgic_its_base;
1822         iodev->iodev_type = IODEV_ITS;
1823         iodev->its = its;
1824         ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
1825                                       KVM_VGIC_V3_ITS_SIZE, &iodev->dev);
1826 out:
1827         mutex_unlock(&kvm->slots_lock);
1828
1829         return ret;
1830 }
1831
1832 /* Default is 16 cached LPIs per vcpu */
1833 #define LPI_DEFAULT_PCPU_CACHE_SIZE     16
1834
1835 void vgic_lpi_translation_cache_init(struct kvm *kvm)
1836 {
1837         struct vgic_dist *dist = &kvm->arch.vgic;
1838         unsigned int sz;
1839         int i;
1840
1841         if (!list_empty(&dist->lpi_translation_cache))
1842                 return;
1843
1844         sz = atomic_read(&kvm->online_vcpus) * LPI_DEFAULT_PCPU_CACHE_SIZE;
1845
1846         for (i = 0; i < sz; i++) {
1847                 struct vgic_translation_cache_entry *cte;
1848
1849                 /* An allocation failure is not fatal */
1850                 cte = kzalloc(sizeof(*cte), GFP_KERNEL_ACCOUNT);
1851                 if (WARN_ON(!cte))
1852                         break;
1853
1854                 INIT_LIST_HEAD(&cte->entry);
1855                 list_add(&cte->entry, &dist->lpi_translation_cache);
1856         }
1857 }
1858
1859 void vgic_lpi_translation_cache_destroy(struct kvm *kvm)
1860 {
1861         struct vgic_dist *dist = &kvm->arch.vgic;
1862         struct vgic_translation_cache_entry *cte, *tmp;
1863
1864         vgic_its_invalidate_cache(kvm);
1865
1866         list_for_each_entry_safe(cte, tmp,
1867                                  &dist->lpi_translation_cache, entry) {
1868                 list_del(&cte->entry);
1869                 kfree(cte);
1870         }
1871 }
1872
1873 #define INITIAL_BASER_VALUE                                               \
1874         (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb)                | \
1875          GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner)         | \
1876          GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable)             | \
1877          GITS_BASER_PAGE_SIZE_64K)
1878
1879 #define INITIAL_PROPBASER_VALUE                                           \
1880         (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb)            | \
1881          GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner)     | \
1882          GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1883
1884 static int vgic_its_create(struct kvm_device *dev, u32 type)
1885 {
1886         struct vgic_its *its;
1887
1888         if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1889                 return -ENODEV;
1890
1891         its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL_ACCOUNT);
1892         if (!its)
1893                 return -ENOMEM;
1894
1895         if (vgic_initialized(dev->kvm)) {
1896                 int ret = vgic_v4_init(dev->kvm);
1897                 if (ret < 0) {
1898                         kfree(its);
1899                         return ret;
1900                 }
1901
1902                 vgic_lpi_translation_cache_init(dev->kvm);
1903         }
1904
1905         mutex_init(&its->its_lock);
1906         mutex_init(&its->cmd_lock);
1907
1908         its->vgic_its_base = VGIC_ADDR_UNDEF;
1909
1910         INIT_LIST_HEAD(&its->device_list);
1911         INIT_LIST_HEAD(&its->collection_list);
1912
1913         dev->kvm->arch.vgic.msis_require_devid = true;
1914         dev->kvm->arch.vgic.has_its = true;
1915         its->enabled = false;
1916         its->dev = dev;
1917
1918         its->baser_device_table = INITIAL_BASER_VALUE                   |
1919                 ((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
1920         its->baser_coll_table = INITIAL_BASER_VALUE |
1921                 ((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
1922         dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
1923
1924         dev->private = its;
1925
1926         return vgic_its_set_abi(its, NR_ITS_ABIS - 1);
1927 }
1928
1929 static void vgic_its_destroy(struct kvm_device *kvm_dev)
1930 {
1931         struct kvm *kvm = kvm_dev->kvm;
1932         struct vgic_its *its = kvm_dev->private;
1933
1934         mutex_lock(&its->its_lock);
1935
1936         vgic_its_free_device_list(kvm, its);
1937         vgic_its_free_collection_list(kvm, its);
1938
1939         mutex_unlock(&its->its_lock);
1940         kfree(its);
1941         kfree(kvm_dev);/* alloc by kvm_ioctl_create_device, free by .destroy */
1942 }
1943
1944 static int vgic_its_has_attr_regs(struct kvm_device *dev,
1945                                   struct kvm_device_attr *attr)
1946 {
1947         const struct vgic_register_region *region;
1948         gpa_t offset = attr->attr;
1949         int align;
1950
1951         align = (offset < GITS_TYPER) || (offset >= GITS_PIDR4) ? 0x3 : 0x7;
1952
1953         if (offset & align)
1954                 return -EINVAL;
1955
1956         region = vgic_find_mmio_region(its_registers,
1957                                        ARRAY_SIZE(its_registers),
1958                                        offset);
1959         if (!region)
1960                 return -ENXIO;
1961
1962         return 0;
1963 }
1964
1965 static int vgic_its_attr_regs_access(struct kvm_device *dev,
1966                                      struct kvm_device_attr *attr,
1967                                      u64 *reg, bool is_write)
1968 {
1969         const struct vgic_register_region *region;
1970         struct vgic_its *its;
1971         gpa_t addr, offset;
1972         unsigned int len;
1973         int align, ret = 0;
1974
1975         its = dev->private;
1976         offset = attr->attr;
1977
1978         /*
1979          * Although the spec supports upper/lower 32-bit accesses to
1980          * 64-bit ITS registers, the userspace ABI requires 64-bit
1981          * accesses to all 64-bit wide registers. We therefore only
1982          * support 32-bit accesses to GITS_CTLR, GITS_IIDR and GITS ID
1983          * registers
1984          */
1985         if ((offset < GITS_TYPER) || (offset >= GITS_PIDR4))
1986                 align = 0x3;
1987         else
1988                 align = 0x7;
1989
1990         if (offset & align)
1991                 return -EINVAL;
1992
1993         mutex_lock(&dev->kvm->lock);
1994
1995         if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1996                 ret = -ENXIO;
1997                 goto out;
1998         }
1999
2000         region = vgic_find_mmio_region(its_registers,
2001                                        ARRAY_SIZE(its_registers),
2002                                        offset);
2003         if (!region) {
2004                 ret = -ENXIO;
2005                 goto out;
2006         }
2007
2008         if (!lock_all_vcpus(dev->kvm)) {
2009                 ret = -EBUSY;
2010                 goto out;
2011         }
2012
2013         addr = its->vgic_its_base + offset;
2014
2015         len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
2016
2017         if (is_write) {
2018                 if (region->uaccess_its_write)
2019                         ret = region->uaccess_its_write(dev->kvm, its, addr,
2020                                                         len, *reg);
2021                 else
2022                         region->its_write(dev->kvm, its, addr, len, *reg);
2023         } else {
2024                 *reg = region->its_read(dev->kvm, its, addr, len);
2025         }
2026         unlock_all_vcpus(dev->kvm);
2027 out:
2028         mutex_unlock(&dev->kvm->lock);
2029         return ret;
2030 }
2031
2032 static u32 compute_next_devid_offset(struct list_head *h,
2033                                      struct its_device *dev)
2034 {
2035         struct its_device *next;
2036         u32 next_offset;
2037
2038         if (list_is_last(&dev->dev_list, h))
2039                 return 0;
2040         next = list_next_entry(dev, dev_list);
2041         next_offset = next->device_id - dev->device_id;
2042
2043         return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET);
2044 }
2045
2046 static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite)
2047 {
2048         struct its_ite *next;
2049         u32 next_offset;
2050
2051         if (list_is_last(&ite->ite_list, h))
2052                 return 0;
2053         next = list_next_entry(ite, ite_list);
2054         next_offset = next->event_id - ite->event_id;
2055
2056         return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET);
2057 }
2058
2059 /**
2060  * entry_fn_t - Callback called on a table entry restore path
2061  * @its: its handle
2062  * @id: id of the entry
2063  * @entry: pointer to the entry
2064  * @opaque: pointer to an opaque data
2065  *
2066  * Return: < 0 on error, 0 if last element was identified, id offset to next
2067  * element otherwise
2068  */
2069 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
2070                           void *opaque);
2071
2072 /**
2073  * scan_its_table - Scan a contiguous table in guest RAM and applies a function
2074  * to each entry
2075  *
2076  * @its: its handle
2077  * @base: base gpa of the table
2078  * @size: size of the table in bytes
2079  * @esz: entry size in bytes
2080  * @start_id: the ID of the first entry in the table
2081  * (non zero for 2d level tables)
2082  * @fn: function to apply on each entry
2083  *
2084  * Return: < 0 on error, 0 if last element was identified, 1 otherwise
2085  * (the last element may not be found on second level tables)
2086  */
2087 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, u32 esz,
2088                           int start_id, entry_fn_t fn, void *opaque)
2089 {
2090         struct kvm *kvm = its->dev->kvm;
2091         unsigned long len = size;
2092         int id = start_id;
2093         gpa_t gpa = base;
2094         char entry[ESZ_MAX];
2095         int ret;
2096
2097         memset(entry, 0, esz);
2098
2099         while (len > 0) {
2100                 int next_offset;
2101                 size_t byte_offset;
2102
2103                 ret = kvm_read_guest_lock(kvm, gpa, entry, esz);
2104                 if (ret)
2105                         return ret;
2106
2107                 next_offset = fn(its, id, entry, opaque);
2108                 if (next_offset <= 0)
2109                         return next_offset;
2110
2111                 byte_offset = next_offset * esz;
2112                 id += next_offset;
2113                 gpa += byte_offset;
2114                 len -= byte_offset;
2115         }
2116         return 1;
2117 }
2118
2119 /**
2120  * vgic_its_save_ite - Save an interrupt translation entry at @gpa
2121  */
2122 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
2123                               struct its_ite *ite, gpa_t gpa, int ite_esz)
2124 {
2125         struct kvm *kvm = its->dev->kvm;
2126         u32 next_offset;
2127         u64 val;
2128
2129         next_offset = compute_next_eventid_offset(&dev->itt_head, ite);
2130         val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) |
2131                ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) |
2132                 ite->collection->collection_id;
2133         val = cpu_to_le64(val);
2134         return kvm_write_guest_lock(kvm, gpa, &val, ite_esz);
2135 }
2136
2137 /**
2138  * vgic_its_restore_ite - restore an interrupt translation entry
2139  * @event_id: id used for indexing
2140  * @ptr: pointer to the ITE entry
2141  * @opaque: pointer to the its_device
2142  */
2143 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
2144                                 void *ptr, void *opaque)
2145 {
2146         struct its_device *dev = opaque;
2147         struct its_collection *collection;
2148         struct kvm *kvm = its->dev->kvm;
2149         struct kvm_vcpu *vcpu = NULL;
2150         u64 val;
2151         u64 *p = (u64 *)ptr;
2152         struct vgic_irq *irq;
2153         u32 coll_id, lpi_id;
2154         struct its_ite *ite;
2155         u32 offset;
2156
2157         val = *p;
2158
2159         val = le64_to_cpu(val);
2160
2161         coll_id = val & KVM_ITS_ITE_ICID_MASK;
2162         lpi_id = (val & KVM_ITS_ITE_PINTID_MASK) >> KVM_ITS_ITE_PINTID_SHIFT;
2163
2164         if (!lpi_id)
2165                 return 1; /* invalid entry, no choice but to scan next entry */
2166
2167         if (lpi_id < VGIC_MIN_LPI)
2168                 return -EINVAL;
2169
2170         offset = val >> KVM_ITS_ITE_NEXT_SHIFT;
2171         if (event_id + offset >= BIT_ULL(dev->num_eventid_bits))
2172                 return -EINVAL;
2173
2174         collection = find_collection(its, coll_id);
2175         if (!collection)
2176                 return -EINVAL;
2177
2178         ite = vgic_its_alloc_ite(dev, collection, event_id);
2179         if (IS_ERR(ite))
2180                 return PTR_ERR(ite);
2181
2182         if (its_is_collection_mapped(collection))
2183                 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
2184
2185         irq = vgic_add_lpi(kvm, lpi_id, vcpu);
2186         if (IS_ERR(irq))
2187                 return PTR_ERR(irq);
2188         ite->irq = irq;
2189
2190         return offset;
2191 }
2192
2193 static int vgic_its_ite_cmp(void *priv, const struct list_head *a,
2194                             const struct list_head *b)
2195 {
2196         struct its_ite *itea = container_of(a, struct its_ite, ite_list);
2197         struct its_ite *iteb = container_of(b, struct its_ite, ite_list);
2198
2199         if (itea->event_id < iteb->event_id)
2200                 return -1;
2201         else
2202                 return 1;
2203 }
2204
2205 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
2206 {
2207         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2208         gpa_t base = device->itt_addr;
2209         struct its_ite *ite;
2210         int ret;
2211         int ite_esz = abi->ite_esz;
2212
2213         list_sort(NULL, &device->itt_head, vgic_its_ite_cmp);
2214
2215         list_for_each_entry(ite, &device->itt_head, ite_list) {
2216                 gpa_t gpa = base + ite->event_id * ite_esz;
2217
2218                 /*
2219                  * If an LPI carries the HW bit, this means that this
2220                  * interrupt is controlled by GICv4, and we do not
2221                  * have direct access to that state without GICv4.1.
2222                  * Let's simply fail the save operation...
2223                  */
2224                 if (ite->irq->hw && !kvm_vgic_global_state.has_gicv4_1)
2225                         return -EACCES;
2226
2227                 ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);
2228                 if (ret)
2229                         return ret;
2230         }
2231         return 0;
2232 }
2233
2234 /**
2235  * vgic_its_restore_itt - restore the ITT of a device
2236  *
2237  * @its: its handle
2238  * @dev: device handle
2239  *
2240  * Return 0 on success, < 0 on error
2241  */
2242 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
2243 {
2244         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2245         gpa_t base = dev->itt_addr;
2246         int ret;
2247         int ite_esz = abi->ite_esz;
2248         size_t max_size = BIT_ULL(dev->num_eventid_bits) * ite_esz;
2249
2250         ret = scan_its_table(its, base, max_size, ite_esz, 0,
2251                              vgic_its_restore_ite, dev);
2252
2253         /* scan_its_table returns +1 if all ITEs are invalid */
2254         if (ret > 0)
2255                 ret = 0;
2256
2257         return ret;
2258 }
2259
2260 /**
2261  * vgic_its_save_dte - Save a device table entry at a given GPA
2262  *
2263  * @its: ITS handle
2264  * @dev: ITS device
2265  * @ptr: GPA
2266  */
2267 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
2268                              gpa_t ptr, int dte_esz)
2269 {
2270         struct kvm *kvm = its->dev->kvm;
2271         u64 val, itt_addr_field;
2272         u32 next_offset;
2273
2274         itt_addr_field = dev->itt_addr >> 8;
2275         next_offset = compute_next_devid_offset(&its->device_list, dev);
2276         val = (1ULL << KVM_ITS_DTE_VALID_SHIFT |
2277                ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) |
2278                (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) |
2279                 (dev->num_eventid_bits - 1));
2280         val = cpu_to_le64(val);
2281         return kvm_write_guest_lock(kvm, ptr, &val, dte_esz);
2282 }
2283
2284 /**
2285  * vgic_its_restore_dte - restore a device table entry
2286  *
2287  * @its: its handle
2288  * @id: device id the DTE corresponds to
2289  * @ptr: kernel VA where the 8 byte DTE is located
2290  * @opaque: unused
2291  *
2292  * Return: < 0 on error, 0 if the dte is the last one, id offset to the
2293  * next dte otherwise
2294  */
2295 static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2296                                 void *ptr, void *opaque)
2297 {
2298         struct its_device *dev;
2299         gpa_t itt_addr;
2300         u8 num_eventid_bits;
2301         u64 entry = *(u64 *)ptr;
2302         bool valid;
2303         u32 offset;
2304         int ret;
2305
2306         entry = le64_to_cpu(entry);
2307
2308         valid = entry >> KVM_ITS_DTE_VALID_SHIFT;
2309         num_eventid_bits = (entry & KVM_ITS_DTE_SIZE_MASK) + 1;
2310         itt_addr = ((entry & KVM_ITS_DTE_ITTADDR_MASK)
2311                         >> KVM_ITS_DTE_ITTADDR_SHIFT) << 8;
2312
2313         if (!valid)
2314                 return 1;
2315
2316         /* dte entry is valid */
2317         offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
2318
2319         dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2320         if (IS_ERR(dev))
2321                 return PTR_ERR(dev);
2322
2323         ret = vgic_its_restore_itt(its, dev);
2324         if (ret) {
2325                 vgic_its_free_device(its->dev->kvm, dev);
2326                 return ret;
2327         }
2328
2329         return offset;
2330 }
2331
2332 static int vgic_its_device_cmp(void *priv, const struct list_head *a,
2333                                const struct list_head *b)
2334 {
2335         struct its_device *deva = container_of(a, struct its_device, dev_list);
2336         struct its_device *devb = container_of(b, struct its_device, dev_list);
2337
2338         if (deva->device_id < devb->device_id)
2339                 return -1;
2340         else
2341                 return 1;
2342 }
2343
2344 /**
2345  * vgic_its_save_device_tables - Save the device table and all ITT
2346  * into guest RAM
2347  *
2348  * L1/L2 handling is hidden by vgic_its_check_id() helper which directly
2349  * returns the GPA of the device entry
2350  */
2351 static int vgic_its_save_device_tables(struct vgic_its *its)
2352 {
2353         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2354         u64 baser = its->baser_device_table;
2355         struct its_device *dev;
2356         int dte_esz = abi->dte_esz;
2357
2358         if (!(baser & GITS_BASER_VALID))
2359                 return 0;
2360
2361         list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2362
2363         list_for_each_entry(dev, &its->device_list, dev_list) {
2364                 int ret;
2365                 gpa_t eaddr;
2366
2367                 if (!vgic_its_check_id(its, baser,
2368                                        dev->device_id, &eaddr))
2369                         return -EINVAL;
2370
2371                 ret = vgic_its_save_itt(its, dev);
2372                 if (ret)
2373                         return ret;
2374
2375                 ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);
2376                 if (ret)
2377                         return ret;
2378         }
2379         return 0;
2380 }
2381
2382 /**
2383  * handle_l1_dte - callback used for L1 device table entries (2 stage case)
2384  *
2385  * @its: its handle
2386  * @id: index of the entry in the L1 table
2387  * @addr: kernel VA
2388  * @opaque: unused
2389  *
2390  * L1 table entries are scanned by steps of 1 entry
2391  * Return < 0 if error, 0 if last dte was found when scanning the L2
2392  * table, +1 otherwise (meaning next L1 entry must be scanned)
2393  */
2394 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2395                          void *opaque)
2396 {
2397         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2398         int l2_start_id = id * (SZ_64K / abi->dte_esz);
2399         u64 entry = *(u64 *)addr;
2400         int dte_esz = abi->dte_esz;
2401         gpa_t gpa;
2402         int ret;
2403
2404         entry = le64_to_cpu(entry);
2405
2406         if (!(entry & KVM_ITS_L1E_VALID_MASK))
2407                 return 1;
2408
2409         gpa = entry & KVM_ITS_L1E_ADDR_MASK;
2410
2411         ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2412                              l2_start_id, vgic_its_restore_dte, NULL);
2413
2414         return ret;
2415 }
2416
2417 /**
2418  * vgic_its_restore_device_tables - Restore the device table and all ITT
2419  * from guest RAM to internal data structs
2420  */
2421 static int vgic_its_restore_device_tables(struct vgic_its *its)
2422 {
2423         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2424         u64 baser = its->baser_device_table;
2425         int l1_esz, ret;
2426         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2427         gpa_t l1_gpa;
2428
2429         if (!(baser & GITS_BASER_VALID))
2430                 return 0;
2431
2432         l1_gpa = GITS_BASER_ADDR_48_to_52(baser);
2433
2434         if (baser & GITS_BASER_INDIRECT) {
2435                 l1_esz = GITS_LVL1_ENTRY_SIZE;
2436                 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2437                                      handle_l1_dte, NULL);
2438         } else {
2439                 l1_esz = abi->dte_esz;
2440                 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2441                                      vgic_its_restore_dte, NULL);
2442         }
2443
2444         /* scan_its_table returns +1 if all entries are invalid */
2445         if (ret > 0)
2446                 ret = 0;
2447
2448         return ret;
2449 }
2450
2451 static int vgic_its_save_cte(struct vgic_its *its,
2452                              struct its_collection *collection,
2453                              gpa_t gpa, int esz)
2454 {
2455         u64 val;
2456
2457         val = (1ULL << KVM_ITS_CTE_VALID_SHIFT |
2458                ((u64)collection->target_addr << KVM_ITS_CTE_RDBASE_SHIFT) |
2459                collection->collection_id);
2460         val = cpu_to_le64(val);
2461         return kvm_write_guest_lock(its->dev->kvm, gpa, &val, esz);
2462 }
2463
2464 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2465 {
2466         struct its_collection *collection;
2467         struct kvm *kvm = its->dev->kvm;
2468         u32 target_addr, coll_id;
2469         u64 val;
2470         int ret;
2471
2472         BUG_ON(esz > sizeof(val));
2473         ret = kvm_read_guest_lock(kvm, gpa, &val, esz);
2474         if (ret)
2475                 return ret;
2476         val = le64_to_cpu(val);
2477         if (!(val & KVM_ITS_CTE_VALID_MASK))
2478                 return 0;
2479
2480         target_addr = (u32)(val >> KVM_ITS_CTE_RDBASE_SHIFT);
2481         coll_id = val & KVM_ITS_CTE_ICID_MASK;
2482
2483         if (target_addr != COLLECTION_NOT_MAPPED &&
2484             target_addr >= atomic_read(&kvm->online_vcpus))
2485                 return -EINVAL;
2486
2487         collection = find_collection(its, coll_id);
2488         if (collection)
2489                 return -EEXIST;
2490         ret = vgic_its_alloc_collection(its, &collection, coll_id);
2491         if (ret)
2492                 return ret;
2493         collection->target_addr = target_addr;
2494         return 1;
2495 }
2496
2497 /**
2498  * vgic_its_save_collection_table - Save the collection table into
2499  * guest RAM
2500  */
2501 static int vgic_its_save_collection_table(struct vgic_its *its)
2502 {
2503         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2504         u64 baser = its->baser_coll_table;
2505         gpa_t gpa = GITS_BASER_ADDR_48_to_52(baser);
2506         struct its_collection *collection;
2507         u64 val;
2508         size_t max_size, filled = 0;
2509         int ret, cte_esz = abi->cte_esz;
2510
2511         if (!(baser & GITS_BASER_VALID))
2512                 return 0;
2513
2514         max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2515
2516         list_for_each_entry(collection, &its->collection_list, coll_list) {
2517                 ret = vgic_its_save_cte(its, collection, gpa, cte_esz);
2518                 if (ret)
2519                         return ret;
2520                 gpa += cte_esz;
2521                 filled += cte_esz;
2522         }
2523
2524         if (filled == max_size)
2525                 return 0;
2526
2527         /*
2528          * table is not fully filled, add a last dummy element
2529          * with valid bit unset
2530          */
2531         val = 0;
2532         BUG_ON(cte_esz > sizeof(val));
2533         ret = kvm_write_guest_lock(its->dev->kvm, gpa, &val, cte_esz);
2534         return ret;
2535 }
2536
2537 /**
2538  * vgic_its_restore_collection_table - reads the collection table
2539  * in guest memory and restores the ITS internal state. Requires the
2540  * BASER registers to be restored before.
2541  */
2542 static int vgic_its_restore_collection_table(struct vgic_its *its)
2543 {
2544         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2545         u64 baser = its->baser_coll_table;
2546         int cte_esz = abi->cte_esz;
2547         size_t max_size, read = 0;
2548         gpa_t gpa;
2549         int ret;
2550
2551         if (!(baser & GITS_BASER_VALID))
2552                 return 0;
2553
2554         gpa = GITS_BASER_ADDR_48_to_52(baser);
2555
2556         max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2557
2558         while (read < max_size) {
2559                 ret = vgic_its_restore_cte(its, gpa, cte_esz);
2560                 if (ret <= 0)
2561                         break;
2562                 gpa += cte_esz;
2563                 read += cte_esz;
2564         }
2565
2566         if (ret > 0)
2567                 return 0;
2568
2569         return ret;
2570 }
2571
2572 /**
2573  * vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
2574  * according to v0 ABI
2575  */
2576 static int vgic_its_save_tables_v0(struct vgic_its *its)
2577 {
2578         int ret;
2579
2580         ret = vgic_its_save_device_tables(its);
2581         if (ret)
2582                 return ret;
2583
2584         return vgic_its_save_collection_table(its);
2585 }
2586
2587 /**
2588  * vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
2589  * to internal data structs according to V0 ABI
2590  *
2591  */
2592 static int vgic_its_restore_tables_v0(struct vgic_its *its)
2593 {
2594         int ret;
2595
2596         ret = vgic_its_restore_collection_table(its);
2597         if (ret)
2598                 return ret;
2599
2600         return vgic_its_restore_device_tables(its);
2601 }
2602
2603 static int vgic_its_commit_v0(struct vgic_its *its)
2604 {
2605         const struct vgic_its_abi *abi;
2606
2607         abi = vgic_its_get_abi(its);
2608         its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2609         its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2610
2611         its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2612                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
2613
2614         its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2615                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
2616         return 0;
2617 }
2618
2619 static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2620 {
2621         /* We need to keep the ABI specific field values */
2622         its->baser_coll_table &= ~GITS_BASER_VALID;
2623         its->baser_device_table &= ~GITS_BASER_VALID;
2624         its->cbaser = 0;
2625         its->creadr = 0;
2626         its->cwriter = 0;
2627         its->enabled = 0;
2628         vgic_its_free_device_list(kvm, its);
2629         vgic_its_free_collection_list(kvm, its);
2630 }
2631
2632 static int vgic_its_has_attr(struct kvm_device *dev,
2633                              struct kvm_device_attr *attr)
2634 {
2635         switch (attr->group) {
2636         case KVM_DEV_ARM_VGIC_GRP_ADDR:
2637                 switch (attr->attr) {
2638                 case KVM_VGIC_ITS_ADDR_TYPE:
2639                         return 0;
2640                 }
2641                 break;
2642         case KVM_DEV_ARM_VGIC_GRP_CTRL:
2643                 switch (attr->attr) {
2644                 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2645                         return 0;
2646                 case KVM_DEV_ARM_ITS_CTRL_RESET:
2647                         return 0;
2648                 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2649                         return 0;
2650                 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2651                         return 0;
2652                 }
2653                 break;
2654         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS:
2655                 return vgic_its_has_attr_regs(dev, attr);
2656         }
2657         return -ENXIO;
2658 }
2659
2660 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2661 {
2662         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2663         int ret = 0;
2664
2665         if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT) /* Nothing to do */
2666                 return 0;
2667
2668         mutex_lock(&kvm->lock);
2669         mutex_lock(&its->its_lock);
2670
2671         if (!lock_all_vcpus(kvm)) {
2672                 mutex_unlock(&its->its_lock);
2673                 mutex_unlock(&kvm->lock);
2674                 return -EBUSY;
2675         }
2676
2677         switch (attr) {
2678         case KVM_DEV_ARM_ITS_CTRL_RESET:
2679                 vgic_its_reset(kvm, its);
2680                 break;
2681         case KVM_DEV_ARM_ITS_SAVE_TABLES:
2682                 ret = abi->save_tables(its);
2683                 break;
2684         case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2685                 ret = abi->restore_tables(its);
2686                 break;
2687         }
2688
2689         unlock_all_vcpus(kvm);
2690         mutex_unlock(&its->its_lock);
2691         mutex_unlock(&kvm->lock);
2692         return ret;
2693 }
2694
2695 static int vgic_its_set_attr(struct kvm_device *dev,
2696                              struct kvm_device_attr *attr)
2697 {
2698         struct vgic_its *its = dev->private;
2699         int ret;
2700
2701         switch (attr->group) {
2702         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2703                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2704                 unsigned long type = (unsigned long)attr->attr;
2705                 u64 addr;
2706
2707                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2708                         return -ENODEV;
2709
2710                 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2711                         return -EFAULT;
2712
2713                 ret = vgic_check_iorange(dev->kvm, its->vgic_its_base,
2714                                          addr, SZ_64K, KVM_VGIC_V3_ITS_SIZE);
2715                 if (ret)
2716                         return ret;
2717
2718                 return vgic_register_its_iodev(dev->kvm, its, addr);
2719         }
2720         case KVM_DEV_ARM_VGIC_GRP_CTRL:
2721                 return vgic_its_ctrl(dev->kvm, its, attr->attr);
2722         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2723                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2724                 u64 reg;
2725
2726                 if (get_user(reg, uaddr))
2727                         return -EFAULT;
2728
2729                 return vgic_its_attr_regs_access(dev, attr, &reg, true);
2730         }
2731         }
2732         return -ENXIO;
2733 }
2734
2735 static int vgic_its_get_attr(struct kvm_device *dev,
2736                              struct kvm_device_attr *attr)
2737 {
2738         switch (attr->group) {
2739         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2740                 struct vgic_its *its = dev->private;
2741                 u64 addr = its->vgic_its_base;
2742                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2743                 unsigned long type = (unsigned long)attr->attr;
2744
2745                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2746                         return -ENODEV;
2747
2748                 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2749                         return -EFAULT;
2750                 break;
2751         }
2752         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2753                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2754                 u64 reg;
2755                 int ret;
2756
2757                 ret = vgic_its_attr_regs_access(dev, attr, &reg, false);
2758                 if (ret)
2759                         return ret;
2760                 return put_user(reg, uaddr);
2761         }
2762         default:
2763                 return -ENXIO;
2764         }
2765
2766         return 0;
2767 }
2768
2769 static struct kvm_device_ops kvm_arm_vgic_its_ops = {
2770         .name = "kvm-arm-vgic-its",
2771         .create = vgic_its_create,
2772         .destroy = vgic_its_destroy,
2773         .set_attr = vgic_its_set_attr,
2774         .get_attr = vgic_its_get_attr,
2775         .has_attr = vgic_its_has_attr,
2776 };
2777
2778 int kvm_vgic_register_its_device(void)
2779 {
2780         return kvm_register_device_ops(&kvm_arm_vgic_its_ops,
2781                                        KVM_DEV_TYPE_ARM_VGIC_ITS);
2782 }