drivers/irqchip/irq-gic-v3-its.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/acpi.h>
   8 #include <linux/acpi_iort.h>
   9 #include <linux/bitfield.h>
  10 #include <linux/bitmap.h>
  11 #include <linux/cpu.h>
  12 #include <linux/crash_dump.h>
  13 #include <linux/delay.h>
  14 #include <linux/efi.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/iommu.h>
  17 #include <linux/iopoll.h>
  18 #include <linux/irqdomain.h>
  19 #include <linux/list.h>
  20 #include <linux/log2.h>
  21 #include <linux/memblock.h>
  22 #include <linux/mm.h>
  23 #include <linux/msi.h>
  24 #include <linux/of.h>
  25 #include <linux/of_address.h>
  26 #include <linux/of_irq.h>
  27 #include <linux/of_pci.h>
  28 #include <linux/of_platform.h>
  29 #include <linux/percpu.h>
  30 #include <linux/slab.h>
  31 #include <linux/syscore_ops.h>
  32
  33 #include <linux/irqchip.h>
  34 #include <linux/irqchip/arm-gic-v3.h>
  35 #include <linux/irqchip/arm-gic-v4.h>
  36
  37 #include <asm/cputype.h>
  38 #include <asm/exception.h>
  39
  40 #include "irq-gic-common.h"
  41
  42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING           (1ULL << 0)
  43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375       (1ULL << 1)
  44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144       (1ULL << 2)
  45 #define ITS_FLAGS_FORCE_NON_SHAREABLE           (1ULL << 3)
  46
  47 #define RD_LOCAL_LPI_ENABLED                    BIT(0)
  48 #define RD_LOCAL_PENDTABLE_PREALLOCATED         BIT(1)
  49 #define RD_LOCAL_MEMRESERVE_DONE                BIT(2)
  50
  51 static u32 lpi_id_bits;
  52
  53 /*
  54  * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
  55  * deal with (one configuration byte per interrupt). PENDBASE has to
  56  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
  57  */
  58 #define LPI_NRBITS              lpi_id_bits
  59 #define LPI_PROPBASE_SZ         ALIGN(BIT(LPI_NRBITS), SZ_64K)
  60 #define LPI_PENDBASE_SZ         ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
  61
  62 #define LPI_PROP_DEFAULT_PRIO   GICD_INT_DEF_PRI
  63
  64 /*
  65  * Collection structure - just an ID, and a redistributor address to
  66  * ping. We use one per CPU as a bag of interrupts assigned to this
  67  * CPU.
  68  */
  69 struct its_collection {
  70         u64                     target_address;
  71         u16                     col_id;
  72 };
  73
  74 /*
  75  * The ITS_BASER structure - contains memory information, cached
  76  * value of BASER register configuration and ITS page size.
  77  */
  78 struct its_baser {
  79         void            *base;
  80         u64             val;
  81         u32             order;
  82         u32             psz;
  83 };
  84
  85 struct its_device;
  86
  87 /*
  88  * The ITS structure - contains most of the infrastructure, with the
  89  * top-level MSI domain, the command queue, the collections, and the
  90  * list of devices writing to it.
  91  *
  92  * dev_alloc_lock has to be taken for device allocations, while the
  93  * spinlock must be taken to parse data structures such as the device
  94  * list.
  95  */
  96 struct its_node {
  97         raw_spinlock_t          lock;
  98         struct mutex            dev_alloc_lock;
  99         struct list_head        entry;
 100         void __iomem            *base;
 101         void __iomem            *sgir_base;
 102         phys_addr_t             phys_base;
 103         struct its_cmd_block    *cmd_base;
 104         struct its_cmd_block    *cmd_write;
 105         struct its_baser        tables[GITS_BASER_NR_REGS];
 106         struct its_collection   *collections;
 107         struct fwnode_handle    *fwnode_handle;
 108         u64                     (*get_msi_base)(struct its_device *its_dev);
 109         u64                     typer;
 110         u64                     cbaser_save;
 111         u32                     ctlr_save;
 112         u32                     mpidr;
 113         struct list_head        its_device_list;
 114         u64                     flags;
 115         unsigned long           list_nr;
 116         int                     numa_node;
 117         unsigned int            msi_domain_flags;
 118         u32                     pre_its_base; /* for Socionext Synquacer */
 119         int                     vlpi_redist_offset;
 120 };
 121
 122 #define is_v4(its)              (!!((its)->typer & GITS_TYPER_VLPIS))
 123 #define is_v4_1(its)            (!!((its)->typer & GITS_TYPER_VMAPP))
 124 #define device_ids(its)         (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
 125
 126 #define ITS_ITT_ALIGN           SZ_256
 127
 128 /* The maximum number of VPEID bits supported by VLPI commands */
 129 #define ITS_MAX_VPEID_BITS                                              \
 130         ({                                                              \
 131                 int nvpeid = 16;                                        \
 132                 if (gic_rdists->has_rvpeid &&                           \
 133                     gic_rdists->gicd_typer2 & GICD_TYPER2_VIL)          \
 134                         nvpeid = 1 + (gic_rdists->gicd_typer2 &         \
 135                                       GICD_TYPER2_VID);                 \
 136                                                                         \
 137                 nvpeid;                                                 \
 138         })
 139 #define ITS_MAX_VPEID           (1 << (ITS_MAX_VPEID_BITS))
 140
 141 /* Convert page order to size in bytes */
 142 #define PAGE_ORDER_TO_SIZE(o)   (PAGE_SIZE << (o))
 143
 144 struct event_lpi_map {
 145         unsigned long           *lpi_map;
 146         u16                     *col_map;
 147         irq_hw_number_t         lpi_base;
 148         int                     nr_lpis;
 149         raw_spinlock_t          vlpi_lock;
 150         struct its_vm           *vm;
 151         struct its_vlpi_map     *vlpi_maps;
 152         int                     nr_vlpis;
 153 };
 154
 155 /*
 156  * The ITS view of a device - belongs to an ITS, owns an interrupt
 157  * translation table, and a list of interrupts.  If it some of its
 158  * LPIs are injected into a guest (GICv4), the event_map.vm field
 159  * indicates which one.
 160  */
 161 struct its_device {
 162         struct list_head        entry;
 163         struct its_node         *its;
 164         struct event_lpi_map    event_map;
 165         void                    *itt;
 166         u32                     nr_ites;
 167         u32                     device_id;
 168         bool                    shared;
 169 };
 170
 171 static struct {
 172         raw_spinlock_t          lock;
 173         struct its_device       *dev;
 174         struct its_vpe          **vpes;
 175         int                     next_victim;
 176 } vpe_proxy;
 177
 178 struct cpu_lpi_count {
 179         atomic_t        managed;
 180         atomic_t        unmanaged;
 181 };
 182
 183 static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
 184
 185 static LIST_HEAD(its_nodes);
 186 static DEFINE_RAW_SPINLOCK(its_lock);
 187 static struct rdists *gic_rdists;
 188 static struct irq_domain *its_parent;
 189
 190 static unsigned long its_list_map;
 191 static u16 vmovp_seq_num;
 192 static DEFINE_RAW_SPINLOCK(vmovp_lock);
 193
 194 static DEFINE_IDA(its_vpeid_ida);
 195
 196 #define gic_data_rdist()                (raw_cpu_ptr(gic_rdists->rdist))
 197 #define gic_data_rdist_cpu(cpu)         (per_cpu_ptr(gic_rdists->rdist, cpu))
 198 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
 199 #define gic_data_rdist_vlpi_base()      (gic_data_rdist_rd_base() + SZ_128K)
 200
 201 /*
 202  * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
 203  * always have vSGIs mapped.
 204  */
 205 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
 206 {
 207         return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
 208 }
 209
 210 static u16 get_its_list(struct its_vm *vm)
 211 {
 212         struct its_node *its;
 213         unsigned long its_list = 0;
 214
 215         list_for_each_entry(its, &its_nodes, entry) {
 216                 if (!is_v4(its))
 217                         continue;
 218
 219                 if (require_its_list_vmovp(vm, its))
 220                         __set_bit(its->list_nr, &its_list);
 221         }
 222
 223         return (u16)its_list;
 224 }
 225
 226 static inline u32 its_get_event_id(struct irq_data *d)
 227 {
 228         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 229         return d->hwirq - its_dev->event_map.lpi_base;
 230 }
 231
 232 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
 233                                                u32 event)
 234 {
 235         struct its_node *its = its_dev->its;
 236
 237         return its->collections + its_dev->event_map.col_map[event];
 238 }
 239
 240 static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
 241                                                u32 event)
 242 {
 243         if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
 244                 return NULL;
 245
 246         return &its_dev->event_map.vlpi_maps[event];
 247 }
 248
 249 static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
 250 {
 251         if (irqd_is_forwarded_to_vcpu(d)) {
 252                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 253                 u32 event = its_get_event_id(d);
 254
 255                 return dev_event_to_vlpi_map(its_dev, event);
 256         }
 257
 258         return NULL;
 259 }
 260
 261 static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
 262 {
 263         raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
 264         return vpe->col_idx;
 265 }
 266
 267 static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
 268 {
 269         raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
 270 }
 271
 272 static struct irq_chip its_vpe_irq_chip;
 273
 274 static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
 275 {
 276         struct its_vpe *vpe = NULL;
 277         int cpu;
 278
 279         if (d->chip == &its_vpe_irq_chip) {
 280                 vpe = irq_data_get_irq_chip_data(d);
 281         } else {
 282                 struct its_vlpi_map *map = get_vlpi_map(d);
 283                 if (map)
 284                         vpe = map->vpe;
 285         }
 286
 287         if (vpe) {
 288                 cpu = vpe_to_cpuid_lock(vpe, flags);
 289         } else {
 290                 /* Physical LPIs are already locked via the irq_desc lock */
 291                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 292                 cpu = its_dev->event_map.col_map[its_get_event_id(d)];
 293                 /* Keep GCC quiet... */
 294                 *flags = 0;
 295         }
 296
 297         return cpu;
 298 }
 299
 300 static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
 301 {
 302         struct its_vpe *vpe = NULL;
 303
 304         if (d->chip == &its_vpe_irq_chip) {
 305                 vpe = irq_data_get_irq_chip_data(d);
 306         } else {
 307                 struct its_vlpi_map *map = get_vlpi_map(d);
 308                 if (map)
 309                         vpe = map->vpe;
 310         }
 311
 312         if (vpe)
 313                 vpe_to_cpuid_unlock(vpe, flags);
 314 }
 315
 316 static struct its_collection *valid_col(struct its_collection *col)
 317 {
 318         if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
 319                 return NULL;
 320
 321         return col;
 322 }
 323
 324 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
 325 {
 326         if (valid_col(its->collections + vpe->col_idx))
 327                 return vpe;
 328
 329         return NULL;
 330 }
 331
 332 /*
 333  * ITS command descriptors - parameters to be encoded in a command
 334  * block.
 335  */
 336 struct its_cmd_desc {
 337         union {
 338                 struct {
 339                         struct its_device *dev;
 340                         u32 event_id;
 341                 } its_inv_cmd;
 342
 343                 struct {
 344                         struct its_device *dev;
 345                         u32 event_id;
 346                 } its_clear_cmd;
 347
 348                 struct {
 349                         struct its_device *dev;
 350                         u32 event_id;
 351                 } its_int_cmd;
 352
 353                 struct {
 354                         struct its_device *dev;
 355                         int valid;
 356                 } its_mapd_cmd;
 357
 358                 struct {
 359                         struct its_collection *col;
 360                         int valid;
 361                 } its_mapc_cmd;
 362
 363                 struct {
 364                         struct its_device *dev;
 365                         u32 phys_id;
 366                         u32 event_id;
 367                 } its_mapti_cmd;
 368
 369                 struct {
 370                         struct its_device *dev;
 371                         struct its_collection *col;
 372                         u32 event_id;
 373                 } its_movi_cmd;
 374
 375                 struct {
 376                         struct its_device *dev;
 377                         u32 event_id;
 378                 } its_discard_cmd;
 379
 380                 struct {
 381                         struct its_collection *col;
 382                 } its_invall_cmd;
 383
 384                 struct {
 385                         struct its_vpe *vpe;
 386                 } its_vinvall_cmd;
 387
 388                 struct {
 389                         struct its_vpe *vpe;
 390                         struct its_collection *col;
 391                         bool valid;
 392                 } its_vmapp_cmd;
 393
 394                 struct {
 395                         struct its_vpe *vpe;
 396                         struct its_device *dev;
 397                         u32 virt_id;
 398                         u32 event_id;
 399                         bool db_enabled;
 400                 } its_vmapti_cmd;
 401
 402                 struct {
 403                         struct its_vpe *vpe;
 404                         struct its_device *dev;
 405                         u32 event_id;
 406                         bool db_enabled;
 407                 } its_vmovi_cmd;
 408
 409                 struct {
 410                         struct its_vpe *vpe;
 411                         struct its_collection *col;
 412                         u16 seq_num;
 413                         u16 its_list;
 414                 } its_vmovp_cmd;
 415
 416                 struct {
 417                         struct its_vpe *vpe;
 418                 } its_invdb_cmd;
 419
 420                 struct {
 421                         struct its_vpe *vpe;
 422                         u8 sgi;
 423                         u8 priority;
 424                         bool enable;
 425                         bool group;
 426                         bool clear;
 427                 } its_vsgi_cmd;
 428         };
 429 };
 430
 431 /*
 432  * The ITS command block, which is what the ITS actually parses.
 433  */
 434 struct its_cmd_block {
 435         union {
 436                 u64     raw_cmd[4];
 437                 __le64  raw_cmd_le[4];
 438         };
 439 };
 440
 441 #define ITS_CMD_QUEUE_SZ                SZ_64K
 442 #define ITS_CMD_QUEUE_NR_ENTRIES        (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
 443
 444 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
 445                                                     struct its_cmd_block *,
 446                                                     struct its_cmd_desc *);
 447
 448 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
 449                                               struct its_cmd_block *,
 450                                               struct its_cmd_desc *);
 451
 452 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
 453 {
 454         u64 mask = GENMASK_ULL(h, l);
 455         *raw_cmd &= ~mask;
 456         *raw_cmd |= (val << l) & mask;
 457 }
 458
 459 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
 460 {
 461         its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
 462 }
 463
 464 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
 465 {
 466         its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
 467 }
 468
 469 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
 470 {
 471         its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
 472 }
 473
 474 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
 475 {
 476         its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
 477 }
 478
 479 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
 480 {
 481         its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
 482 }
 483
 484 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
 485 {
 486         its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
 487 }
 488
 489 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
 490 {
 491         its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
 492 }
 493
 494 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
 495 {
 496         its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
 497 }
 498
 499 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
 500 {
 501         its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
 502 }
 503
 504 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
 505 {
 506         its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
 507 }
 508
 509 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
 510 {
 511         its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
 512 }
 513
 514 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
 515 {
 516         its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
 517 }
 518
 519 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
 520 {
 521         its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
 522 }
 523
 524 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
 525 {
 526         its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
 527 }
 528
 529 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
 530 {
 531         its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
 532 }
 533
 534 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
 535 {
 536         its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
 537 }
 538
 539 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
 540 {
 541         its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
 542 }
 543
 544 static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
 545 {
 546         its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
 547 }
 548
 549 static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
 550 {
 551         its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
 552 }
 553
 554 static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
 555 {
 556         its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
 557 }
 558
 559 static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
 560                                         u32 vpe_db_lpi)
 561 {
 562         its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
 563 }
 564
 565 static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
 566                                         u32 vpe_db_lpi)
 567 {
 568         its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
 569 }
 570
 571 static void its_encode_db(struct its_cmd_block *cmd, bool db)
 572 {
 573         its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
 574 }
 575
 576 static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
 577 {
 578         its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
 579 }
 580
 581 static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
 582 {
 583         its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
 584 }
 585
 586 static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
 587 {
 588         its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
 589 }
 590
 591 static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
 592 {
 593         its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
 594 }
 595
 596 static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
 597 {
 598         its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
 599 }
 600
 601 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
 602 {
 603         /* Let's fixup BE commands */
 604         cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
 605         cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
 606         cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
 607         cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
 608 }
 609
 610 static struct its_collection *its_build_mapd_cmd(struct its_node *its,
 611                                                  struct its_cmd_block *cmd,
 612                                                  struct its_cmd_desc *desc)
 613 {
 614         unsigned long itt_addr;
 615         u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
 616
 617         itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
 618         itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
 619
 620         its_encode_cmd(cmd, GITS_CMD_MAPD);
 621         its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
 622         its_encode_size(cmd, size - 1);
 623         its_encode_itt(cmd, itt_addr);
 624         its_encode_valid(cmd, desc->its_mapd_cmd.valid);
 625
 626         its_fixup_cmd(cmd);
 627
 628         return NULL;
 629 }
 630
 631 static struct its_collection *its_build_mapc_cmd(struct its_node *its,
 632                                                  struct its_cmd_block *cmd,
 633                                                  struct its_cmd_desc *desc)
 634 {
 635         its_encode_cmd(cmd, GITS_CMD_MAPC);
 636         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
 637         its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
 638         its_encode_valid(cmd, desc->its_mapc_cmd.valid);
 639
 640         its_fixup_cmd(cmd);
 641
 642         return desc->its_mapc_cmd.col;
 643 }
 644
 645 static struct its_collection *its_build_mapti_cmd(struct its_node *its,
 646                                                   struct its_cmd_block *cmd,
 647                                                   struct its_cmd_desc *desc)
 648 {
 649         struct its_collection *col;
 650
 651         col = dev_event_to_col(desc->its_mapti_cmd.dev,
 652                                desc->its_mapti_cmd.event_id);
 653
 654         its_encode_cmd(cmd, GITS_CMD_MAPTI);
 655         its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
 656         its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
 657         its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
 658         its_encode_collection(cmd, col->col_id);
 659
 660         its_fixup_cmd(cmd);
 661
 662         return valid_col(col);
 663 }
 664
 665 static struct its_collection *its_build_movi_cmd(struct its_node *its,
 666                                                  struct its_cmd_block *cmd,
 667                                                  struct its_cmd_desc *desc)
 668 {
 669         struct its_collection *col;
 670
 671         col = dev_event_to_col(desc->its_movi_cmd.dev,
 672                                desc->its_movi_cmd.event_id);
 673
 674         its_encode_cmd(cmd, GITS_CMD_MOVI);
 675         its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
 676         its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
 677         its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
 678
 679         its_fixup_cmd(cmd);
 680
 681         return valid_col(col);
 682 }
 683
 684 static struct its_collection *its_build_discard_cmd(struct its_node *its,
 685                                                     struct its_cmd_block *cmd,
 686                                                     struct its_cmd_desc *desc)
 687 {
 688         struct its_collection *col;
 689
 690         col = dev_event_to_col(desc->its_discard_cmd.dev,
 691                                desc->its_discard_cmd.event_id);
 692
 693         its_encode_cmd(cmd, GITS_CMD_DISCARD);
 694         its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
 695         its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
 696
 697         its_fixup_cmd(cmd);
 698
 699         return valid_col(col);
 700 }
 701
 702 static struct its_collection *its_build_inv_cmd(struct its_node *its,
 703                                                 struct its_cmd_block *cmd,
 704                                                 struct its_cmd_desc *desc)
 705 {
 706         struct its_collection *col;
 707
 708         col = dev_event_to_col(desc->its_inv_cmd.dev,
 709                                desc->its_inv_cmd.event_id);
 710
 711         its_encode_cmd(cmd, GITS_CMD_INV);
 712         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 713         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 714
 715         its_fixup_cmd(cmd);
 716
 717         return valid_col(col);
 718 }
 719
 720 static struct its_collection *its_build_int_cmd(struct its_node *its,
 721                                                 struct its_cmd_block *cmd,
 722                                                 struct its_cmd_desc *desc)
 723 {
 724         struct its_collection *col;
 725
 726         col = dev_event_to_col(desc->its_int_cmd.dev,
 727                                desc->its_int_cmd.event_id);
 728
 729         its_encode_cmd(cmd, GITS_CMD_INT);
 730         its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
 731         its_encode_event_id(cmd, desc->its_int_cmd.event_id);
 732
 733         its_fixup_cmd(cmd);
 734
 735         return valid_col(col);
 736 }
 737
 738 static struct its_collection *its_build_clear_cmd(struct its_node *its,
 739                                                   struct its_cmd_block *cmd,
 740                                                   struct its_cmd_desc *desc)
 741 {
 742         struct its_collection *col;
 743
 744         col = dev_event_to_col(desc->its_clear_cmd.dev,
 745                                desc->its_clear_cmd.event_id);
 746
 747         its_encode_cmd(cmd, GITS_CMD_CLEAR);
 748         its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
 749         its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
 750
 751         its_fixup_cmd(cmd);
 752
 753         return valid_col(col);
 754 }
 755
 756 static struct its_collection *its_build_invall_cmd(struct its_node *its,
 757                                                    struct its_cmd_block *cmd,
 758                                                    struct its_cmd_desc *desc)
 759 {
 760         its_encode_cmd(cmd, GITS_CMD_INVALL);
 761         its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
 762
 763         its_fixup_cmd(cmd);
 764
 765         return desc->its_invall_cmd.col;
 766 }
 767
 768 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
 769                                              struct its_cmd_block *cmd,
 770                                              struct its_cmd_desc *desc)
 771 {
 772         its_encode_cmd(cmd, GITS_CMD_VINVALL);
 773         its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
 774
 775         its_fixup_cmd(cmd);
 776
 777         return valid_vpe(its, desc->its_vinvall_cmd.vpe);
 778 }
 779
 780 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
 781                                            struct its_cmd_block *cmd,
 782                                            struct its_cmd_desc *desc)
 783 {
 784         unsigned long vpt_addr, vconf_addr;
 785         u64 target;
 786         bool alloc;
 787
 788         its_encode_cmd(cmd, GITS_CMD_VMAPP);
 789         its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
 790         its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
 791
 792         if (!desc->its_vmapp_cmd.valid) {
 793                 if (is_v4_1(its)) {
 794                         alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
 795                         its_encode_alloc(cmd, alloc);
 796                 }
 797
 798                 goto out;
 799         }
 800
 801         vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
 802         target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
 803
 804         its_encode_target(cmd, target);
 805         its_encode_vpt_addr(cmd, vpt_addr);
 806         its_encode_vpt_size(cmd, LPI_NRBITS - 1);
 807
 808         if (!is_v4_1(its))
 809                 goto out;
 810
 811         vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
 812
 813         alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
 814
 815         its_encode_alloc(cmd, alloc);
 816
 817         /*
 818          * GICv4.1 provides a way to get the VLPI state, which needs the vPE
 819          * to be unmapped first, and in this case, we may remap the vPE
 820          * back while the VPT is not empty. So we can't assume that the
 821          * VPT is empty on map. This is why we never advertise PTZ.
 822          */
 823         its_encode_ptz(cmd, false);
 824         its_encode_vconf_addr(cmd, vconf_addr);
 825         its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
 826
 827 out:
 828         its_fixup_cmd(cmd);
 829
 830         return valid_vpe(its, desc->its_vmapp_cmd.vpe);
 831 }
 832
 833 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
 834                                             struct its_cmd_block *cmd,
 835                                             struct its_cmd_desc *desc)
 836 {
 837         u32 db;
 838
 839         if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
 840                 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
 841         else
 842                 db = 1023;
 843
 844         its_encode_cmd(cmd, GITS_CMD_VMAPTI);
 845         its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
 846         its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
 847         its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
 848         its_encode_db_phys_id(cmd, db);
 849         its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
 850
 851         its_fixup_cmd(cmd);
 852
 853         return valid_vpe(its, desc->its_vmapti_cmd.vpe);
 854 }
 855
 856 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
 857                                            struct its_cmd_block *cmd,
 858                                            struct its_cmd_desc *desc)
 859 {
 860         u32 db;
 861
 862         if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
 863                 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
 864         else
 865                 db = 1023;
 866
 867         its_encode_cmd(cmd, GITS_CMD_VMOVI);
 868         its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
 869         its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
 870         its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
 871         its_encode_db_phys_id(cmd, db);
 872         its_encode_db_valid(cmd, true);
 873
 874         its_fixup_cmd(cmd);
 875
 876         return valid_vpe(its, desc->its_vmovi_cmd.vpe);
 877 }
 878
 879 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
 880                                            struct its_cmd_block *cmd,
 881                                            struct its_cmd_desc *desc)
 882 {
 883         u64 target;
 884
 885         target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
 886         its_encode_cmd(cmd, GITS_CMD_VMOVP);
 887         its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
 888         its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
 889         its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
 890         its_encode_target(cmd, target);
 891
 892         if (is_v4_1(its)) {
 893                 its_encode_db(cmd, true);
 894                 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
 895         }
 896
 897         its_fixup_cmd(cmd);
 898
 899         return valid_vpe(its, desc->its_vmovp_cmd.vpe);
 900 }
 901
 902 static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
 903                                           struct its_cmd_block *cmd,
 904                                           struct its_cmd_desc *desc)
 905 {
 906         struct its_vlpi_map *map;
 907
 908         map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
 909                                     desc->its_inv_cmd.event_id);
 910
 911         its_encode_cmd(cmd, GITS_CMD_INV);
 912         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 913         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 914
 915         its_fixup_cmd(cmd);
 916
 917         return valid_vpe(its, map->vpe);
 918 }
 919
 920 static struct its_vpe *its_build_vint_cmd(struct its_node *its,
 921                                           struct its_cmd_block *cmd,
 922                                           struct its_cmd_desc *desc)
 923 {
 924         struct its_vlpi_map *map;
 925
 926         map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
 927                                     desc->its_int_cmd.event_id);
 928
 929         its_encode_cmd(cmd, GITS_CMD_INT);
 930         its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
 931         its_encode_event_id(cmd, desc->its_int_cmd.event_id);
 932
 933         its_fixup_cmd(cmd);
 934
 935         return valid_vpe(its, map->vpe);
 936 }
 937
 938 static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
 939                                             struct its_cmd_block *cmd,
 940                                             struct its_cmd_desc *desc)
 941 {
 942         struct its_vlpi_map *map;
 943
 944         map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
 945                                     desc->its_clear_cmd.event_id);
 946
 947         its_encode_cmd(cmd, GITS_CMD_CLEAR);
 948         its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
 949         its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
 950
 951         its_fixup_cmd(cmd);
 952
 953         return valid_vpe(its, map->vpe);
 954 }
 955
 956 static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
 957                                            struct its_cmd_block *cmd,
 958                                            struct its_cmd_desc *desc)
 959 {
 960         if (WARN_ON(!is_v4_1(its)))
 961                 return NULL;
 962
 963         its_encode_cmd(cmd, GITS_CMD_INVDB);
 964         its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
 965
 966         its_fixup_cmd(cmd);
 967
 968         return valid_vpe(its, desc->its_invdb_cmd.vpe);
 969 }
 970
 971 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
 972                                           struct its_cmd_block *cmd,
 973                                           struct its_cmd_desc *desc)
 974 {
 975         if (WARN_ON(!is_v4_1(its)))
 976                 return NULL;
 977
 978         its_encode_cmd(cmd, GITS_CMD_VSGI);
 979         its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
 980         its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
 981         its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
 982         its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
 983         its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
 984         its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
 985
 986         its_fixup_cmd(cmd);
 987
 988         return valid_vpe(its, desc->its_vsgi_cmd.vpe);
 989 }
 990
 991 static u64 its_cmd_ptr_to_offset(struct its_node *its,
 992                                  struct its_cmd_block *ptr)
 993 {
 994         return (ptr - its->cmd_base) * sizeof(*ptr);
 995 }
 996
 997 static int its_queue_full(struct its_node *its)
 998 {
 999         int widx;
1000         int ridx;
1001
1002         widx = its->cmd_write - its->cmd_base;
1003         ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
1004
1005         /* This is incredibly unlikely to happen, unless the ITS locks up. */
1006         if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
1007                 return 1;
1008
1009         return 0;
1010 }
1011
1012 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
1013 {
1014         struct its_cmd_block *cmd;
1015         u32 count = 1000000;    /* 1s! */
1016
1017         while (its_queue_full(its)) {
1018                 count--;
1019                 if (!count) {
1020                         pr_err_ratelimited("ITS queue not draining\n");
1021                         return NULL;
1022                 }
1023                 cpu_relax();
1024                 udelay(1);
1025         }
1026
1027         cmd = its->cmd_write++;
1028
1029         /* Handle queue wrapping */
1030         if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
1031                 its->cmd_write = its->cmd_base;
1032
1033         /* Clear command  */
1034         cmd->raw_cmd[0] = 0;
1035         cmd->raw_cmd[1] = 0;
1036         cmd->raw_cmd[2] = 0;
1037         cmd->raw_cmd[3] = 0;
1038
1039         return cmd;
1040 }
1041
1042 static struct its_cmd_block *its_post_commands(struct its_node *its)
1043 {
1044         u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
1045
1046         writel_relaxed(wr, its->base + GITS_CWRITER);
1047
1048         return its->cmd_write;
1049 }
1050
1051 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
1052 {
1053         /*
1054          * Make sure the commands written to memory are observable by
1055          * the ITS.
1056          */
1057         if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
1058                 gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
1059         else
1060                 dsb(ishst);
1061 }
1062
1063 static int its_wait_for_range_completion(struct its_node *its,
1064                                          u64    prev_idx,
1065                                          struct its_cmd_block *to)
1066 {
1067         u64 rd_idx, to_idx, linear_idx;
1068         u32 count = 1000000;    /* 1s! */
1069
1070         /* Linearize to_idx if the command set has wrapped around */
1071         to_idx = its_cmd_ptr_to_offset(its, to);
1072         if (to_idx < prev_idx)
1073                 to_idx += ITS_CMD_QUEUE_SZ;
1074
1075         linear_idx = prev_idx;
1076
1077         while (1) {
1078                 s64 delta;
1079
1080                 rd_idx = readl_relaxed(its->base + GITS_CREADR);
1081
1082                 /*
1083                  * Compute the read pointer progress, taking the
1084                  * potential wrap-around into account.
1085                  */
1086                 delta = rd_idx - prev_idx;
1087                 if (rd_idx < prev_idx)
1088                         delta += ITS_CMD_QUEUE_SZ;
1089
1090                 linear_idx += delta;
1091                 if (linear_idx >= to_idx)
1092                         break;
1093
1094                 count--;
1095                 if (!count) {
1096                         pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1097                                            to_idx, linear_idx);
1098                         return -1;
1099                 }
1100                 prev_idx = rd_idx;
1101                 cpu_relax();
1102                 udelay(1);
1103         }
1104
1105         return 0;
1106 }
1107
1108 /* Warning, macro hell follows */
1109 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn)       \
1110 void name(struct its_node *its,                                         \
1111           buildtype builder,                                            \
1112           struct its_cmd_desc *desc)                                    \
1113 {                                                                       \
1114         struct its_cmd_block *cmd, *sync_cmd, *next_cmd;                \
1115         synctype *sync_obj;                                             \
1116         unsigned long flags;                                            \
1117         u64 rd_idx;                                                     \
1118                                                                         \
1119         raw_spin_lock_irqsave(&its->lock, flags);                       \
1120                                                                         \
1121         cmd = its_allocate_entry(its);                                  \
1122         if (!cmd) {             /* We're soooooo screewed... */         \
1123                 raw_spin_unlock_irqrestore(&its->lock, flags);          \
1124                 return;                                                 \
1125         }                                                               \
1126         sync_obj = builder(its, cmd, desc);                             \
1127         its_flush_cmd(its, cmd);                                        \
1128                                                                         \
1129         if (sync_obj) {                                                 \
1130                 sync_cmd = its_allocate_entry(its);                     \
1131                 if (!sync_cmd)                                          \
1132                         goto post;                                      \
1133                                                                         \
1134                 buildfn(its, sync_cmd, sync_obj);                       \
1135                 its_flush_cmd(its, sync_cmd);                           \
1136         }                                                               \
1137                                                                         \
1138 post:                                                                   \
1139         rd_idx = readl_relaxed(its->base + GITS_CREADR);                \
1140         next_cmd = its_post_commands(its);                              \
1141         raw_spin_unlock_irqrestore(&its->lock, flags);                  \
1142                                                                         \
1143         if (its_wait_for_range_completion(its, rd_idx, next_cmd))       \
1144                 pr_err_ratelimited("ITS cmd %ps failed\n", builder);    \
1145 }
1146
1147 static void its_build_sync_cmd(struct its_node *its,
1148                                struct its_cmd_block *sync_cmd,
1149                                struct its_collection *sync_col)
1150 {
1151         its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
1152         its_encode_target(sync_cmd, sync_col->target_address);
1153
1154         its_fixup_cmd(sync_cmd);
1155 }
1156
1157 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
1158                              struct its_collection, its_build_sync_cmd)
1159
1160 static void its_build_vsync_cmd(struct its_node *its,
1161                                 struct its_cmd_block *sync_cmd,
1162                                 struct its_vpe *sync_vpe)
1163 {
1164         its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
1165         its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
1166
1167         its_fixup_cmd(sync_cmd);
1168 }
1169
1170 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
1171                              struct its_vpe, its_build_vsync_cmd)
1172
1173 static void its_send_int(struct its_device *dev, u32 event_id)
1174 {
1175         struct its_cmd_desc desc;
1176
1177         desc.its_int_cmd.dev = dev;
1178         desc.its_int_cmd.event_id = event_id;
1179
1180         its_send_single_command(dev->its, its_build_int_cmd, &desc);
1181 }
1182
1183 static void its_send_clear(struct its_device *dev, u32 event_id)
1184 {
1185         struct its_cmd_desc desc;
1186
1187         desc.its_clear_cmd.dev = dev;
1188         desc.its_clear_cmd.event_id = event_id;
1189
1190         its_send_single_command(dev->its, its_build_clear_cmd, &desc);
1191 }
1192
1193 static void its_send_inv(struct its_device *dev, u32 event_id)
1194 {
1195         struct its_cmd_desc desc;
1196
1197         desc.its_inv_cmd.dev = dev;
1198         desc.its_inv_cmd.event_id = event_id;
1199
1200         its_send_single_command(dev->its, its_build_inv_cmd, &desc);
1201 }
1202
1203 static void its_send_mapd(struct its_device *dev, int valid)
1204 {
1205         struct its_cmd_desc desc;
1206
1207         desc.its_mapd_cmd.dev = dev;
1208         desc.its_mapd_cmd.valid = !!valid;
1209
1210         its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
1211 }
1212
1213 static void its_send_mapc(struct its_node *its, struct its_collection *col,
1214                           int valid)
1215 {
1216         struct its_cmd_desc desc;
1217
1218         desc.its_mapc_cmd.col = col;
1219         desc.its_mapc_cmd.valid = !!valid;
1220
1221         its_send_single_command(its, its_build_mapc_cmd, &desc);
1222 }
1223
1224 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
1225 {
1226         struct its_cmd_desc desc;
1227
1228         desc.its_mapti_cmd.dev = dev;
1229         desc.its_mapti_cmd.phys_id = irq_id;
1230         desc.its_mapti_cmd.event_id = id;
1231
1232         its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
1233 }
1234
1235 static void its_send_movi(struct its_device *dev,
1236                           struct its_collection *col, u32 id)
1237 {
1238         struct its_cmd_desc desc;
1239
1240         desc.its_movi_cmd.dev = dev;
1241         desc.its_movi_cmd.col = col;
1242         desc.its_movi_cmd.event_id = id;
1243
1244         its_send_single_command(dev->its, its_build_movi_cmd, &desc);
1245 }
1246
1247 static void its_send_discard(struct its_device *dev, u32 id)
1248 {
1249         struct its_cmd_desc desc;
1250
1251         desc.its_discard_cmd.dev = dev;
1252         desc.its_discard_cmd.event_id = id;
1253
1254         its_send_single_command(dev->its, its_build_discard_cmd, &desc);
1255 }
1256
1257 static void its_send_invall(struct its_node *its, struct its_collection *col)
1258 {
1259         struct its_cmd_desc desc;
1260
1261         desc.its_invall_cmd.col = col;
1262
1263         its_send_single_command(its, its_build_invall_cmd, &desc);
1264 }
1265
1266 static void its_send_vmapti(struct its_device *dev, u32 id)
1267 {
1268         struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1269         struct its_cmd_desc desc;
1270
1271         desc.its_vmapti_cmd.vpe = map->vpe;
1272         desc.its_vmapti_cmd.dev = dev;
1273         desc.its_vmapti_cmd.virt_id = map->vintid;
1274         desc.its_vmapti_cmd.event_id = id;
1275         desc.its_vmapti_cmd.db_enabled = map->db_enabled;
1276
1277         its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
1278 }
1279
1280 static void its_send_vmovi(struct its_device *dev, u32 id)
1281 {
1282         struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1283         struct its_cmd_desc desc;
1284
1285         desc.its_vmovi_cmd.vpe = map->vpe;
1286         desc.its_vmovi_cmd.dev = dev;
1287         desc.its_vmovi_cmd.event_id = id;
1288         desc.its_vmovi_cmd.db_enabled = map->db_enabled;
1289
1290         its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
1291 }
1292
1293 static void its_send_vmapp(struct its_node *its,
1294                            struct its_vpe *vpe, bool valid)
1295 {
1296         struct its_cmd_desc desc;
1297
1298         desc.its_vmapp_cmd.vpe = vpe;
1299         desc.its_vmapp_cmd.valid = valid;
1300         desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
1301
1302         its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
1303 }
1304
1305 static void its_send_vmovp(struct its_vpe *vpe)
1306 {
1307         struct its_cmd_desc desc = {};
1308         struct its_node *its;
1309         unsigned long flags;
1310         int col_id = vpe->col_idx;
1311
1312         desc.its_vmovp_cmd.vpe = vpe;
1313
1314         if (!its_list_map) {
1315                 its = list_first_entry(&its_nodes, struct its_node, entry);
1316                 desc.its_vmovp_cmd.col = &its->collections[col_id];
1317                 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1318                 return;
1319         }
1320
1321         /*
1322          * Yet another marvel of the architecture. If using the
1323          * its_list "feature", we need to make sure that all ITSs
1324          * receive all VMOVP commands in the same order. The only way
1325          * to guarantee this is to make vmovp a serialization point.
1326          *
1327          * Wall <-- Head.
1328          */
1329         raw_spin_lock_irqsave(&vmovp_lock, flags);
1330
1331         desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
1332         desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
1333
1334         /* Emit VMOVPs */
1335         list_for_each_entry(its, &its_nodes, entry) {
1336                 if (!is_v4(its))
1337                         continue;
1338
1339                 if (!require_its_list_vmovp(vpe->its_vm, its))
1340                         continue;
1341
1342                 desc.its_vmovp_cmd.col = &its->collections[col_id];
1343                 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1344         }
1345
1346         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1347 }
1348
1349 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1350 {
1351         struct its_cmd_desc desc;
1352
1353         desc.its_vinvall_cmd.vpe = vpe;
1354         its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1355 }
1356
1357 static void its_send_vinv(struct its_device *dev, u32 event_id)
1358 {
1359         struct its_cmd_desc desc;
1360
1361         /*
1362          * There is no real VINV command. This is just a normal INV,
1363          * with a VSYNC instead of a SYNC.
1364          */
1365         desc.its_inv_cmd.dev = dev;
1366         desc.its_inv_cmd.event_id = event_id;
1367
1368         its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
1369 }
1370
1371 static void its_send_vint(struct its_device *dev, u32 event_id)
1372 {
1373         struct its_cmd_desc desc;
1374
1375         /*
1376          * There is no real VINT command. This is just a normal INT,
1377          * with a VSYNC instead of a SYNC.
1378          */
1379         desc.its_int_cmd.dev = dev;
1380         desc.its_int_cmd.event_id = event_id;
1381
1382         its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
1383 }
1384
1385 static void its_send_vclear(struct its_device *dev, u32 event_id)
1386 {
1387         struct its_cmd_desc desc;
1388
1389         /*
1390          * There is no real VCLEAR command. This is just a normal CLEAR,
1391          * with a VSYNC instead of a SYNC.
1392          */
1393         desc.its_clear_cmd.dev = dev;
1394         desc.its_clear_cmd.event_id = event_id;
1395
1396         its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
1397 }
1398
1399 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1400 {
1401         struct its_cmd_desc desc;
1402
1403         desc.its_invdb_cmd.vpe = vpe;
1404         its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
1405 }
1406
1407 /*
1408  * irqchip functions - assumes MSI, mostly.
1409  */
1410 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1411 {
1412         struct its_vlpi_map *map = get_vlpi_map(d);
1413         irq_hw_number_t hwirq;
1414         void *va;
1415         u8 *cfg;
1416
1417         if (map) {
1418                 va = page_address(map->vm->vprop_page);
1419                 hwirq = map->vintid;
1420
1421                 /* Remember the updated property */
1422                 map->properties &= ~clr;
1423                 map->properties |= set | LPI_PROP_GROUP1;
1424         } else {
1425                 va = gic_rdists->prop_table_va;
1426                 hwirq = d->hwirq;
1427         }
1428
1429         cfg = va + hwirq - 8192;
1430         *cfg &= ~clr;
1431         *cfg |= set | LPI_PROP_GROUP1;
1432
1433         /*
1434          * Make the above write visible to the redistributors.
1435          * And yes, we're flushing exactly: One. Single. Byte.
1436          * Humpf...
1437          */
1438         if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
1439                 gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
1440         else
1441                 dsb(ishst);
1442 }
1443
1444 static void wait_for_syncr(void __iomem *rdbase)
1445 {
1446         while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
1447                 cpu_relax();
1448 }
1449
1450 static void __direct_lpi_inv(struct irq_data *d, u64 val)
1451 {
1452         void __iomem *rdbase;
1453         unsigned long flags;
1454         int cpu;
1455
1456         /* Target the redistributor this LPI is currently routed to */
1457         cpu = irq_to_cpuid_lock(d, &flags);
1458         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
1459
1460         rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
1461         gic_write_lpir(val, rdbase + GICR_INVLPIR);
1462         wait_for_syncr(rdbase);
1463
1464         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
1465         irq_to_cpuid_unlock(d, flags);
1466 }
1467
1468 static void direct_lpi_inv(struct irq_data *d)
1469 {
1470         struct its_vlpi_map *map = get_vlpi_map(d);
1471         u64 val;
1472
1473         if (map) {
1474                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1475
1476                 WARN_ON(!is_v4_1(its_dev->its));
1477
1478                 val  = GICR_INVLPIR_V;
1479                 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1480                 val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
1481         } else {
1482                 val = d->hwirq;
1483         }
1484
1485         __direct_lpi_inv(d, val);
1486 }
1487
1488 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1489 {
1490         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1491
1492         lpi_write_config(d, clr, set);
1493         if (gic_rdists->has_direct_lpi &&
1494             (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
1495                 direct_lpi_inv(d);
1496         else if (!irqd_is_forwarded_to_vcpu(d))
1497                 its_send_inv(its_dev, its_get_event_id(d));
1498         else
1499                 its_send_vinv(its_dev, its_get_event_id(d));
1500 }
1501
1502 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1503 {
1504         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1505         u32 event = its_get_event_id(d);
1506         struct its_vlpi_map *map;
1507
1508         /*
1509          * GICv4.1 does away with the per-LPI nonsense, nothing to do
1510          * here.
1511          */
1512         if (is_v4_1(its_dev->its))
1513                 return;
1514
1515         map = dev_event_to_vlpi_map(its_dev, event);
1516
1517         if (map->db_enabled == enable)
1518                 return;
1519
1520         map->db_enabled = enable;
1521
1522         /*
1523          * More fun with the architecture:
1524          *
1525          * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1526          * value or to 1023, depending on the enable bit. But that
1527          * would be issuing a mapping for an /existing/ DevID+EventID
1528          * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1529          * to the /same/ vPE, using this opportunity to adjust the
1530          * doorbell. Mouahahahaha. We loves it, Precious.
1531          */
1532         its_send_vmovi(its_dev, event);
1533 }
1534
1535 static void its_mask_irq(struct irq_data *d)
1536 {
1537         if (irqd_is_forwarded_to_vcpu(d))
1538                 its_vlpi_set_doorbell(d, false);
1539
1540         lpi_update_config(d, LPI_PROP_ENABLED, 0);
1541 }
1542
1543 static void its_unmask_irq(struct irq_data *d)
1544 {
1545         if (irqd_is_forwarded_to_vcpu(d))
1546                 its_vlpi_set_doorbell(d, true);
1547
1548         lpi_update_config(d, 0, LPI_PROP_ENABLED);
1549 }
1550
1551 static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
1552 {
1553         if (irqd_affinity_is_managed(d))
1554                 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1555
1556         return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1557 }
1558
1559 static void its_inc_lpi_count(struct irq_data *d, int cpu)
1560 {
1561         if (irqd_affinity_is_managed(d))
1562                 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1563         else
1564                 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1565 }
1566
1567 static void its_dec_lpi_count(struct irq_data *d, int cpu)
1568 {
1569         if (irqd_affinity_is_managed(d))
1570                 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1571         else
1572                 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1573 }
1574
1575 static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
1576                                               const struct cpumask *cpu_mask)
1577 {
1578         unsigned int cpu = nr_cpu_ids, tmp;
1579         int count = S32_MAX;
1580
1581         for_each_cpu(tmp, cpu_mask) {
1582                 int this_count = its_read_lpi_count(d, tmp);
1583                 if (this_count < count) {
1584                         cpu = tmp;
1585                         count = this_count;
1586                 }
1587         }
1588
1589         return cpu;
1590 }
1591
1592 /*
1593  * As suggested by Thomas Gleixner in:
1594  * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1595  */
1596 static int its_select_cpu(struct irq_data *d,
1597                           const struct cpumask *aff_mask)
1598 {
1599         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1600         static DEFINE_RAW_SPINLOCK(tmpmask_lock);
1601         static struct cpumask __tmpmask;
1602         struct cpumask *tmpmask;
1603         unsigned long flags;
1604         int cpu, node;
1605         node = its_dev->its->numa_node;
1606         tmpmask = &__tmpmask;
1607
1608         raw_spin_lock_irqsave(&tmpmask_lock, flags);
1609
1610         if (!irqd_affinity_is_managed(d)) {
1611                 /* First try the NUMA node */
1612                 if (node != NUMA_NO_NODE) {
1613                         /*
1614                          * Try the intersection of the affinity mask and the
1615                          * node mask (and the online mask, just to be safe).
1616                          */
1617                         cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
1618                         cpumask_and(tmpmask, tmpmask, cpu_online_mask);
1619
1620                         /*
1621                          * Ideally, we would check if the mask is empty, and
1622                          * try again on the full node here.
1623                          *
1624                          * But it turns out that the way ACPI describes the
1625                          * affinity for ITSs only deals about memory, and
1626                          * not target CPUs, so it cannot describe a single
1627                          * ITS placed next to two NUMA nodes.
1628                          *
1629                          * Instead, just fallback on the online mask. This
1630                          * diverges from Thomas' suggestion above.
1631                          */
1632                         cpu = cpumask_pick_least_loaded(d, tmpmask);
1633                         if (cpu < nr_cpu_ids)
1634                                 goto out;
1635
1636                         /* If we can't cross sockets, give up */
1637                         if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
1638                                 goto out;
1639
1640                         /* If the above failed, expand the search */
1641                 }
1642
1643                 /* Try the intersection of the affinity and online masks */
1644                 cpumask_and(tmpmask, aff_mask, cpu_online_mask);
1645
1646                 /* If that doesn't fly, the online mask is the last resort */
1647                 if (cpumask_empty(tmpmask))
1648                         cpumask_copy(tmpmask, cpu_online_mask);
1649
1650                 cpu = cpumask_pick_least_loaded(d, tmpmask);
1651         } else {
1652                 cpumask_copy(tmpmask, aff_mask);
1653
1654                 /* If we cannot cross sockets, limit the search to that node */
1655                 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
1656                     node != NUMA_NO_NODE)
1657                         cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
1658
1659                 cpu = cpumask_pick_least_loaded(d, tmpmask);
1660         }
1661 out:
1662         raw_spin_unlock_irqrestore(&tmpmask_lock, flags);
1663
1664         pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
1665         return cpu;
1666 }
1667
1668 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1669                             bool force)
1670 {
1671         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1672         struct its_collection *target_col;
1673         u32 id = its_get_event_id(d);
1674         int cpu, prev_cpu;
1675
1676         /* A forwarded interrupt should use irq_set_vcpu_affinity */
1677         if (irqd_is_forwarded_to_vcpu(d))
1678                 return -EINVAL;
1679
1680         prev_cpu = its_dev->event_map.col_map[id];
1681         its_dec_lpi_count(d, prev_cpu);
1682
1683         if (!force)
1684                 cpu = its_select_cpu(d, mask_val);
1685         else
1686                 cpu = cpumask_pick_least_loaded(d, mask_val);
1687
1688         if (cpu < 0 || cpu >= nr_cpu_ids)
1689                 goto err;
1690
1691         /* don't set the affinity when the target cpu is same as current one */
1692         if (cpu != prev_cpu) {
1693                 target_col = &its_dev->its->collections[cpu];
1694                 its_send_movi(its_dev, target_col, id);
1695                 its_dev->event_map.col_map[id] = cpu;
1696                 irq_data_update_effective_affinity(d, cpumask_of(cpu));
1697         }
1698
1699         its_inc_lpi_count(d, cpu);
1700
1701         return IRQ_SET_MASK_OK_DONE;
1702
1703 err:
1704         its_inc_lpi_count(d, prev_cpu);
1705         return -EINVAL;
1706 }
1707
1708 static u64 its_irq_get_msi_base(struct its_device *its_dev)
1709 {
1710         struct its_node *its = its_dev->its;
1711
1712         return its->phys_base + GITS_TRANSLATER;
1713 }
1714
1715 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1716 {
1717         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1718         struct its_node *its;
1719         u64 addr;
1720
1721         its = its_dev->its;
1722         addr = its->get_msi_base(its_dev);
1723
1724         msg->address_lo         = lower_32_bits(addr);
1725         msg->address_hi         = upper_32_bits(addr);
1726         msg->data               = its_get_event_id(d);
1727
1728         iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1729 }
1730
1731 static int its_irq_set_irqchip_state(struct irq_data *d,
1732                                      enum irqchip_irq_state which,
1733                                      bool state)
1734 {
1735         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1736         u32 event = its_get_event_id(d);
1737
1738         if (which != IRQCHIP_STATE_PENDING)
1739                 return -EINVAL;
1740
1741         if (irqd_is_forwarded_to_vcpu(d)) {
1742                 if (state)
1743                         its_send_vint(its_dev, event);
1744                 else
1745                         its_send_vclear(its_dev, event);
1746         } else {
1747                 if (state)
1748                         its_send_int(its_dev, event);
1749                 else
1750                         its_send_clear(its_dev, event);
1751         }
1752
1753         return 0;
1754 }
1755
1756 static int its_irq_retrigger(struct irq_data *d)
1757 {
1758         return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
1759 }
1760
1761 /*
1762  * Two favourable cases:
1763  *
1764  * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1765  *     for vSGI delivery
1766  *
1767  * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1768  *     and we're better off mapping all VPEs always
1769  *
1770  * If neither (a) nor (b) is true, then we map vPEs on demand.
1771  *
1772  */
1773 static bool gic_requires_eager_mapping(void)
1774 {
1775         if (!its_list_map || gic_rdists->has_rvpeid)
1776                 return true;
1777
1778         return false;
1779 }
1780
1781 static void its_map_vm(struct its_node *its, struct its_vm *vm)
1782 {
1783         unsigned long flags;
1784
1785         if (gic_requires_eager_mapping())
1786                 return;
1787
1788         raw_spin_lock_irqsave(&vmovp_lock, flags);
1789
1790         /*
1791          * If the VM wasn't mapped yet, iterate over the vpes and get
1792          * them mapped now.
1793          */
1794         vm->vlpi_count[its->list_nr]++;
1795
1796         if (vm->vlpi_count[its->list_nr] == 1) {
1797                 int i;
1798
1799                 for (i = 0; i < vm->nr_vpes; i++) {
1800                         struct its_vpe *vpe = vm->vpes[i];
1801                         struct irq_data *d = irq_get_irq_data(vpe->irq);
1802
1803                         /* Map the VPE to the first possible CPU */
1804                         vpe->col_idx = cpumask_first(cpu_online_mask);
1805                         its_send_vmapp(its, vpe, true);
1806                         its_send_vinvall(its, vpe);
1807                         irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
1808                 }
1809         }
1810
1811         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1812 }
1813
1814 static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1815 {
1816         unsigned long flags;
1817
1818         /* Not using the ITS list? Everything is always mapped. */
1819         if (gic_requires_eager_mapping())
1820                 return;
1821
1822         raw_spin_lock_irqsave(&vmovp_lock, flags);
1823
1824         if (!--vm->vlpi_count[its->list_nr]) {
1825                 int i;
1826
1827                 for (i = 0; i < vm->nr_vpes; i++)
1828                         its_send_vmapp(its, vm->vpes[i], false);
1829         }
1830
1831         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1832 }
1833
1834 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1835 {
1836         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1837         u32 event = its_get_event_id(d);
1838         int ret = 0;
1839
1840         if (!info->map)
1841                 return -EINVAL;
1842
1843         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1844
1845         if (!its_dev->event_map.vm) {
1846                 struct its_vlpi_map *maps;
1847
1848                 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1849                                GFP_ATOMIC);
1850                 if (!maps) {
1851                         ret = -ENOMEM;
1852                         goto out;
1853                 }
1854
1855                 its_dev->event_map.vm = info->map->vm;
1856                 its_dev->event_map.vlpi_maps = maps;
1857         } else if (its_dev->event_map.vm != info->map->vm) {
1858                 ret = -EINVAL;
1859                 goto out;
1860         }
1861
1862         /* Get our private copy of the mapping information */
1863         its_dev->event_map.vlpi_maps[event] = *info->map;
1864
1865         if (irqd_is_forwarded_to_vcpu(d)) {
1866                 /* Already mapped, move it around */
1867                 its_send_vmovi(its_dev, event);
1868         } else {
1869                 /* Ensure all the VPEs are mapped on this ITS */
1870                 its_map_vm(its_dev->its, info->map->vm);
1871
1872                 /*
1873                  * Flag the interrupt as forwarded so that we can
1874                  * start poking the virtual property table.
1875                  */
1876                 irqd_set_forwarded_to_vcpu(d);
1877
1878                 /* Write out the property to the prop table */
1879                 lpi_write_config(d, 0xff, info->map->properties);
1880
1881                 /* Drop the physical mapping */
1882                 its_send_discard(its_dev, event);
1883
1884                 /* and install the virtual one */
1885                 its_send_vmapti(its_dev, event);
1886
1887                 /* Increment the number of VLPIs */
1888                 its_dev->event_map.nr_vlpis++;
1889         }
1890
1891 out:
1892         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1893         return ret;
1894 }
1895
1896 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1897 {
1898         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1899         struct its_vlpi_map *map;
1900         int ret = 0;
1901
1902         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1903
1904         map = get_vlpi_map(d);
1905
1906         if (!its_dev->event_map.vm || !map) {
1907                 ret = -EINVAL;
1908                 goto out;
1909         }
1910
1911         /* Copy our mapping information to the incoming request */
1912         *info->map = *map;
1913
1914 out:
1915         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1916         return ret;
1917 }
1918
1919 static int its_vlpi_unmap(struct irq_data *d)
1920 {
1921         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1922         u32 event = its_get_event_id(d);
1923         int ret = 0;
1924
1925         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1926
1927         if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) {
1928                 ret = -EINVAL;
1929                 goto out;
1930         }
1931
1932         /* Drop the virtual mapping */
1933         its_send_discard(its_dev, event);
1934
1935         /* and restore the physical one */
1936         irqd_clr_forwarded_to_vcpu(d);
1937         its_send_mapti(its_dev, d->hwirq, event);
1938         lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
1939                                     LPI_PROP_ENABLED |
1940                                     LPI_PROP_GROUP1));
1941
1942         /* Potentially unmap the VM from this ITS */
1943         its_unmap_vm(its_dev->its, its_dev->event_map.vm);
1944
1945         /*
1946          * Drop the refcount and make the device available again if
1947          * this was the last VLPI.
1948          */
1949         if (!--its_dev->event_map.nr_vlpis) {
1950                 its_dev->event_map.vm = NULL;
1951                 kfree(its_dev->event_map.vlpi_maps);
1952         }
1953
1954 out:
1955         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1956         return ret;
1957 }
1958
1959 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
1960 {
1961         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1962
1963         if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1964                 return -EINVAL;
1965
1966         if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
1967                 lpi_update_config(d, 0xff, info->config);
1968         else
1969                 lpi_write_config(d, 0xff, info->config);
1970         its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
1971
1972         return 0;
1973 }
1974
1975 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
1976 {
1977         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1978         struct its_cmd_info *info = vcpu_info;
1979
1980         /* Need a v4 ITS */
1981         if (!is_v4(its_dev->its))
1982                 return -EINVAL;
1983
1984         /* Unmap request? */
1985         if (!info)
1986                 return its_vlpi_unmap(d);
1987
1988         switch (info->cmd_type) {
1989         case MAP_VLPI:
1990                 return its_vlpi_map(d, info);
1991
1992         case GET_VLPI:
1993                 return its_vlpi_get(d, info);
1994
1995         case PROP_UPDATE_VLPI:
1996         case PROP_UPDATE_AND_INV_VLPI:
1997                 return its_vlpi_prop_update(d, info);
1998
1999         default:
2000                 return -EINVAL;
2001         }
2002 }
2003
2004 static struct irq_chip its_irq_chip = {
2005         .name                   = "ITS",
2006         .irq_mask               = its_mask_irq,
2007         .irq_unmask             = its_unmask_irq,
2008         .irq_eoi                = irq_chip_eoi_parent,
2009         .irq_set_affinity       = its_set_affinity,
2010         .irq_compose_msi_msg    = its_irq_compose_msi_msg,
2011         .irq_set_irqchip_state  = its_irq_set_irqchip_state,
2012         .irq_retrigger          = its_irq_retrigger,
2013         .irq_set_vcpu_affinity  = its_irq_set_vcpu_affinity,
2014 };
2015
2016
2017 /*
2018  * How we allocate LPIs:
2019  *
2020  * lpi_range_list contains ranges of LPIs that are to available to
2021  * allocate from. To allocate LPIs, just pick the first range that
2022  * fits the required allocation, and reduce it by the required
2023  * amount. Once empty, remove the range from the list.
2024  *
2025  * To free a range of LPIs, add a free range to the list, sort it and
2026  * merge the result if the new range happens to be adjacent to an
2027  * already free block.
2028  *
2029  * The consequence of the above is that allocation is cost is low, but
2030  * freeing is expensive. We assumes that freeing rarely occurs.
2031  */
2032 #define ITS_MAX_LPI_NRBITS      16 /* 64K LPIs */
2033
2034 static DEFINE_MUTEX(lpi_range_lock);
2035 static LIST_HEAD(lpi_range_list);
2036
2037 struct lpi_range {
2038         struct list_head        entry;
2039         u32                     base_id;
2040         u32                     span;
2041 };
2042
2043 static struct lpi_range *mk_lpi_range(u32 base, u32 span)
2044 {
2045         struct lpi_range *range;
2046
2047         range = kmalloc(sizeof(*range), GFP_KERNEL);
2048         if (range) {
2049                 range->base_id = base;
2050                 range->span = span;
2051         }
2052
2053         return range;
2054 }
2055
2056 static int alloc_lpi_range(u32 nr_lpis, u32 *base)
2057 {
2058         struct lpi_range *range, *tmp;
2059         int err = -ENOSPC;
2060
2061         mutex_lock(&lpi_range_lock);
2062
2063         list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
2064                 if (range->span >= nr_lpis) {
2065                         *base = range->base_id;
2066                         range->base_id += nr_lpis;
2067                         range->span -= nr_lpis;
2068
2069                         if (range->span == 0) {
2070                                 list_del(&range->entry);
2071                                 kfree(range);
2072                         }
2073
2074                         err = 0;
2075                         break;
2076                 }
2077         }
2078
2079         mutex_unlock(&lpi_range_lock);
2080
2081         pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
2082         return err;
2083 }
2084
2085 static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
2086 {
2087         if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
2088                 return;
2089         if (a->base_id + a->span != b->base_id)
2090                 return;
2091         b->base_id = a->base_id;
2092         b->span += a->span;
2093         list_del(&a->entry);
2094         kfree(a);
2095 }
2096
2097 static int free_lpi_range(u32 base, u32 nr_lpis)
2098 {
2099         struct lpi_range *new, *old;
2100
2101         new = mk_lpi_range(base, nr_lpis);
2102         if (!new)
2103                 return -ENOMEM;
2104
2105         mutex_lock(&lpi_range_lock);
2106
2107         list_for_each_entry_reverse(old, &lpi_range_list, entry) {
2108                 if (old->base_id < base)
2109                         break;
2110         }
2111         /*
2112          * old is the last element with ->base_id smaller than base,
2113          * so new goes right after it. If there are no elements with
2114          * ->base_id smaller than base, &old->entry ends up pointing
2115          * at the head of the list, and inserting new it the start of
2116          * the list is the right thing to do in that case as well.
2117          */
2118         list_add(&new->entry, &old->entry);
2119         /*
2120          * Now check if we can merge with the preceding and/or
2121          * following ranges.
2122          */
2123         merge_lpi_ranges(old, new);
2124         merge_lpi_ranges(new, list_next_entry(new, entry));
2125
2126         mutex_unlock(&lpi_range_lock);
2127         return 0;
2128 }
2129
2130 static int __init its_lpi_init(u32 id_bits)
2131 {
2132         u32 lpis = (1UL << id_bits) - 8192;
2133         u32 numlpis;
2134         int err;
2135
2136         numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
2137
2138         if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
2139                 lpis = numlpis;
2140                 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
2141                         lpis);
2142         }
2143
2144         /*
2145          * Initializing the allocator is just the same as freeing the
2146          * full range of LPIs.
2147          */
2148         err = free_lpi_range(8192, lpis);
2149         pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
2150         return err;
2151 }
2152
2153 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
2154 {
2155         unsigned long *bitmap = NULL;
2156         int err = 0;
2157
2158         do {
2159                 err = alloc_lpi_range(nr_irqs, base);
2160                 if (!err)
2161                         break;
2162
2163                 nr_irqs /= 2;
2164         } while (nr_irqs > 0);
2165
2166         if (!nr_irqs)
2167                 err = -ENOSPC;
2168
2169         if (err)
2170                 goto out;
2171
2172         bitmap = bitmap_zalloc(nr_irqs, GFP_ATOMIC);
2173         if (!bitmap)
2174                 goto out;
2175
2176         *nr_ids = nr_irqs;
2177
2178 out:
2179         if (!bitmap)
2180                 *base = *nr_ids = 0;
2181
2182         return bitmap;
2183 }
2184
2185 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
2186 {
2187         WARN_ON(free_lpi_range(base, nr_ids));
2188         bitmap_free(bitmap);
2189 }
2190
2191 static void gic_reset_prop_table(void *va)
2192 {
2193         /* Priority 0xa0, Group-1, disabled */
2194         memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
2195
2196         /* Make sure the GIC will observe the written configuration */
2197         gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
2198 }
2199
2200 static struct page *its_allocate_prop_table(gfp_t gfp_flags)
2201 {
2202         struct page *prop_page;
2203
2204         prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
2205         if (!prop_page)
2206                 return NULL;
2207
2208         gic_reset_prop_table(page_address(prop_page));
2209
2210         return prop_page;
2211 }
2212
2213 static void its_free_prop_table(struct page *prop_page)
2214 {
2215         free_pages((unsigned long)page_address(prop_page),
2216                    get_order(LPI_PROPBASE_SZ));
2217 }
2218
2219 static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
2220 {
2221         phys_addr_t start, end, addr_end;
2222         u64 i;
2223
2224         /*
2225          * We don't bother checking for a kdump kernel as by
2226          * construction, the LPI tables are out of this kernel's
2227          * memory map.
2228          */
2229         if (is_kdump_kernel())
2230                 return true;
2231
2232         addr_end = addr + size - 1;
2233
2234         for_each_reserved_mem_range(i, &start, &end) {
2235                 if (addr >= start && addr_end <= end)
2236                         return true;
2237         }
2238
2239         /* Not found, not a good sign... */
2240         pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
2241                 &addr, &addr_end);
2242         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2243         return false;
2244 }
2245
2246 static int gic_reserve_range(phys_addr_t addr, unsigned long size)
2247 {
2248         if (efi_enabled(EFI_CONFIG_TABLES))
2249                 return efi_mem_reserve_persistent(addr, size);
2250
2251         return 0;
2252 }
2253
2254 static int __init its_setup_lpi_prop_table(void)
2255 {
2256         if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
2257                 u64 val;
2258
2259                 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2260                 lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
2261
2262                 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
2263                 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
2264                                                      LPI_PROPBASE_SZ,
2265                                                      MEMREMAP_WB);
2266                 gic_reset_prop_table(gic_rdists->prop_table_va);
2267         } else {
2268                 struct page *page;
2269
2270                 lpi_id_bits = min_t(u32,
2271                                     GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
2272                                     ITS_MAX_LPI_NRBITS);
2273                 page = its_allocate_prop_table(GFP_NOWAIT);
2274                 if (!page) {
2275                         pr_err("Failed to allocate PROPBASE\n");
2276                         return -ENOMEM;
2277                 }
2278
2279                 gic_rdists->prop_table_pa = page_to_phys(page);
2280                 gic_rdists->prop_table_va = page_address(page);
2281                 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
2282                                           LPI_PROPBASE_SZ));
2283         }
2284
2285         pr_info("GICv3: using LPI property table @%pa\n",
2286                 &gic_rdists->prop_table_pa);
2287
2288         return its_lpi_init(lpi_id_bits);
2289 }
2290
2291 static const char *its_base_type_string[] = {
2292         [GITS_BASER_TYPE_DEVICE]        = "Devices",
2293         [GITS_BASER_TYPE_VCPU]          = "Virtual CPUs",
2294         [GITS_BASER_TYPE_RESERVED3]     = "Reserved (3)",
2295         [GITS_BASER_TYPE_COLLECTION]    = "Interrupt Collections",
2296         [GITS_BASER_TYPE_RESERVED5]     = "Reserved (5)",
2297         [GITS_BASER_TYPE_RESERVED6]     = "Reserved (6)",
2298         [GITS_BASER_TYPE_RESERVED7]     = "Reserved (7)",
2299 };
2300
2301 static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
2302 {
2303         u32 idx = baser - its->tables;
2304
2305         return gits_read_baser(its->base + GITS_BASER + (idx << 3));
2306 }
2307
2308 static void its_write_baser(struct its_node *its, struct its_baser *baser,
2309                             u64 val)
2310 {
2311         u32 idx = baser - its->tables;
2312
2313         gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
2314         baser->val = its_read_baser(its, baser);
2315 }
2316
2317 static int its_setup_baser(struct its_node *its, struct its_baser *baser,
2318                            u64 cache, u64 shr, u32 order, bool indirect)
2319 {
2320         u64 val = its_read_baser(its, baser);
2321         u64 esz = GITS_BASER_ENTRY_SIZE(val);
2322         u64 type = GITS_BASER_TYPE(val);
2323         u64 baser_phys, tmp;
2324         u32 alloc_pages, psz;
2325         struct page *page;
2326         void *base;
2327
2328         psz = baser->psz;
2329         alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
2330         if (alloc_pages > GITS_BASER_PAGES_MAX) {
2331                 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
2332                         &its->phys_base, its_base_type_string[type],
2333                         alloc_pages, GITS_BASER_PAGES_MAX);
2334                 alloc_pages = GITS_BASER_PAGES_MAX;
2335                 order = get_order(GITS_BASER_PAGES_MAX * psz);
2336         }
2337
2338         page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
2339         if (!page)
2340                 return -ENOMEM;
2341
2342         base = (void *)page_address(page);
2343         baser_phys = virt_to_phys(base);
2344
2345         /* Check if the physical address of the memory is above 48bits */
2346         if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
2347
2348                 /* 52bit PA is supported only when PageSize=64K */
2349                 if (psz != SZ_64K) {
2350                         pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
2351                         free_pages((unsigned long)base, order);
2352                         return -ENXIO;
2353                 }
2354
2355                 /* Convert 52bit PA to 48bit field */
2356                 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
2357         }
2358
2359 retry_baser:
2360         val = (baser_phys                                        |
2361                 (type << GITS_BASER_TYPE_SHIFT)                  |
2362                 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT)       |
2363                 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT)    |
2364                 cache                                            |
2365                 shr                                              |
2366                 GITS_BASER_VALID);
2367
2368         val |=  indirect ? GITS_BASER_INDIRECT : 0x0;
2369
2370         switch (psz) {
2371         case SZ_4K:
2372                 val |= GITS_BASER_PAGE_SIZE_4K;
2373                 break;
2374         case SZ_16K:
2375                 val |= GITS_BASER_PAGE_SIZE_16K;
2376                 break;
2377         case SZ_64K:
2378                 val |= GITS_BASER_PAGE_SIZE_64K;
2379                 break;
2380         }
2381
2382         if (!shr)
2383                 gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
2384
2385         its_write_baser(its, baser, val);
2386         tmp = baser->val;
2387
2388         if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
2389                 /*
2390                  * Shareability didn't stick. Just use
2391                  * whatever the read reported, which is likely
2392                  * to be the only thing this redistributor
2393                  * supports. If that's zero, make it
2394                  * non-cacheable as well.
2395                  */
2396                 shr = tmp & GITS_BASER_SHAREABILITY_MASK;
2397                 if (!shr)
2398                         cache = GITS_BASER_nC;
2399
2400                 goto retry_baser;
2401         }
2402
2403         if (val != tmp) {
2404                 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
2405                        &its->phys_base, its_base_type_string[type],
2406                        val, tmp);
2407                 free_pages((unsigned long)base, order);
2408                 return -ENXIO;
2409         }
2410
2411         baser->order = order;
2412         baser->base = base;
2413         baser->psz = psz;
2414         tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
2415
2416         pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
2417                 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
2418                 its_base_type_string[type],
2419                 (unsigned long)virt_to_phys(base),
2420                 indirect ? "indirect" : "flat", (int)esz,
2421                 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
2422
2423         return 0;
2424 }
2425
2426 static bool its_parse_indirect_baser(struct its_node *its,
2427                                      struct its_baser *baser,
2428                                      u32 *order, u32 ids)
2429 {
2430         u64 tmp = its_read_baser(its, baser);
2431         u64 type = GITS_BASER_TYPE(tmp);
2432         u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
2433         u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
2434         u32 new_order = *order;
2435         u32 psz = baser->psz;
2436         bool indirect = false;
2437
2438         /* No need to enable Indirection if memory requirement < (psz*2)bytes */
2439         if ((esz << ids) > (psz * 2)) {
2440                 /*
2441                  * Find out whether hw supports a single or two-level table by
2442                  * table by reading bit at offset '62' after writing '1' to it.
2443                  */
2444                 its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
2445                 indirect = !!(baser->val & GITS_BASER_INDIRECT);
2446
2447                 if (indirect) {
2448                         /*
2449                          * The size of the lvl2 table is equal to ITS page size
2450                          * which is 'psz'. For computing lvl1 table size,
2451                          * subtract ID bits that sparse lvl2 table from 'ids'
2452                          * which is reported by ITS hardware times lvl1 table
2453                          * entry size.
2454                          */
2455                         ids -= ilog2(psz / (int)esz);
2456                         esz = GITS_LVL1_ENTRY_SIZE;
2457                 }
2458         }
2459
2460         /*
2461          * Allocate as many entries as required to fit the
2462          * range of device IDs that the ITS can grok... The ID
2463          * space being incredibly sparse, this results in a
2464          * massive waste of memory if two-level device table
2465          * feature is not supported by hardware.
2466          */
2467         new_order = max_t(u32, get_order(esz << ids), new_order);
2468         if (new_order > MAX_ORDER) {
2469                 new_order = MAX_ORDER;
2470                 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
2471                 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
2472                         &its->phys_base, its_base_type_string[type],
2473                         device_ids(its), ids);
2474         }
2475
2476         *order = new_order;
2477
2478         return indirect;
2479 }
2480
2481 static u32 compute_common_aff(u64 val)
2482 {
2483         u32 aff, clpiaff;
2484
2485         aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
2486         clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
2487
2488         return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
2489 }
2490
2491 static u32 compute_its_aff(struct its_node *its)
2492 {
2493         u64 val;
2494         u32 svpet;
2495
2496         /*
2497          * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
2498          * the resulting affinity. We then use that to see if this match
2499          * our own affinity.
2500          */
2501         svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
2502         val  = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
2503         val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
2504         return compute_common_aff(val);
2505 }
2506
2507 static struct its_node *find_sibling_its(struct its_node *cur_its)
2508 {
2509         struct its_node *its;
2510         u32 aff;
2511
2512         if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
2513                 return NULL;
2514
2515         aff = compute_its_aff(cur_its);
2516
2517         list_for_each_entry(its, &its_nodes, entry) {
2518                 u64 baser;
2519
2520                 if (!is_v4_1(its) || its == cur_its)
2521                         continue;
2522
2523                 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2524                         continue;
2525
2526                 if (aff != compute_its_aff(its))
2527                         continue;
2528
2529                 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2530                 baser = its->tables[2].val;
2531                 if (!(baser & GITS_BASER_VALID))
2532                         continue;
2533
2534                 return its;
2535         }
2536
2537         return NULL;
2538 }
2539
2540 static void its_free_tables(struct its_node *its)
2541 {
2542         int i;
2543
2544         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2545                 if (its->tables[i].base) {
2546                         free_pages((unsigned long)its->tables[i].base,
2547                                    its->tables[i].order);
2548                         its->tables[i].base = NULL;
2549                 }
2550         }
2551 }
2552
2553 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
2554 {
2555         u64 psz = SZ_64K;
2556
2557         while (psz) {
2558                 u64 val, gpsz;
2559
2560                 val = its_read_baser(its, baser);
2561                 val &= ~GITS_BASER_PAGE_SIZE_MASK;
2562
2563                 switch (psz) {
2564                 case SZ_64K:
2565                         gpsz = GITS_BASER_PAGE_SIZE_64K;
2566                         break;
2567                 case SZ_16K:
2568                         gpsz = GITS_BASER_PAGE_SIZE_16K;
2569                         break;
2570                 case SZ_4K:
2571                 default:
2572                         gpsz = GITS_BASER_PAGE_SIZE_4K;
2573                         break;
2574                 }
2575
2576                 gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
2577
2578                 val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
2579                 its_write_baser(its, baser, val);
2580
2581                 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
2582                         break;
2583
2584                 switch (psz) {
2585                 case SZ_64K:
2586                         psz = SZ_16K;
2587                         break;
2588                 case SZ_16K:
2589                         psz = SZ_4K;
2590                         break;
2591                 case SZ_4K:
2592                 default:
2593                         return -1;
2594                 }
2595         }
2596
2597         baser->psz = psz;
2598         return 0;
2599 }
2600
2601 static int its_alloc_tables(struct its_node *its)
2602 {
2603         u64 shr = GITS_BASER_InnerShareable;
2604         u64 cache = GITS_BASER_RaWaWb;
2605         int err, i;
2606
2607         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
2608                 /* erratum 24313: ignore memory access type */
2609                 cache = GITS_BASER_nCnB;
2610
2611         if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) {
2612                 cache = GITS_BASER_nC;
2613                 shr = 0;
2614         }
2615
2616         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2617                 struct its_baser *baser = its->tables + i;
2618                 u64 val = its_read_baser(its, baser);
2619                 u64 type = GITS_BASER_TYPE(val);
2620                 bool indirect = false;
2621                 u32 order;
2622
2623                 if (type == GITS_BASER_TYPE_NONE)
2624                         continue;
2625
2626                 if (its_probe_baser_psz(its, baser)) {
2627                         its_free_tables(its);
2628                         return -ENXIO;
2629                 }
2630
2631                 order = get_order(baser->psz);
2632
2633                 switch (type) {
2634                 case GITS_BASER_TYPE_DEVICE:
2635                         indirect = its_parse_indirect_baser(its, baser, &order,
2636                                                             device_ids(its));
2637                         break;
2638
2639                 case GITS_BASER_TYPE_VCPU:
2640                         if (is_v4_1(its)) {
2641                                 struct its_node *sibling;
2642
2643                                 WARN_ON(i != 2);
2644                                 if ((sibling = find_sibling_its(its))) {
2645                                         *baser = sibling->tables[2];
2646                                         its_write_baser(its, baser, baser->val);
2647                                         continue;
2648                                 }
2649                         }
2650
2651                         indirect = its_parse_indirect_baser(its, baser, &order,
2652                                                             ITS_MAX_VPEID_BITS);
2653                         break;
2654                 }
2655
2656                 err = its_setup_baser(its, baser, cache, shr, order, indirect);
2657                 if (err < 0) {
2658                         its_free_tables(its);
2659                         return err;
2660                 }
2661
2662                 /* Update settings which will be used for next BASERn */
2663                 cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
2664                 shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
2665         }
2666
2667         return 0;
2668 }
2669
2670 static u64 inherit_vpe_l1_table_from_its(void)
2671 {
2672         struct its_node *its;
2673         u64 val;
2674         u32 aff;
2675
2676         val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2677         aff = compute_common_aff(val);
2678
2679         list_for_each_entry(its, &its_nodes, entry) {
2680                 u64 baser, addr;
2681
2682                 if (!is_v4_1(its))
2683                         continue;
2684
2685                 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2686                         continue;
2687
2688                 if (aff != compute_its_aff(its))
2689                         continue;
2690
2691                 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2692                 baser = its->tables[2].val;
2693                 if (!(baser & GITS_BASER_VALID))
2694                         continue;
2695
2696                 /* We have a winner! */
2697                 gic_data_rdist()->vpe_l1_base = its->tables[2].base;
2698
2699                 val  = GICR_VPROPBASER_4_1_VALID;
2700                 if (baser & GITS_BASER_INDIRECT)
2701                         val |= GICR_VPROPBASER_4_1_INDIRECT;
2702                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
2703                                   FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
2704                 switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
2705                 case GIC_PAGE_SIZE_64K:
2706                         addr = GITS_BASER_ADDR_48_to_52(baser);
2707                         break;
2708                 default:
2709                         addr = baser & GENMASK_ULL(47, 12);
2710                         break;
2711                 }
2712                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
2713                 val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
2714                                   FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
2715                 val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
2716                                   FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
2717                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
2718
2719                 return val;
2720         }
2721
2722         return 0;
2723 }
2724
2725 static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
2726 {
2727         u32 aff;
2728         u64 val;
2729         int cpu;
2730
2731         val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2732         aff = compute_common_aff(val);
2733
2734         for_each_possible_cpu(cpu) {
2735                 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2736
2737                 if (!base || cpu == smp_processor_id())
2738                         continue;
2739
2740                 val = gic_read_typer(base + GICR_TYPER);
2741                 if (aff != compute_common_aff(val))
2742                         continue;
2743
2744                 /*
2745                  * At this point, we have a victim. This particular CPU
2746                  * has already booted, and has an affinity that matches
2747                  * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
2748                  * Make sure we don't write the Z bit in that case.
2749                  */
2750                 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2751                 val &= ~GICR_VPROPBASER_4_1_Z;
2752
2753                 gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2754                 *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
2755
2756                 return val;
2757         }
2758
2759         return 0;
2760 }
2761
2762 static bool allocate_vpe_l2_table(int cpu, u32 id)
2763 {
2764         void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2765         unsigned int psz, esz, idx, npg, gpsz;
2766         u64 val;
2767         struct page *page;
2768         __le64 *table;
2769
2770         if (!gic_rdists->has_rvpeid)
2771                 return true;
2772
2773         /* Skip non-present CPUs */
2774         if (!base)
2775                 return true;
2776
2777         val  = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2778
2779         esz  = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
2780         gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2781         npg  = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
2782
2783         switch (gpsz) {
2784         default:
2785                 WARN_ON(1);
2786                 fallthrough;
2787         case GIC_PAGE_SIZE_4K:
2788                 psz = SZ_4K;
2789                 break;
2790         case GIC_PAGE_SIZE_16K:
2791                 psz = SZ_16K;
2792                 break;
2793         case GIC_PAGE_SIZE_64K:
2794                 psz = SZ_64K;
2795                 break;
2796         }
2797
2798         /* Don't allow vpe_id that exceeds single, flat table limit */
2799         if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
2800                 return (id < (npg * psz / (esz * SZ_8)));
2801
2802         /* Compute 1st level table index & check if that exceeds table limit */
2803         idx = id >> ilog2(psz / (esz * SZ_8));
2804         if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
2805                 return false;
2806
2807         table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2808
2809         /* Allocate memory for 2nd level table */
2810         if (!table[idx]) {
2811                 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
2812                 if (!page)
2813                         return false;
2814
2815                 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
2816                 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2817                         gic_flush_dcache_to_poc(page_address(page), psz);
2818
2819                 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
2820
2821                 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2822                 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2823                         gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
2824
2825                 /* Ensure updated table contents are visible to RD hardware */
2826                 dsb(sy);
2827         }
2828
2829         return true;
2830 }
2831
2832 static int allocate_vpe_l1_table(void)
2833 {
2834         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2835         u64 val, gpsz, npg, pa;
2836         unsigned int psz = SZ_64K;
2837         unsigned int np, epp, esz;
2838         struct page *page;
2839
2840         if (!gic_rdists->has_rvpeid)
2841                 return 0;
2842
2843         /*
2844          * if VPENDBASER.Valid is set, disable any previously programmed
2845          * VPE by setting PendingLast while clearing Valid. This has the
2846          * effect of making sure no doorbell will be generated and we can
2847          * then safely clear VPROPBASER.Valid.
2848          */
2849         if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
2850                 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
2851                                       vlpi_base + GICR_VPENDBASER);
2852
2853         /*
2854          * If we can inherit the configuration from another RD, let's do
2855          * so. Otherwise, we have to go through the allocation process. We
2856          * assume that all RDs have the exact same requirements, as
2857          * nothing will work otherwise.
2858          */
2859         val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
2860         if (val & GICR_VPROPBASER_4_1_VALID)
2861                 goto out;
2862
2863         gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
2864         if (!gic_data_rdist()->vpe_table_mask)
2865                 return -ENOMEM;
2866
2867         val = inherit_vpe_l1_table_from_its();
2868         if (val & GICR_VPROPBASER_4_1_VALID)
2869                 goto out;
2870
2871         /* First probe the page size */
2872         val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
2873         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2874         val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
2875         gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2876         esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
2877
2878         switch (gpsz) {
2879         default:
2880                 gpsz = GIC_PAGE_SIZE_4K;
2881                 fallthrough;
2882         case GIC_PAGE_SIZE_4K:
2883                 psz = SZ_4K;
2884                 break;
2885         case GIC_PAGE_SIZE_16K:
2886                 psz = SZ_16K;
2887                 break;
2888         case GIC_PAGE_SIZE_64K:
2889                 psz = SZ_64K;
2890                 break;
2891         }
2892
2893         /*
2894          * Start populating the register from scratch, including RO fields
2895          * (which we want to print in debug cases...)
2896          */
2897         val = 0;
2898         val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
2899         val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
2900
2901         /* How many entries per GIC page? */
2902         esz++;
2903         epp = psz / (esz * SZ_8);
2904
2905         /*
2906          * If we need more than just a single L1 page, flag the table
2907          * as indirect and compute the number of required L1 pages.
2908          */
2909         if (epp < ITS_MAX_VPEID) {
2910                 int nl2;
2911
2912                 val |= GICR_VPROPBASER_4_1_INDIRECT;
2913
2914                 /* Number of L2 pages required to cover the VPEID space */
2915                 nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
2916
2917                 /* Number of L1 pages to point to the L2 pages */
2918                 npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
2919         } else {
2920                 npg = 1;
2921         }
2922
2923         val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
2924
2925         /* Right, that's the number of CPU pages we need for L1 */
2926         np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
2927
2928         pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
2929                  np, npg, psz, epp, esz);
2930         page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
2931         if (!page)
2932                 return -ENOMEM;
2933
2934         gic_data_rdist()->vpe_l1_base = page_address(page);
2935         pa = virt_to_phys(page_address(page));
2936         WARN_ON(!IS_ALIGNED(pa, psz));
2937
2938         val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
2939         val |= GICR_VPROPBASER_RaWb;
2940         val |= GICR_VPROPBASER_InnerShareable;
2941         val |= GICR_VPROPBASER_4_1_Z;
2942         val |= GICR_VPROPBASER_4_1_VALID;
2943
2944 out:
2945         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2946         cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
2947
2948         pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
2949                  smp_processor_id(), val,
2950                  cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
2951
2952         return 0;
2953 }
2954
2955 static int its_alloc_collections(struct its_node *its)
2956 {
2957         int i;
2958
2959         its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
2960                                    GFP_KERNEL);
2961         if (!its->collections)
2962                 return -ENOMEM;
2963
2964         for (i = 0; i < nr_cpu_ids; i++)
2965                 its->collections[i].target_address = ~0ULL;
2966
2967         return 0;
2968 }
2969
2970 static struct page *its_allocate_pending_table(gfp_t gfp_flags)
2971 {
2972         struct page *pend_page;
2973
2974         pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
2975                                 get_order(LPI_PENDBASE_SZ));
2976         if (!pend_page)
2977                 return NULL;
2978
2979         /* Make sure the GIC will observe the zero-ed page */
2980         gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
2981
2982         return pend_page;
2983 }
2984
2985 static void its_free_pending_table(struct page *pt)
2986 {
2987         free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
2988 }
2989
2990 /*
2991  * Booting with kdump and LPIs enabled is generally fine. Any other
2992  * case is wrong in the absence of firmware/EFI support.
2993  */
2994 static bool enabled_lpis_allowed(void)
2995 {
2996         phys_addr_t addr;
2997         u64 val;
2998
2999         /* Check whether the property table is in a reserved region */
3000         val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
3001         addr = val & GENMASK_ULL(51, 12);
3002
3003         return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
3004 }
3005
3006 static int __init allocate_lpi_tables(void)
3007 {
3008         u64 val;
3009         int err, cpu;
3010
3011         /*
3012          * If LPIs are enabled while we run this from the boot CPU,
3013          * flag the RD tables as pre-allocated if the stars do align.
3014          */
3015         val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
3016         if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
3017                 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
3018                                       RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
3019                 pr_info("GICv3: Using preallocated redistributor tables\n");
3020         }
3021
3022         err = its_setup_lpi_prop_table();
3023         if (err)
3024                 return err;
3025
3026         /*
3027          * We allocate all the pending tables anyway, as we may have a
3028          * mix of RDs that have had LPIs enabled, and some that
3029          * don't. We'll free the unused ones as each CPU comes online.
3030          */
3031         for_each_possible_cpu(cpu) {
3032                 struct page *pend_page;
3033
3034                 pend_page = its_allocate_pending_table(GFP_NOWAIT);
3035                 if (!pend_page) {
3036                         pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
3037                         return -ENOMEM;
3038                 }
3039
3040                 gic_data_rdist_cpu(cpu)->pend_page = pend_page;
3041         }
3042
3043         return 0;
3044 }
3045
3046 static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
3047 {
3048         u32 count = 1000000;    /* 1s! */
3049         bool clean;
3050         u64 val;
3051
3052         do {
3053                 val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3054                 clean = !(val & GICR_VPENDBASER_Dirty);
3055                 if (!clean) {
3056                         count--;
3057                         cpu_relax();
3058                         udelay(1);
3059                 }
3060         } while (!clean && count);
3061
3062         if (unlikely(!clean))
3063                 pr_err_ratelimited("ITS virtual pending table not cleaning\n");
3064
3065         return val;
3066 }
3067
3068 static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
3069 {
3070         u64 val;
3071
3072         /* Make sure we wait until the RD is done with the initial scan */
3073         val = read_vpend_dirty_clear(vlpi_base);
3074         val &= ~GICR_VPENDBASER_Valid;
3075         val &= ~clr;
3076         val |= set;
3077         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3078
3079         val = read_vpend_dirty_clear(vlpi_base);
3080         if (unlikely(val & GICR_VPENDBASER_Dirty))
3081                 val |= GICR_VPENDBASER_PendingLast;
3082
3083         return val;
3084 }
3085
3086 static void its_cpu_init_lpis(void)
3087 {
3088         void __iomem *rbase = gic_data_rdist_rd_base();
3089         struct page *pend_page;
3090         phys_addr_t paddr;
3091         u64 val, tmp;
3092
3093         if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED)
3094                 return;
3095
3096         val = readl_relaxed(rbase + GICR_CTLR);
3097         if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
3098             (val & GICR_CTLR_ENABLE_LPIS)) {
3099                 /*
3100                  * Check that we get the same property table on all
3101                  * RDs. If we don't, this is hopeless.
3102                  */
3103                 paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
3104                 paddr &= GENMASK_ULL(51, 12);
3105                 if (WARN_ON(gic_rdists->prop_table_pa != paddr))
3106                         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3107
3108                 paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3109                 paddr &= GENMASK_ULL(51, 16);
3110
3111                 WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
3112                 gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED;
3113
3114                 goto out;
3115         }
3116
3117         pend_page = gic_data_rdist()->pend_page;
3118         paddr = page_to_phys(pend_page);
3119
3120         /* set PROPBASE */
3121         val = (gic_rdists->prop_table_pa |
3122                GICR_PROPBASER_InnerShareable |
3123                GICR_PROPBASER_RaWaWb |
3124                ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
3125
3126         gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3127         tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
3128
3129         if (gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE)
3130                 tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
3131
3132         if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
3133                 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
3134                         /*
3135                          * The HW reports non-shareable, we must
3136                          * remove the cacheability attributes as
3137                          * well.
3138                          */
3139                         val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
3140                                  GICR_PROPBASER_CACHEABILITY_MASK);
3141                         val |= GICR_PROPBASER_nC;
3142                         gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3143                 }
3144                 pr_info_once("GIC: using cache flushing for LPI property table\n");
3145                 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
3146         }
3147
3148         /* set PENDBASE */
3149         val = (page_to_phys(pend_page) |
3150                GICR_PENDBASER_InnerShareable |
3151                GICR_PENDBASER_RaWaWb);
3152
3153         gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3154         tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3155
3156         if (gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE)
3157                 tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
3158
3159         if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
3160                 /*
3161                  * The HW reports non-shareable, we must remove the
3162                  * cacheability attributes as well.
3163                  */
3164                 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
3165                          GICR_PENDBASER_CACHEABILITY_MASK);
3166                 val |= GICR_PENDBASER_nC;
3167                 gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3168         }
3169
3170         /* Enable LPIs */
3171         val = readl_relaxed(rbase + GICR_CTLR);
3172         val |= GICR_CTLR_ENABLE_LPIS;
3173         writel_relaxed(val, rbase + GICR_CTLR);
3174
3175         if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
3176                 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3177
3178                 /*
3179                  * It's possible for CPU to receive VLPIs before it is
3180                  * scheduled as a vPE, especially for the first CPU, and the
3181                  * VLPI with INTID larger than 2^(IDbits+1) will be considered
3182                  * as out of range and dropped by GIC.
3183                  * So we initialize IDbits to known value to avoid VLPI drop.
3184                  */
3185                 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3186                 pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
3187                         smp_processor_id(), val);
3188                 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3189
3190                 /*
3191                  * Also clear Valid bit of GICR_VPENDBASER, in case some
3192                  * ancient programming gets left in and has possibility of
3193                  * corrupting memory.
3194                  */
3195                 val = its_clear_vpend_valid(vlpi_base, 0, 0);
3196         }
3197
3198         if (allocate_vpe_l1_table()) {
3199                 /*
3200                  * If the allocation has failed, we're in massive trouble.
3201                  * Disable direct injection, and pray that no VM was
3202                  * already running...
3203                  */
3204                 gic_rdists->has_rvpeid = false;
3205                 gic_rdists->has_vlpis = false;
3206         }
3207
3208         /* Make sure the GIC has seen the above */
3209         dsb(sy);
3210 out:
3211         gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED;
3212         pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
3213                 smp_processor_id(),
3214                 gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ?
3215                 "reserved" : "allocated",
3216                 &paddr);
3217 }
3218
3219 static void its_cpu_init_collection(struct its_node *its)
3220 {
3221         int cpu = smp_processor_id();
3222         u64 target;
3223
3224         /* avoid cross node collections and its mapping */
3225         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
3226                 struct device_node *cpu_node;
3227
3228                 cpu_node = of_get_cpu_node(cpu, NULL);
3229                 if (its->numa_node != NUMA_NO_NODE &&
3230                         its->numa_node != of_node_to_nid(cpu_node))
3231                         return;
3232         }
3233
3234         /*
3235          * We now have to bind each collection to its target
3236          * redistributor.
3237          */
3238         if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
3239                 /*
3240                  * This ITS wants the physical address of the
3241                  * redistributor.
3242                  */
3243                 target = gic_data_rdist()->phys_base;
3244         } else {
3245                 /* This ITS wants a linear CPU number. */
3246                 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
3247                 target = GICR_TYPER_CPU_NUMBER(target) << 16;
3248         }
3249
3250         /* Perform collection mapping */
3251         its->collections[cpu].target_address = target;
3252         its->collections[cpu].col_id = cpu;
3253
3254         its_send_mapc(its, &its->collections[cpu], 1);
3255         its_send_invall(its, &its->collections[cpu]);
3256 }
3257
3258 static void its_cpu_init_collections(void)
3259 {
3260         struct its_node *its;
3261
3262         raw_spin_lock(&its_lock);
3263
3264         list_for_each_entry(its, &its_nodes, entry)
3265                 its_cpu_init_collection(its);
3266
3267         raw_spin_unlock(&its_lock);
3268 }
3269
3270 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
3271 {
3272         struct its_device *its_dev = NULL, *tmp;
3273         unsigned long flags;
3274
3275         raw_spin_lock_irqsave(&its->lock, flags);
3276
3277         list_for_each_entry(tmp, &its->its_device_list, entry) {
3278                 if (tmp->device_id == dev_id) {
3279                         its_dev = tmp;
3280                         break;
3281                 }
3282         }
3283
3284         raw_spin_unlock_irqrestore(&its->lock, flags);
3285
3286         return its_dev;
3287 }
3288
3289 static struct its_baser *its_get_baser(struct its_node *its, u32 type)
3290 {
3291         int i;
3292
3293         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3294                 if (GITS_BASER_TYPE(its->tables[i].val) == type)
3295                         return &its->tables[i];
3296         }
3297
3298         return NULL;
3299 }
3300
3301 static bool its_alloc_table_entry(struct its_node *its,
3302                                   struct its_baser *baser, u32 id)
3303 {
3304         struct page *page;
3305         u32 esz, idx;
3306         __le64 *table;
3307
3308         /* Don't allow device id that exceeds single, flat table limit */
3309         esz = GITS_BASER_ENTRY_SIZE(baser->val);
3310         if (!(baser->val & GITS_BASER_INDIRECT))
3311                 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
3312
3313         /* Compute 1st level table index & check if that exceeds table limit */
3314         idx = id >> ilog2(baser->psz / esz);
3315         if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
3316                 return false;
3317
3318         table = baser->base;
3319
3320         /* Allocate memory for 2nd level table */
3321         if (!table[idx]) {
3322                 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
3323                                         get_order(baser->psz));
3324                 if (!page)
3325                         return false;
3326
3327                 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
3328                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3329                         gic_flush_dcache_to_poc(page_address(page), baser->psz);
3330
3331                 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
3332
3333                 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
3334                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3335                         gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
3336
3337                 /* Ensure updated table contents are visible to ITS hardware */
3338                 dsb(sy);
3339         }
3340
3341         return true;
3342 }
3343
3344 static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
3345 {
3346         struct its_baser *baser;
3347
3348         baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
3349
3350         /* Don't allow device id that exceeds ITS hardware limit */
3351         if (!baser)
3352                 return (ilog2(dev_id) < device_ids(its));
3353
3354         return its_alloc_table_entry(its, baser, dev_id);
3355 }
3356
3357 static bool its_alloc_vpe_table(u32 vpe_id)
3358 {
3359         struct its_node *its;
3360         int cpu;
3361
3362         /*
3363          * Make sure the L2 tables are allocated on *all* v4 ITSs. We
3364          * could try and only do it on ITSs corresponding to devices
3365          * that have interrupts targeted at this VPE, but the
3366          * complexity becomes crazy (and you have tons of memory
3367          * anyway, right?).
3368          */
3369         list_for_each_entry(its, &its_nodes, entry) {
3370                 struct its_baser *baser;
3371
3372                 if (!is_v4(its))
3373                         continue;
3374
3375                 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
3376                 if (!baser)
3377                         return false;
3378
3379                 if (!its_alloc_table_entry(its, baser, vpe_id))
3380                         return false;
3381         }
3382
3383         /* Non v4.1? No need to iterate RDs and go back early. */
3384         if (!gic_rdists->has_rvpeid)
3385                 return true;
3386
3387         /*
3388          * Make sure the L2 tables are allocated for all copies of
3389          * the L1 table on *all* v4.1 RDs.
3390          */
3391         for_each_possible_cpu(cpu) {
3392                 if (!allocate_vpe_l2_table(cpu, vpe_id))
3393                         return false;
3394         }
3395
3396         return true;
3397 }
3398
3399 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
3400                                             int nvecs, bool alloc_lpis)
3401 {
3402         struct its_device *dev;
3403         unsigned long *lpi_map = NULL;
3404         unsigned long flags;
3405         u16 *col_map = NULL;
3406         void *itt;
3407         int lpi_base;
3408         int nr_lpis;
3409         int nr_ites;
3410         int sz;
3411
3412         if (!its_alloc_device_table(its, dev_id))
3413                 return NULL;
3414
3415         if (WARN_ON(!is_power_of_2(nvecs)))
3416                 nvecs = roundup_pow_of_two(nvecs);
3417
3418         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
3419         /*
3420          * Even if the device wants a single LPI, the ITT must be
3421          * sized as a power of two (and you need at least one bit...).
3422          */
3423         nr_ites = max(2, nvecs);
3424         sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
3425         sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
3426         itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
3427         if (alloc_lpis) {
3428                 lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
3429                 if (lpi_map)
3430                         col_map = kcalloc(nr_lpis, sizeof(*col_map),
3431                                           GFP_KERNEL);
3432         } else {
3433                 col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
3434                 nr_lpis = 0;
3435                 lpi_base = 0;
3436         }
3437
3438         if (!dev || !itt ||  !col_map || (!lpi_map && alloc_lpis)) {
3439                 kfree(dev);
3440                 kfree(itt);
3441                 bitmap_free(lpi_map);
3442                 kfree(col_map);
3443                 return NULL;
3444         }
3445
3446         gic_flush_dcache_to_poc(itt, sz);
3447
3448         dev->its = its;
3449         dev->itt = itt;
3450         dev->nr_ites = nr_ites;
3451         dev->event_map.lpi_map = lpi_map;
3452         dev->event_map.col_map = col_map;
3453         dev->event_map.lpi_base = lpi_base;
3454         dev->event_map.nr_lpis = nr_lpis;
3455         raw_spin_lock_init(&dev->event_map.vlpi_lock);
3456         dev->device_id = dev_id;
3457         INIT_LIST_HEAD(&dev->entry);
3458
3459         raw_spin_lock_irqsave(&its->lock, flags);
3460         list_add(&dev->entry, &its->its_device_list);
3461         raw_spin_unlock_irqrestore(&its->lock, flags);
3462
3463         /* Map device to its ITT */
3464         its_send_mapd(dev, 1);
3465
3466         return dev;
3467 }
3468
3469 static void its_free_device(struct its_device *its_dev)
3470 {
3471         unsigned long flags;
3472
3473         raw_spin_lock_irqsave(&its_dev->its->lock, flags);
3474         list_del(&its_dev->entry);
3475         raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
3476         kfree(its_dev->event_map.col_map);
3477         kfree(its_dev->itt);
3478         kfree(its_dev);
3479 }
3480
3481 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
3482 {
3483         int idx;
3484
3485         /* Find a free LPI region in lpi_map and allocate them. */
3486         idx = bitmap_find_free_region(dev->event_map.lpi_map,
3487                                       dev->event_map.nr_lpis,
3488                                       get_count_order(nvecs));
3489         if (idx < 0)
3490                 return -ENOSPC;
3491
3492         *hwirq = dev->event_map.lpi_base + idx;
3493
3494         return 0;
3495 }
3496
3497 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
3498                            int nvec, msi_alloc_info_t *info)
3499 {
3500         struct its_node *its;
3501         struct its_device *its_dev;
3502         struct msi_domain_info *msi_info;
3503         u32 dev_id;
3504         int err = 0;
3505
3506         /*
3507          * We ignore "dev" entirely, and rely on the dev_id that has
3508          * been passed via the scratchpad. This limits this domain's
3509          * usefulness to upper layers that definitely know that they
3510          * are built on top of the ITS.
3511          */
3512         dev_id = info->scratchpad[0].ul;
3513
3514         msi_info = msi_get_domain_info(domain);
3515         its = msi_info->data;
3516
3517         if (!gic_rdists->has_direct_lpi &&
3518             vpe_proxy.dev &&
3519             vpe_proxy.dev->its == its &&
3520             dev_id == vpe_proxy.dev->device_id) {
3521                 /* Bad luck. Get yourself a better implementation */
3522                 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
3523                           dev_id);
3524                 return -EINVAL;
3525         }
3526
3527         mutex_lock(&its->dev_alloc_lock);
3528         its_dev = its_find_device(its, dev_id);
3529         if (its_dev) {
3530                 /*
3531                  * We already have seen this ID, probably through
3532                  * another alias (PCI bridge of some sort). No need to
3533                  * create the device.
3534                  */
3535                 its_dev->shared = true;
3536                 pr_debug("Reusing ITT for devID %x\n", dev_id);
3537                 goto out;
3538         }
3539
3540         its_dev = its_create_device(its, dev_id, nvec, true);
3541         if (!its_dev) {
3542                 err = -ENOMEM;
3543                 goto out;
3544         }
3545
3546         if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE)
3547                 its_dev->shared = true;
3548
3549         pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
3550 out:
3551         mutex_unlock(&its->dev_alloc_lock);
3552         info->scratchpad[0].ptr = its_dev;
3553         return err;
3554 }
3555
3556 static struct msi_domain_ops its_msi_domain_ops = {
3557         .msi_prepare    = its_msi_prepare,
3558 };
3559
3560 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
3561                                     unsigned int virq,
3562                                     irq_hw_number_t hwirq)
3563 {
3564         struct irq_fwspec fwspec;
3565
3566         if (irq_domain_get_of_node(domain->parent)) {
3567                 fwspec.fwnode = domain->parent->fwnode;
3568                 fwspec.param_count = 3;
3569                 fwspec.param[0] = GIC_IRQ_TYPE_LPI;
3570                 fwspec.param[1] = hwirq;
3571                 fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
3572         } else if (is_fwnode_irqchip(domain->parent->fwnode)) {
3573                 fwspec.fwnode = domain->parent->fwnode;
3574                 fwspec.param_count = 2;
3575                 fwspec.param[0] = hwirq;
3576                 fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
3577         } else {
3578                 return -EINVAL;
3579         }
3580
3581         return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
3582 }
3583
3584 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3585                                 unsigned int nr_irqs, void *args)
3586 {
3587         msi_alloc_info_t *info = args;
3588         struct its_device *its_dev = info->scratchpad[0].ptr;
3589         struct its_node *its = its_dev->its;
3590         struct irq_data *irqd;
3591         irq_hw_number_t hwirq;
3592         int err;
3593         int i;
3594
3595         err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
3596         if (err)
3597                 return err;
3598
3599         err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
3600         if (err)
3601                 return err;
3602
3603         for (i = 0; i < nr_irqs; i++) {
3604                 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
3605                 if (err)
3606                         return err;
3607
3608                 irq_domain_set_hwirq_and_chip(domain, virq + i,
3609                                               hwirq + i, &its_irq_chip, its_dev);
3610                 irqd = irq_get_irq_data(virq + i);
3611                 irqd_set_single_target(irqd);
3612                 irqd_set_affinity_on_activate(irqd);
3613                 irqd_set_resend_when_in_progress(irqd);
3614                 pr_debug("ID:%d pID:%d vID:%d\n",
3615                          (int)(hwirq + i - its_dev->event_map.lpi_base),
3616                          (int)(hwirq + i), virq + i);
3617         }
3618
3619         return 0;
3620 }
3621
3622 static int its_irq_domain_activate(struct irq_domain *domain,
3623                                    struct irq_data *d, bool reserve)
3624 {
3625         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3626         u32 event = its_get_event_id(d);
3627         int cpu;
3628
3629         cpu = its_select_cpu(d, cpu_online_mask);
3630         if (cpu < 0 || cpu >= nr_cpu_ids)
3631                 return -EINVAL;
3632
3633         its_inc_lpi_count(d, cpu);
3634         its_dev->event_map.col_map[event] = cpu;
3635         irq_data_update_effective_affinity(d, cpumask_of(cpu));
3636
3637         /* Map the GIC IRQ and event to the device */
3638         its_send_mapti(its_dev, d->hwirq, event);
3639         return 0;
3640 }
3641
3642 static void its_irq_domain_deactivate(struct irq_domain *domain,
3643                                       struct irq_data *d)
3644 {
3645         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3646         u32 event = its_get_event_id(d);
3647
3648         its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
3649         /* Stop the delivery of interrupts */
3650         its_send_discard(its_dev, event);
3651 }
3652
3653 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
3654                                 unsigned int nr_irqs)
3655 {
3656         struct irq_data *d = irq_domain_get_irq_data(domain, virq);
3657         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3658         struct its_node *its = its_dev->its;
3659         int i;
3660
3661         bitmap_release_region(its_dev->event_map.lpi_map,
3662                               its_get_event_id(irq_domain_get_irq_data(domain, virq)),
3663                               get_count_order(nr_irqs));
3664
3665         for (i = 0; i < nr_irqs; i++) {
3666                 struct irq_data *data = irq_domain_get_irq_data(domain,
3667                                                                 virq + i);
3668                 /* Nuke the entry in the domain */
3669                 irq_domain_reset_irq_data(data);
3670         }
3671
3672         mutex_lock(&its->dev_alloc_lock);
3673
3674         /*
3675          * If all interrupts have been freed, start mopping the
3676          * floor. This is conditioned on the device not being shared.
3677          */
3678         if (!its_dev->shared &&
3679             bitmap_empty(its_dev->event_map.lpi_map,
3680                          its_dev->event_map.nr_lpis)) {
3681                 its_lpi_free(its_dev->event_map.lpi_map,
3682                              its_dev->event_map.lpi_base,
3683                              its_dev->event_map.nr_lpis);
3684
3685                 /* Unmap device/itt */
3686                 its_send_mapd(its_dev, 0);
3687                 its_free_device(its_dev);
3688         }
3689
3690         mutex_unlock(&its->dev_alloc_lock);
3691
3692         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
3693 }
3694
3695 static const struct irq_domain_ops its_domain_ops = {
3696         .alloc                  = its_irq_domain_alloc,
3697         .free                   = its_irq_domain_free,
3698         .activate               = its_irq_domain_activate,
3699         .deactivate             = its_irq_domain_deactivate,
3700 };
3701
3702 /*
3703  * This is insane.
3704  *
3705  * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
3706  * likely), the only way to perform an invalidate is to use a fake
3707  * device to issue an INV command, implying that the LPI has first
3708  * been mapped to some event on that device. Since this is not exactly
3709  * cheap, we try to keep that mapping around as long as possible, and
3710  * only issue an UNMAP if we're short on available slots.
3711  *
3712  * Broken by design(tm).
3713  *
3714  * GICv4.1, on the other hand, mandates that we're able to invalidate
3715  * by writing to a MMIO register. It doesn't implement the whole of
3716  * DirectLPI, but that's good enough. And most of the time, we don't
3717  * even have to invalidate anything, as the redistributor can be told
3718  * whether to generate a doorbell or not (we thus leave it enabled,
3719  * always).
3720  */
3721 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
3722 {
3723         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3724         if (gic_rdists->has_rvpeid)
3725                 return;
3726
3727         /* Already unmapped? */
3728         if (vpe->vpe_proxy_event == -1)
3729                 return;
3730
3731         its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
3732         vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
3733
3734         /*
3735          * We don't track empty slots at all, so let's move the
3736          * next_victim pointer if we can quickly reuse that slot
3737          * instead of nuking an existing entry. Not clear that this is
3738          * always a win though, and this might just generate a ripple
3739          * effect... Let's just hope VPEs don't migrate too often.
3740          */
3741         if (vpe_proxy.vpes[vpe_proxy.next_victim])
3742                 vpe_proxy.next_victim = vpe->vpe_proxy_event;
3743
3744         vpe->vpe_proxy_event = -1;
3745 }
3746
3747 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
3748 {
3749         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3750         if (gic_rdists->has_rvpeid)
3751                 return;
3752
3753         if (!gic_rdists->has_direct_lpi) {
3754                 unsigned long flags;
3755
3756                 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3757                 its_vpe_db_proxy_unmap_locked(vpe);
3758                 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3759         }
3760 }
3761
3762 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
3763 {
3764         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3765         if (gic_rdists->has_rvpeid)
3766                 return;
3767
3768         /* Already mapped? */
3769         if (vpe->vpe_proxy_event != -1)
3770                 return;
3771
3772         /* This slot was already allocated. Kick the other VPE out. */
3773         if (vpe_proxy.vpes[vpe_proxy.next_victim])
3774                 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
3775
3776         /* Map the new VPE instead */
3777         vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
3778         vpe->vpe_proxy_event = vpe_proxy.next_victim;
3779         vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
3780
3781         vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
3782         its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
3783 }
3784
3785 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3786 {
3787         unsigned long flags;
3788         struct its_collection *target_col;
3789
3790         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3791         if (gic_rdists->has_rvpeid)
3792                 return;
3793
3794         if (gic_rdists->has_direct_lpi) {
3795                 void __iomem *rdbase;
3796
3797                 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
3798                 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3799                 wait_for_syncr(rdbase);
3800
3801                 return;
3802         }
3803
3804         raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3805
3806         its_vpe_db_proxy_map_locked(vpe);
3807
3808         target_col = &vpe_proxy.dev->its->collections[to];
3809         its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3810         vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3811
3812         raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3813 }
3814
3815 static int its_vpe_set_affinity(struct irq_data *d,
3816                                 const struct cpumask *mask_val,
3817                                 bool force)
3818 {
3819         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3820         int from, cpu = cpumask_first(mask_val);
3821         unsigned long flags;
3822
3823         /*
3824          * Changing affinity is mega expensive, so let's be as lazy as
3825          * we can and only do it if we really have to. Also, if mapped
3826          * into the proxy device, we need to move the doorbell
3827          * interrupt to its new location.
3828          *
3829          * Another thing is that changing the affinity of a vPE affects
3830          * *other interrupts* such as all the vLPIs that are routed to
3831          * this vPE. This means that the irq_desc lock is not enough to
3832          * protect us, and that we must ensure nobody samples vpe->col_idx
3833          * during the update, hence the lock below which must also be
3834          * taken on any vLPI handling path that evaluates vpe->col_idx.
3835          */
3836         from = vpe_to_cpuid_lock(vpe, &flags);
3837         if (from == cpu)
3838                 goto out;
3839
3840         vpe->col_idx = cpu;
3841
3842         /*
3843          * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD
3844          * is sharing its VPE table with the current one.
3845          */
3846         if (gic_data_rdist_cpu(cpu)->vpe_table_mask &&
3847             cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask))
3848                 goto out;
3849
3850         its_send_vmovp(vpe);
3851         its_vpe_db_proxy_move(vpe, from, cpu);
3852
3853 out:
3854         irq_data_update_effective_affinity(d, cpumask_of(cpu));
3855         vpe_to_cpuid_unlock(vpe, flags);
3856
3857         return IRQ_SET_MASK_OK_DONE;
3858 }
3859
3860 static void its_wait_vpt_parse_complete(void)
3861 {
3862         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3863         u64 val;
3864
3865         if (!gic_rdists->has_vpend_valid_dirty)
3866                 return;
3867
3868         WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
3869                                                        val,
3870                                                        !(val & GICR_VPENDBASER_Dirty),
3871                                                        1, 500));
3872 }
3873
3874 static void its_vpe_schedule(struct its_vpe *vpe)
3875 {
3876         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3877         u64 val;
3878
3879         /* Schedule the VPE */
3880         val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
3881                 GENMASK_ULL(51, 12);
3882         val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3883         val |= GICR_VPROPBASER_RaWb;
3884         val |= GICR_VPROPBASER_InnerShareable;
3885         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3886
3887         val  = virt_to_phys(page_address(vpe->vpt_page)) &
3888                 GENMASK_ULL(51, 16);
3889         val |= GICR_VPENDBASER_RaWaWb;
3890         val |= GICR_VPENDBASER_InnerShareable;
3891         /*
3892          * There is no good way of finding out if the pending table is
3893          * empty as we can race against the doorbell interrupt very
3894          * easily. So in the end, vpe->pending_last is only an
3895          * indication that the vcpu has something pending, not one
3896          * that the pending table is empty. A good implementation
3897          * would be able to read its coarse map pretty quickly anyway,
3898          * making this a tolerable issue.
3899          */
3900         val |= GICR_VPENDBASER_PendingLast;
3901         val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
3902         val |= GICR_VPENDBASER_Valid;
3903         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3904 }
3905
3906 static void its_vpe_deschedule(struct its_vpe *vpe)
3907 {
3908         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3909         u64 val;
3910
3911         val = its_clear_vpend_valid(vlpi_base, 0, 0);
3912
3913         vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
3914         vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
3915 }
3916
3917 static void its_vpe_invall(struct its_vpe *vpe)
3918 {
3919         struct its_node *its;
3920
3921         list_for_each_entry(its, &its_nodes, entry) {
3922                 if (!is_v4(its))
3923                         continue;
3924
3925                 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
3926                         continue;
3927
3928                 /*
3929                  * Sending a VINVALL to a single ITS is enough, as all
3930                  * we need is to reach the redistributors.
3931                  */
3932                 its_send_vinvall(its, vpe);
3933                 return;
3934         }
3935 }
3936
3937 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
3938 {
3939         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3940         struct its_cmd_info *info = vcpu_info;
3941
3942         switch (info->cmd_type) {
3943         case SCHEDULE_VPE:
3944                 its_vpe_schedule(vpe);
3945                 return 0;
3946
3947         case DESCHEDULE_VPE:
3948                 its_vpe_deschedule(vpe);
3949                 return 0;
3950
3951         case COMMIT_VPE:
3952                 its_wait_vpt_parse_complete();
3953                 return 0;
3954
3955         case INVALL_VPE:
3956                 its_vpe_invall(vpe);
3957                 return 0;
3958
3959         default:
3960                 return -EINVAL;
3961         }
3962 }
3963
3964 static void its_vpe_send_cmd(struct its_vpe *vpe,
3965                              void (*cmd)(struct its_device *, u32))
3966 {
3967         unsigned long flags;
3968
3969         raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3970
3971         its_vpe_db_proxy_map_locked(vpe);
3972         cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
3973
3974         raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3975 }
3976
3977 static void its_vpe_send_inv(struct irq_data *d)
3978 {
3979         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3980
3981         if (gic_rdists->has_direct_lpi)
3982                 __direct_lpi_inv(d, d->parent_data->hwirq);
3983         else
3984                 its_vpe_send_cmd(vpe, its_send_inv);
3985 }
3986
3987 static void its_vpe_mask_irq(struct irq_data *d)
3988 {
3989         /*
3990          * We need to unmask the LPI, which is described by the parent
3991          * irq_data. Instead of calling into the parent (which won't
3992          * exactly do the right thing, let's simply use the
3993          * parent_data pointer. Yes, I'm naughty.
3994          */
3995         lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
3996         its_vpe_send_inv(d);
3997 }
3998
3999 static void its_vpe_unmask_irq(struct irq_data *d)
4000 {
4001         /* Same hack as above... */
4002         lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4003         its_vpe_send_inv(d);
4004 }
4005
4006 static int its_vpe_set_irqchip_state(struct irq_data *d,
4007                                      enum irqchip_irq_state which,
4008                                      bool state)
4009 {
4010         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4011
4012         if (which != IRQCHIP_STATE_PENDING)
4013                 return -EINVAL;
4014
4015         if (gic_rdists->has_direct_lpi) {
4016                 void __iomem *rdbase;
4017
4018                 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
4019                 if (state) {
4020                         gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
4021                 } else {
4022                         gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
4023                         wait_for_syncr(rdbase);
4024                 }
4025         } else {
4026                 if (state)
4027                         its_vpe_send_cmd(vpe, its_send_int);
4028                 else
4029                         its_vpe_send_cmd(vpe, its_send_clear);
4030         }
4031
4032         return 0;
4033 }
4034
4035 static int its_vpe_retrigger(struct irq_data *d)
4036 {
4037         return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
4038 }
4039
4040 static struct irq_chip its_vpe_irq_chip = {
4041         .name                   = "GICv4-vpe",
4042         .irq_mask               = its_vpe_mask_irq,
4043         .irq_unmask             = its_vpe_unmask_irq,
4044         .irq_eoi                = irq_chip_eoi_parent,
4045         .irq_set_affinity       = its_vpe_set_affinity,
4046         .irq_retrigger          = its_vpe_retrigger,
4047         .irq_set_irqchip_state  = its_vpe_set_irqchip_state,
4048         .irq_set_vcpu_affinity  = its_vpe_set_vcpu_affinity,
4049 };
4050
4051 static struct its_node *find_4_1_its(void)
4052 {
4053         static struct its_node *its = NULL;
4054
4055         if (!its) {
4056                 list_for_each_entry(its, &its_nodes, entry) {
4057                         if (is_v4_1(its))
4058                                 return its;
4059                 }
4060
4061                 /* Oops? */
4062                 its = NULL;
4063         }
4064
4065         return its;
4066 }
4067
4068 static void its_vpe_4_1_send_inv(struct irq_data *d)
4069 {
4070         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4071         struct its_node *its;
4072
4073         /*
4074          * GICv4.1 wants doorbells to be invalidated using the
4075          * INVDB command in order to be broadcast to all RDs. Send
4076          * it to the first valid ITS, and let the HW do its magic.
4077          */
4078         its = find_4_1_its();
4079         if (its)
4080                 its_send_invdb(its, vpe);
4081 }
4082
4083 static void its_vpe_4_1_mask_irq(struct irq_data *d)
4084 {
4085         lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4086         its_vpe_4_1_send_inv(d);
4087 }
4088
4089 static void its_vpe_4_1_unmask_irq(struct irq_data *d)
4090 {
4091         lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4092         its_vpe_4_1_send_inv(d);
4093 }
4094
4095 static void its_vpe_4_1_schedule(struct its_vpe *vpe,
4096                                  struct its_cmd_info *info)
4097 {
4098         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4099         u64 val = 0;
4100
4101         /* Schedule the VPE */
4102         val |= GICR_VPENDBASER_Valid;
4103         val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
4104         val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
4105         val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
4106
4107         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
4108 }
4109
4110 static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4111                                    struct its_cmd_info *info)
4112 {
4113         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4114         u64 val;
4115
4116         if (info->req_db) {
4117                 unsigned long flags;
4118
4119                 /*
4120                  * vPE is going to block: make the vPE non-resident with
4121                  * PendingLast clear and DB set. The GIC guarantees that if
4122                  * we read-back PendingLast clear, then a doorbell will be
4123                  * delivered when an interrupt comes.
4124                  *
4125                  * Note the locking to deal with the concurrent update of
4126                  * pending_last from the doorbell interrupt handler that can
4127                  * run concurrently.
4128                  */
4129                 raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
4130                 val = its_clear_vpend_valid(vlpi_base,
4131                                             GICR_VPENDBASER_PendingLast,
4132                                             GICR_VPENDBASER_4_1_DB);
4133                 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4134                 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
4135         } else {
4136                 /*
4137                  * We're not blocking, so just make the vPE non-resident
4138                  * with PendingLast set, indicating that we'll be back.
4139                  */
4140                 val = its_clear_vpend_valid(vlpi_base,
4141                                             0,
4142                                             GICR_VPENDBASER_PendingLast);
4143                 vpe->pending_last = true;
4144         }
4145 }
4146
4147 static void its_vpe_4_1_invall(struct its_vpe *vpe)
4148 {
4149         void __iomem *rdbase;
4150         unsigned long flags;
4151         u64 val;
4152         int cpu;
4153
4154         val  = GICR_INVALLR_V;
4155         val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
4156
4157         /* Target the redistributor this vPE is currently known on */
4158         cpu = vpe_to_cpuid_lock(vpe, &flags);
4159         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4160         rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
4161         gic_write_lpir(val, rdbase + GICR_INVALLR);
4162
4163         wait_for_syncr(rdbase);
4164         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4165         vpe_to_cpuid_unlock(vpe, flags);
4166 }
4167
4168 static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4169 {
4170         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4171         struct its_cmd_info *info = vcpu_info;
4172
4173         switch (info->cmd_type) {
4174         case SCHEDULE_VPE:
4175                 its_vpe_4_1_schedule(vpe, info);
4176                 return 0;
4177
4178         case DESCHEDULE_VPE:
4179                 its_vpe_4_1_deschedule(vpe, info);
4180                 return 0;
4181
4182         case COMMIT_VPE:
4183                 its_wait_vpt_parse_complete();
4184                 return 0;
4185
4186         case INVALL_VPE:
4187                 its_vpe_4_1_invall(vpe);
4188                 return 0;
4189
4190         default:
4191                 return -EINVAL;
4192         }
4193 }
4194
4195 static struct irq_chip its_vpe_4_1_irq_chip = {
4196         .name                   = "GICv4.1-vpe",
4197         .irq_mask               = its_vpe_4_1_mask_irq,
4198         .irq_unmask             = its_vpe_4_1_unmask_irq,
4199         .irq_eoi                = irq_chip_eoi_parent,
4200         .irq_set_affinity       = its_vpe_set_affinity,
4201         .irq_set_vcpu_affinity  = its_vpe_4_1_set_vcpu_affinity,
4202 };
4203
4204 static void its_configure_sgi(struct irq_data *d, bool clear)
4205 {
4206         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4207         struct its_cmd_desc desc;
4208
4209         desc.its_vsgi_cmd.vpe = vpe;
4210         desc.its_vsgi_cmd.sgi = d->hwirq;
4211         desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4212         desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4213         desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4214         desc.its_vsgi_cmd.clear = clear;
4215
4216         /*
4217          * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4218          * destination VPE is mapped there. Since we map them eagerly at
4219          * activation time, we're pretty sure the first GICv4.1 ITS will do.
4220          */
4221         its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
4222 }
4223
4224 static void its_sgi_mask_irq(struct irq_data *d)
4225 {
4226         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4227
4228         vpe->sgi_config[d->hwirq].enabled = false;
4229         its_configure_sgi(d, false);
4230 }
4231
4232 static void its_sgi_unmask_irq(struct irq_data *d)
4233 {
4234         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4235
4236         vpe->sgi_config[d->hwirq].enabled = true;
4237         its_configure_sgi(d, false);
4238 }
4239
4240 static int its_sgi_set_affinity(struct irq_data *d,
4241                                 const struct cpumask *mask_val,
4242                                 bool force)
4243 {
4244         /*
4245          * There is no notion of affinity for virtual SGIs, at least
4246          * not on the host (since they can only be targeting a vPE).
4247          * Tell the kernel we've done whatever it asked for.
4248          */
4249         irq_data_update_effective_affinity(d, mask_val);
4250         return IRQ_SET_MASK_OK;
4251 }
4252
4253 static int its_sgi_set_irqchip_state(struct irq_data *d,
4254                                      enum irqchip_irq_state which,
4255                                      bool state)
4256 {
4257         if (which != IRQCHIP_STATE_PENDING)
4258                 return -EINVAL;
4259
4260         if (state) {
4261                 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4262                 struct its_node *its = find_4_1_its();
4263                 u64 val;
4264
4265                 val  = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
4266                 val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
4267                 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
4268         } else {
4269                 its_configure_sgi(d, true);
4270         }
4271
4272         return 0;
4273 }
4274
4275 static int its_sgi_get_irqchip_state(struct irq_data *d,
4276                                      enum irqchip_irq_state which, bool *val)
4277 {
4278         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4279         void __iomem *base;
4280         unsigned long flags;
4281         u32 count = 1000000;    /* 1s! */
4282         u32 status;
4283         int cpu;
4284
4285         if (which != IRQCHIP_STATE_PENDING)
4286                 return -EINVAL;
4287
4288         /*
4289          * Locking galore! We can race against two different events:
4290          *
4291          * - Concurrent vPE affinity change: we must make sure it cannot
4292          *   happen, or we'll talk to the wrong redistributor. This is
4293          *   identical to what happens with vLPIs.
4294          *
4295          * - Concurrent VSGIPENDR access: As it involves accessing two
4296          *   MMIO registers, this must be made atomic one way or another.
4297          */
4298         cpu = vpe_to_cpuid_lock(vpe, &flags);
4299         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4300         base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
4301         writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
4302         do {
4303                 status = readl_relaxed(base + GICR_VSGIPENDR);
4304                 if (!(status & GICR_VSGIPENDR_BUSY))
4305                         goto out;
4306
4307                 count--;
4308                 if (!count) {
4309                         pr_err_ratelimited("Unable to get SGI status\n");
4310                         goto out;
4311                 }
4312                 cpu_relax();
4313                 udelay(1);
4314         } while (count);
4315
4316 out:
4317         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4318         vpe_to_cpuid_unlock(vpe, flags);
4319
4320         if (!count)
4321                 return -ENXIO;
4322
4323         *val = !!(status & (1 << d->hwirq));
4324
4325         return 0;
4326 }
4327
4328 static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4329 {
4330         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4331         struct its_cmd_info *info = vcpu_info;
4332
4333         switch (info->cmd_type) {
4334         case PROP_UPDATE_VSGI:
4335                 vpe->sgi_config[d->hwirq].priority = info->priority;
4336                 vpe->sgi_config[d->hwirq].group = info->group;
4337                 its_configure_sgi(d, false);
4338                 return 0;
4339
4340         default:
4341                 return -EINVAL;
4342         }
4343 }
4344
4345 static struct irq_chip its_sgi_irq_chip = {
4346         .name                   = "GICv4.1-sgi",
4347         .irq_mask               = its_sgi_mask_irq,
4348         .irq_unmask             = its_sgi_unmask_irq,
4349         .irq_set_affinity       = its_sgi_set_affinity,
4350         .irq_set_irqchip_state  = its_sgi_set_irqchip_state,
4351         .irq_get_irqchip_state  = its_sgi_get_irqchip_state,
4352         .irq_set_vcpu_affinity  = its_sgi_set_vcpu_affinity,
4353 };
4354
4355 static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
4356                                     unsigned int virq, unsigned int nr_irqs,
4357                                     void *args)
4358 {
4359         struct its_vpe *vpe = args;
4360         int i;
4361
4362         /* Yes, we do want 16 SGIs */
4363         WARN_ON(nr_irqs != 16);
4364
4365         for (i = 0; i < 16; i++) {
4366                 vpe->sgi_config[i].priority = 0;
4367                 vpe->sgi_config[i].enabled = false;
4368                 vpe->sgi_config[i].group = false;
4369
4370                 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4371                                               &its_sgi_irq_chip, vpe);
4372                 irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
4373         }
4374
4375         return 0;
4376 }
4377
4378 static void its_sgi_irq_domain_free(struct irq_domain *domain,
4379                                     unsigned int virq,
4380                                     unsigned int nr_irqs)
4381 {
4382         /* Nothing to do */
4383 }
4384
4385 static int its_sgi_irq_domain_activate(struct irq_domain *domain,
4386                                        struct irq_data *d, bool reserve)
4387 {
4388         /* Write out the initial SGI configuration */
4389         its_configure_sgi(d, false);
4390         return 0;
4391 }
4392
4393 static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
4394                                           struct irq_data *d)
4395 {
4396         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4397
4398         /*
4399          * The VSGI command is awkward:
4400          *
4401          * - To change the configuration, CLEAR must be set to false,
4402          *   leaving the pending bit unchanged.
4403          * - To clear the pending bit, CLEAR must be set to true, leaving
4404          *   the configuration unchanged.
4405          *
4406          * You just can't do both at once, hence the two commands below.
4407          */
4408         vpe->sgi_config[d->hwirq].enabled = false;
4409         its_configure_sgi(d, false);
4410         its_configure_sgi(d, true);
4411 }
4412
4413 static const struct irq_domain_ops its_sgi_domain_ops = {
4414         .alloc          = its_sgi_irq_domain_alloc,
4415         .free           = its_sgi_irq_domain_free,
4416         .activate       = its_sgi_irq_domain_activate,
4417         .deactivate     = its_sgi_irq_domain_deactivate,
4418 };
4419
4420 static int its_vpe_id_alloc(void)
4421 {
4422         return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
4423 }
4424
4425 static void its_vpe_id_free(u16 id)
4426 {
4427         ida_simple_remove(&its_vpeid_ida, id);
4428 }
4429
4430 static int its_vpe_init(struct its_vpe *vpe)
4431 {
4432         struct page *vpt_page;
4433         int vpe_id;
4434
4435         /* Allocate vpe_id */
4436         vpe_id = its_vpe_id_alloc();
4437         if (vpe_id < 0)
4438                 return vpe_id;
4439
4440         /* Allocate VPT */
4441         vpt_page = its_allocate_pending_table(GFP_KERNEL);
4442         if (!vpt_page) {
4443                 its_vpe_id_free(vpe_id);
4444                 return -ENOMEM;
4445         }
4446
4447         if (!its_alloc_vpe_table(vpe_id)) {
4448                 its_vpe_id_free(vpe_id);
4449                 its_free_pending_table(vpt_page);
4450                 return -ENOMEM;
4451         }
4452
4453         raw_spin_lock_init(&vpe->vpe_lock);
4454         vpe->vpe_id = vpe_id;
4455         vpe->vpt_page = vpt_page;
4456         if (gic_rdists->has_rvpeid)
4457                 atomic_set(&vpe->vmapp_count, 0);
4458         else
4459                 vpe->vpe_proxy_event = -1;
4460
4461         return 0;
4462 }
4463
4464 static void its_vpe_teardown(struct its_vpe *vpe)
4465 {
4466         its_vpe_db_proxy_unmap(vpe);
4467         its_vpe_id_free(vpe->vpe_id);
4468         its_free_pending_table(vpe->vpt_page);
4469 }
4470
4471 static void its_vpe_irq_domain_free(struct irq_domain *domain,
4472                                     unsigned int virq,
4473                                     unsigned int nr_irqs)
4474 {
4475         struct its_vm *vm = domain->host_data;
4476         int i;
4477
4478         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
4479
4480         for (i = 0; i < nr_irqs; i++) {
4481                 struct irq_data *data = irq_domain_get_irq_data(domain,
4482                                                                 virq + i);
4483                 struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
4484
4485                 BUG_ON(vm != vpe->its_vm);
4486
4487                 clear_bit(data->hwirq, vm->db_bitmap);
4488                 its_vpe_teardown(vpe);
4489                 irq_domain_reset_irq_data(data);
4490         }
4491
4492         if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
4493                 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
4494                 its_free_prop_table(vm->vprop_page);
4495         }
4496 }
4497
4498 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
4499                                     unsigned int nr_irqs, void *args)
4500 {
4501         struct irq_chip *irqchip = &its_vpe_irq_chip;
4502         struct its_vm *vm = args;
4503         unsigned long *bitmap;
4504         struct page *vprop_page;
4505         int base, nr_ids, i, err = 0;
4506
4507         BUG_ON(!vm);
4508
4509         bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
4510         if (!bitmap)
4511                 return -ENOMEM;
4512
4513         if (nr_ids < nr_irqs) {
4514                 its_lpi_free(bitmap, base, nr_ids);
4515                 return -ENOMEM;
4516         }
4517
4518         vprop_page = its_allocate_prop_table(GFP_KERNEL);
4519         if (!vprop_page) {
4520                 its_lpi_free(bitmap, base, nr_ids);
4521                 return -ENOMEM;
4522         }
4523
4524         vm->db_bitmap = bitmap;
4525         vm->db_lpi_base = base;
4526         vm->nr_db_lpis = nr_ids;
4527         vm->vprop_page = vprop_page;
4528
4529         if (gic_rdists->has_rvpeid)
4530                 irqchip = &its_vpe_4_1_irq_chip;
4531
4532         for (i = 0; i < nr_irqs; i++) {
4533                 vm->vpes[i]->vpe_db_lpi = base + i;
4534                 err = its_vpe_init(vm->vpes[i]);
4535                 if (err)
4536                         break;
4537                 err = its_irq_gic_domain_alloc(domain, virq + i,
4538                                                vm->vpes[i]->vpe_db_lpi);
4539                 if (err)
4540                         break;
4541                 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4542                                               irqchip, vm->vpes[i]);
4543                 set_bit(i, bitmap);
4544                 irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i));
4545         }
4546
4547         if (err) {
4548                 if (i > 0)
4549                         its_vpe_irq_domain_free(domain, virq, i);
4550
4551                 its_lpi_free(bitmap, base, nr_ids);
4552                 its_free_prop_table(vprop_page);
4553         }
4554
4555         return err;
4556 }
4557
4558 static int its_vpe_irq_domain_activate(struct irq_domain *domain,
4559                                        struct irq_data *d, bool reserve)
4560 {
4561         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4562         struct its_node *its;
4563
4564         /*
4565          * If we use the list map, we issue VMAPP on demand... Unless
4566          * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4567          * so that VSGIs can work.
4568          */
4569         if (!gic_requires_eager_mapping())
4570                 return 0;
4571
4572         /* Map the VPE to the first possible CPU */
4573         vpe->col_idx = cpumask_first(cpu_online_mask);
4574
4575         list_for_each_entry(its, &its_nodes, entry) {
4576                 if (!is_v4(its))
4577                         continue;
4578
4579                 its_send_vmapp(its, vpe, true);
4580                 its_send_vinvall(its, vpe);
4581         }
4582
4583         irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
4584
4585         return 0;
4586 }
4587
4588 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
4589                                           struct irq_data *d)
4590 {
4591         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4592         struct its_node *its;
4593
4594         /*
4595          * If we use the list map on GICv4.0, we unmap the VPE once no
4596          * VLPIs are associated with the VM.
4597          */
4598         if (!gic_requires_eager_mapping())
4599                 return;
4600
4601         list_for_each_entry(its, &its_nodes, entry) {
4602                 if (!is_v4(its))
4603                         continue;
4604
4605                 its_send_vmapp(its, vpe, false);
4606         }
4607
4608         /*
4609          * There may be a direct read to the VPT after unmapping the
4610          * vPE, to guarantee the validity of this, we make the VPT
4611          * memory coherent with the CPU caches here.
4612          */
4613         if (find_4_1_its() && !atomic_read(&vpe->vmapp_count))
4614                 gic_flush_dcache_to_poc(page_address(vpe->vpt_page),
4615                                         LPI_PENDBASE_SZ);
4616 }
4617
4618 static const struct irq_domain_ops its_vpe_domain_ops = {
4619         .alloc                  = its_vpe_irq_domain_alloc,
4620         .free                   = its_vpe_irq_domain_free,
4621         .activate               = its_vpe_irq_domain_activate,
4622         .deactivate             = its_vpe_irq_domain_deactivate,
4623 };
4624
4625 static int its_force_quiescent(void __iomem *base)
4626 {
4627         u32 count = 1000000;    /* 1s */
4628         u32 val;
4629
4630         val = readl_relaxed(base + GITS_CTLR);
4631         /*
4632          * GIC architecture specification requires the ITS to be both
4633          * disabled and quiescent for writes to GITS_BASER<n> or
4634          * GITS_CBASER to not have UNPREDICTABLE results.
4635          */
4636         if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
4637                 return 0;
4638
4639         /* Disable the generation of all interrupts to this ITS */
4640         val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
4641         writel_relaxed(val, base + GITS_CTLR);
4642
4643         /* Poll GITS_CTLR and wait until ITS becomes quiescent */
4644         while (1) {
4645                 val = readl_relaxed(base + GITS_CTLR);
4646                 if (val & GITS_CTLR_QUIESCENT)
4647                         return 0;
4648
4649                 count--;
4650                 if (!count)
4651                         return -EBUSY;
4652
4653                 cpu_relax();
4654                 udelay(1);
4655         }
4656 }
4657
4658 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
4659 {
4660         struct its_node *its = data;
4661
4662         /* erratum 22375: only alloc 8MB table size (20 bits) */
4663         its->typer &= ~GITS_TYPER_DEVBITS;
4664         its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
4665         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
4666
4667         return true;
4668 }
4669
4670 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
4671 {
4672         struct its_node *its = data;
4673
4674         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
4675
4676         return true;
4677 }
4678
4679 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
4680 {
4681         struct its_node *its = data;
4682
4683         /* On QDF2400, the size of the ITE is 16Bytes */
4684         its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
4685         its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
4686
4687         return true;
4688 }
4689
4690 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
4691 {
4692         struct its_node *its = its_dev->its;
4693
4694         /*
4695          * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4696          * which maps 32-bit writes targeted at a separate window of
4697          * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4698          * with device ID taken from bits [device_id_bits + 1:2] of
4699          * the window offset.
4700          */
4701         return its->pre_its_base + (its_dev->device_id << 2);
4702 }
4703
4704 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
4705 {
4706         struct its_node *its = data;
4707         u32 pre_its_window[2];
4708         u32 ids;
4709
4710         if (!fwnode_property_read_u32_array(its->fwnode_handle,
4711                                            "socionext,synquacer-pre-its",
4712                                            pre_its_window,
4713                                            ARRAY_SIZE(pre_its_window))) {
4714
4715                 its->pre_its_base = pre_its_window[0];
4716                 its->get_msi_base = its_irq_get_msi_base_pre_its;
4717
4718                 ids = ilog2(pre_its_window[1]) - 2;
4719                 if (device_ids(its) > ids) {
4720                         its->typer &= ~GITS_TYPER_DEVBITS;
4721                         its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
4722                 }
4723
4724                 /* the pre-ITS breaks isolation, so disable MSI remapping */
4725                 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI;
4726                 return true;
4727         }
4728         return false;
4729 }
4730
4731 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
4732 {
4733         struct its_node *its = data;
4734
4735         /*
4736          * Hip07 insists on using the wrong address for the VLPI
4737          * page. Trick it into doing the right thing...
4738          */
4739         its->vlpi_redist_offset = SZ_128K;
4740         return true;
4741 }
4742
4743 static bool __maybe_unused its_enable_rk3588001(void *data)
4744 {
4745         struct its_node *its = data;
4746
4747         if (!of_machine_is_compatible("rockchip,rk3588") &&
4748             !of_machine_is_compatible("rockchip,rk3588s"))
4749                 return false;
4750
4751         its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4752         gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
4753
4754         return true;
4755 }
4756
4757 static bool its_set_non_coherent(void *data)
4758 {
4759         struct its_node *its = data;
4760
4761         its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4762         return true;
4763 }
4764
4765 static const struct gic_quirk its_quirks[] = {
4766 #ifdef CONFIG_CAVIUM_ERRATUM_22375
4767         {
4768                 .desc   = "ITS: Cavium errata 22375, 24313",
4769                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
4770                 .mask   = 0xffff0fff,
4771                 .init   = its_enable_quirk_cavium_22375,
4772         },
4773 #endif
4774 #ifdef CONFIG_CAVIUM_ERRATUM_23144
4775         {
4776                 .desc   = "ITS: Cavium erratum 23144",
4777                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
4778                 .mask   = 0xffff0fff,
4779                 .init   = its_enable_quirk_cavium_23144,
4780         },
4781 #endif
4782 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
4783         {
4784                 .desc   = "ITS: QDF2400 erratum 0065",
4785                 .iidr   = 0x00001070, /* QDF2400 ITS rev 1.x */
4786                 .mask   = 0xffffffff,
4787                 .init   = its_enable_quirk_qdf2400_e0065,
4788         },
4789 #endif
4790 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4791         {
4792                 /*
4793                  * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4794                  * implementation, but with a 'pre-ITS' added that requires
4795                  * special handling in software.
4796                  */
4797                 .desc   = "ITS: Socionext Synquacer pre-ITS",
4798                 .iidr   = 0x0001143b,
4799                 .mask   = 0xffffffff,
4800                 .init   = its_enable_quirk_socionext_synquacer,
4801         },
4802 #endif
4803 #ifdef CONFIG_HISILICON_ERRATUM_161600802
4804         {
4805                 .desc   = "ITS: Hip07 erratum 161600802",
4806                 .iidr   = 0x00000004,
4807                 .mask   = 0xffffffff,
4808                 .init   = its_enable_quirk_hip07_161600802,
4809         },
4810 #endif
4811 #ifdef CONFIG_ROCKCHIP_ERRATUM_3588001
4812         {
4813                 .desc   = "ITS: Rockchip erratum RK3588001",
4814                 .iidr   = 0x0201743b,
4815                 .mask   = 0xffffffff,
4816                 .init   = its_enable_rk3588001,
4817         },
4818 #endif
4819         {
4820                 .desc   = "ITS: non-coherent attribute",
4821                 .property = "dma-noncoherent",
4822                 .init   = its_set_non_coherent,
4823         },
4824         {
4825         }
4826 };
4827
4828 static void its_enable_quirks(struct its_node *its)
4829 {
4830         u32 iidr = readl_relaxed(its->base + GITS_IIDR);
4831
4832         gic_enable_quirks(iidr, its_quirks, its);
4833
4834         if (is_of_node(its->fwnode_handle))
4835                 gic_enable_of_quirks(to_of_node(its->fwnode_handle),
4836                                      its_quirks, its);
4837 }
4838
4839 static int its_save_disable(void)
4840 {
4841         struct its_node *its;
4842         int err = 0;
4843
4844         raw_spin_lock(&its_lock);
4845         list_for_each_entry(its, &its_nodes, entry) {
4846                 void __iomem *base;
4847
4848                 base = its->base;
4849                 its->ctlr_save = readl_relaxed(base + GITS_CTLR);
4850                 err = its_force_quiescent(base);
4851                 if (err) {
4852                         pr_err("ITS@%pa: failed to quiesce: %d\n",
4853                                &its->phys_base, err);
4854                         writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4855                         goto err;
4856                 }
4857
4858                 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
4859         }
4860
4861 err:
4862         if (err) {
4863                 list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
4864                         void __iomem *base;
4865
4866                         base = its->base;
4867                         writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4868                 }
4869         }
4870         raw_spin_unlock(&its_lock);
4871
4872         return err;
4873 }
4874
4875 static void its_restore_enable(void)
4876 {
4877         struct its_node *its;
4878         int ret;
4879
4880         raw_spin_lock(&its_lock);
4881         list_for_each_entry(its, &its_nodes, entry) {
4882                 void __iomem *base;
4883                 int i;
4884
4885                 base = its->base;
4886
4887                 /*
4888                  * Make sure that the ITS is disabled. If it fails to quiesce,
4889                  * don't restore it since writing to CBASER or BASER<n>
4890                  * registers is undefined according to the GIC v3 ITS
4891                  * Specification.
4892                  *
4893                  * Firmware resuming with the ITS enabled is terminally broken.
4894                  */
4895                 WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
4896                 ret = its_force_quiescent(base);
4897                 if (ret) {
4898                         pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
4899                                &its->phys_base, ret);
4900                         continue;
4901                 }
4902
4903                 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
4904
4905                 /*
4906                  * Writing CBASER resets CREADR to 0, so make CWRITER and
4907                  * cmd_write line up with it.
4908                  */
4909                 its->cmd_write = its->cmd_base;
4910                 gits_write_cwriter(0, base + GITS_CWRITER);
4911
4912                 /* Restore GITS_BASER from the value cache. */
4913                 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
4914                         struct its_baser *baser = &its->tables[i];
4915
4916                         if (!(baser->val & GITS_BASER_VALID))
4917                                 continue;
4918
4919                         its_write_baser(its, baser, baser->val);
4920                 }
4921                 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4922
4923                 /*
4924                  * Reinit the collection if it's stored in the ITS. This is
4925                  * indicated by the col_id being less than the HCC field.
4926                  * CID < HCC as specified in the GIC v3 Documentation.
4927                  */
4928                 if (its->collections[smp_processor_id()].col_id <
4929                     GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
4930                         its_cpu_init_collection(its);
4931         }
4932         raw_spin_unlock(&its_lock);
4933 }
4934
4935 static struct syscore_ops its_syscore_ops = {
4936         .suspend = its_save_disable,
4937         .resume = its_restore_enable,
4938 };
4939
4940 static void __init __iomem *its_map_one(struct resource *res, int *err)
4941 {
4942         void __iomem *its_base;
4943         u32 val;
4944
4945         its_base = ioremap(res->start, SZ_64K);
4946         if (!its_base) {
4947                 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
4948                 *err = -ENOMEM;
4949                 return NULL;
4950         }
4951
4952         val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
4953         if (val != 0x30 && val != 0x40) {
4954                 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
4955                 *err = -ENODEV;
4956                 goto out_unmap;
4957         }
4958
4959         *err = its_force_quiescent(its_base);
4960         if (*err) {
4961                 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
4962                 goto out_unmap;
4963         }
4964
4965         return its_base;
4966
4967 out_unmap:
4968         iounmap(its_base);
4969         return NULL;
4970 }
4971
4972 static int its_init_domain(struct its_node *its)
4973 {
4974         struct irq_domain *inner_domain;
4975         struct msi_domain_info *info;
4976
4977         info = kzalloc(sizeof(*info), GFP_KERNEL);
4978         if (!info)
4979                 return -ENOMEM;
4980
4981         info->ops = &its_msi_domain_ops;
4982         info->data = its;
4983
4984         inner_domain = irq_domain_create_hierarchy(its_parent,
4985                                                    its->msi_domain_flags, 0,
4986                                                    its->fwnode_handle, &its_domain_ops,
4987                                                    info);
4988         if (!inner_domain) {
4989                 kfree(info);
4990                 return -ENOMEM;
4991         }
4992
4993         irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
4994
4995         return 0;
4996 }
4997
4998 static int its_init_vpe_domain(void)
4999 {
5000         struct its_node *its;
5001         u32 devid;
5002         int entries;
5003
5004         if (gic_rdists->has_direct_lpi) {
5005                 pr_info("ITS: Using DirectLPI for VPE invalidation\n");
5006                 return 0;
5007         }
5008
5009         /* Any ITS will do, even if not v4 */
5010         its = list_first_entry(&its_nodes, struct its_node, entry);
5011
5012         entries = roundup_pow_of_two(nr_cpu_ids);
5013         vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
5014                                  GFP_KERNEL);
5015         if (!vpe_proxy.vpes)
5016                 return -ENOMEM;
5017
5018         /* Use the last possible DevID */
5019         devid = GENMASK(device_ids(its) - 1, 0);
5020         vpe_proxy.dev = its_create_device(its, devid, entries, false);
5021         if (!vpe_proxy.dev) {
5022                 kfree(vpe_proxy.vpes);
5023                 pr_err("ITS: Can't allocate GICv4 proxy device\n");
5024                 return -ENOMEM;
5025         }
5026
5027         BUG_ON(entries > vpe_proxy.dev->nr_ites);
5028
5029         raw_spin_lock_init(&vpe_proxy.lock);
5030         vpe_proxy.next_victim = 0;
5031         pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
5032                 devid, vpe_proxy.dev->nr_ites);
5033
5034         return 0;
5035 }
5036
5037 static int __init its_compute_its_list_map(struct its_node *its)
5038 {
5039         int its_number;
5040         u32 ctlr;
5041
5042         /*
5043          * This is assumed to be done early enough that we're
5044          * guaranteed to be single-threaded, hence no
5045          * locking. Should this change, we should address
5046          * this.
5047          */
5048         its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
5049         if (its_number >= GICv4_ITS_LIST_MAX) {
5050                 pr_err("ITS@%pa: No ITSList entry available!\n",
5051                        &its->phys_base);
5052                 return -EINVAL;
5053         }
5054
5055         ctlr = readl_relaxed(its->base + GITS_CTLR);
5056         ctlr &= ~GITS_CTLR_ITS_NUMBER;
5057         ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
5058         writel_relaxed(ctlr, its->base + GITS_CTLR);
5059         ctlr = readl_relaxed(its->base + GITS_CTLR);
5060         if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
5061                 its_number = ctlr & GITS_CTLR_ITS_NUMBER;
5062                 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
5063         }
5064
5065         if (test_and_set_bit(its_number, &its_list_map)) {
5066                 pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
5067                        &its->phys_base, its_number);
5068                 return -EINVAL;
5069         }
5070
5071         return its_number;
5072 }
5073
5074 static int __init its_probe_one(struct its_node *its)
5075 {
5076         u64 baser, tmp;
5077         struct page *page;
5078         u32 ctlr;
5079         int err;
5080
5081         if (is_v4(its)) {
5082                 if (!(its->typer & GITS_TYPER_VMOVP)) {
5083                         err = its_compute_its_list_map(its);
5084                         if (err < 0)
5085                                 goto out;
5086
5087                         its->list_nr = err;
5088
5089                         pr_info("ITS@%pa: Using ITS number %d\n",
5090                                 &its->phys_base, err);
5091                 } else {
5092                         pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base);
5093                 }
5094
5095                 if (is_v4_1(its)) {
5096                         u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
5097
5098                         its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K);
5099                         if (!its->sgir_base) {
5100                                 err = -ENOMEM;
5101                                 goto out;
5102                         }
5103
5104                         its->mpidr = readl_relaxed(its->base + GITS_MPIDR);
5105
5106                         pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
5107                                 &its->phys_base, its->mpidr, svpet);
5108                 }
5109         }
5110
5111         page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
5112                                 get_order(ITS_CMD_QUEUE_SZ));
5113         if (!page) {
5114                 err = -ENOMEM;
5115                 goto out_unmap_sgir;
5116         }
5117         its->cmd_base = (void *)page_address(page);
5118         its->cmd_write = its->cmd_base;
5119
5120         err = its_alloc_tables(its);
5121         if (err)
5122                 goto out_free_cmd;
5123
5124         err = its_alloc_collections(its);
5125         if (err)
5126                 goto out_free_tables;
5127
5128         baser = (virt_to_phys(its->cmd_base)    |
5129                  GITS_CBASER_RaWaWb             |
5130                  GITS_CBASER_InnerShareable     |
5131                  (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
5132                  GITS_CBASER_VALID);
5133
5134         gits_write_cbaser(baser, its->base + GITS_CBASER);
5135         tmp = gits_read_cbaser(its->base + GITS_CBASER);
5136
5137         if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)
5138                 tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
5139
5140         if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
5141                 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
5142                         /*
5143                          * The HW reports non-shareable, we must
5144                          * remove the cacheability attributes as
5145                          * well.
5146                          */
5147                         baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
5148                                    GITS_CBASER_CACHEABILITY_MASK);
5149                         baser |= GITS_CBASER_nC;
5150                         gits_write_cbaser(baser, its->base + GITS_CBASER);
5151                 }
5152                 pr_info("ITS: using cache flushing for cmd queue\n");
5153                 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
5154         }
5155
5156         gits_write_cwriter(0, its->base + GITS_CWRITER);
5157         ctlr = readl_relaxed(its->base + GITS_CTLR);
5158         ctlr |= GITS_CTLR_ENABLE;
5159         if (is_v4(its))
5160                 ctlr |= GITS_CTLR_ImDe;
5161         writel_relaxed(ctlr, its->base + GITS_CTLR);
5162
5163         err = its_init_domain(its);
5164         if (err)
5165                 goto out_free_tables;
5166
5167         raw_spin_lock(&its_lock);
5168         list_add(&its->entry, &its_nodes);
5169         raw_spin_unlock(&its_lock);
5170
5171         return 0;
5172
5173 out_free_tables:
5174         its_free_tables(its);
5175 out_free_cmd:
5176         free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
5177 out_unmap_sgir:
5178         if (its->sgir_base)
5179                 iounmap(its->sgir_base);
5180 out:
5181         pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err);
5182         return err;
5183 }
5184
5185 static bool gic_rdists_supports_plpis(void)
5186 {
5187         return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
5188 }
5189
5190 static int redist_disable_lpis(void)
5191 {
5192         void __iomem *rbase = gic_data_rdist_rd_base();
5193         u64 timeout = USEC_PER_SEC;
5194         u64 val;
5195
5196         if (!gic_rdists_supports_plpis()) {
5197                 pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
5198                 return -ENXIO;
5199         }
5200
5201         val = readl_relaxed(rbase + GICR_CTLR);
5202         if (!(val & GICR_CTLR_ENABLE_LPIS))
5203                 return 0;
5204
5205         /*
5206          * If coming via a CPU hotplug event, we don't need to disable
5207          * LPIs before trying to re-enable them. They are already
5208          * configured and all is well in the world.
5209          *
5210          * If running with preallocated tables, there is nothing to do.
5211          */
5212         if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) ||
5213             (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
5214                 return 0;
5215
5216         /*
5217          * From that point on, we only try to do some damage control.
5218          */
5219         pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
5220                 smp_processor_id());
5221         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
5222
5223         /* Disable LPIs */
5224         val &= ~GICR_CTLR_ENABLE_LPIS;
5225         writel_relaxed(val, rbase + GICR_CTLR);
5226
5227         /* Make sure any change to GICR_CTLR is observable by the GIC */
5228         dsb(sy);
5229
5230         /*
5231          * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
5232          * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
5233          * Error out if we time out waiting for RWP to clear.
5234          */
5235         while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
5236                 if (!timeout) {
5237                         pr_err("CPU%d: Timeout while disabling LPIs\n",
5238                                smp_processor_id());
5239                         return -ETIMEDOUT;
5240                 }
5241                 udelay(1);
5242                 timeout--;
5243         }
5244
5245         /*
5246          * After it has been written to 1, it is IMPLEMENTATION
5247          * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
5248          * cleared to 0. Error out if clearing the bit failed.
5249          */
5250         if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
5251                 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
5252                 return -EBUSY;
5253         }
5254
5255         return 0;
5256 }
5257
5258 int its_cpu_init(void)
5259 {
5260         if (!list_empty(&its_nodes)) {
5261                 int ret;
5262
5263                 ret = redist_disable_lpis();
5264                 if (ret)
5265                         return ret;
5266
5267                 its_cpu_init_lpis();
5268                 its_cpu_init_collections();
5269         }
5270
5271         return 0;
5272 }
5273
5274 static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work)
5275 {
5276         cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state);
5277         gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5278 }
5279
5280 static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work,
5281                     rdist_memreserve_cpuhp_cleanup_workfn);
5282
5283 static int its_cpu_memreserve_lpi(unsigned int cpu)
5284 {
5285         struct page *pend_page;
5286         int ret = 0;
5287
5288         /* This gets to run exactly once per CPU */
5289         if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE)
5290                 return 0;
5291
5292         pend_page = gic_data_rdist()->pend_page;
5293         if (WARN_ON(!pend_page)) {
5294                 ret = -ENOMEM;
5295                 goto out;
5296         }
5297         /*
5298          * If the pending table was pre-programmed, free the memory we
5299          * preemptively allocated. Otherwise, reserve that memory for
5300          * later kexecs.
5301          */
5302         if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) {
5303                 its_free_pending_table(pend_page);
5304                 gic_data_rdist()->pend_page = NULL;
5305         } else {
5306                 phys_addr_t paddr = page_to_phys(pend_page);
5307                 WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
5308         }
5309
5310 out:
5311         /* Last CPU being brought up gets to issue the cleanup */
5312         if (!IS_ENABLED(CONFIG_SMP) ||
5313             cpumask_equal(&cpus_booted_once_mask, cpu_possible_mask))
5314                 schedule_work(&rdist_memreserve_cpuhp_cleanup_work);
5315
5316         gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE;
5317         return ret;
5318 }
5319
5320 /* Mark all the BASER registers as invalid before they get reprogrammed */
5321 static int __init its_reset_one(struct resource *res)
5322 {
5323         void __iomem *its_base;
5324         int err, i;
5325
5326         its_base = its_map_one(res, &err);
5327         if (!its_base)
5328                 return err;
5329
5330         for (i = 0; i < GITS_BASER_NR_REGS; i++)
5331                 gits_write_baser(0, its_base + GITS_BASER + (i << 3));
5332
5333         iounmap(its_base);
5334         return 0;
5335 }
5336
5337 static const struct of_device_id its_device_id[] = {
5338         {       .compatible     = "arm,gic-v3-its",     },
5339         {},
5340 };
5341
5342 static struct its_node __init *its_node_init(struct resource *res,
5343                                              struct fwnode_handle *handle, int numa_node)
5344 {
5345         void __iomem *its_base;
5346         struct its_node *its;
5347         int err;
5348
5349         its_base = its_map_one(res, &err);
5350         if (!its_base)
5351                 return NULL;
5352
5353         pr_info("ITS %pR\n", res);
5354
5355         its = kzalloc(sizeof(*its), GFP_KERNEL);
5356         if (!its)
5357                 goto out_unmap;
5358
5359         raw_spin_lock_init(&its->lock);
5360         mutex_init(&its->dev_alloc_lock);
5361         INIT_LIST_HEAD(&its->entry);
5362         INIT_LIST_HEAD(&its->its_device_list);
5363
5364         its->typer = gic_read_typer(its_base + GITS_TYPER);
5365         its->base = its_base;
5366         its->phys_base = res->start;
5367         its->get_msi_base = its_irq_get_msi_base;
5368         its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI;
5369
5370         its->numa_node = numa_node;
5371         its->fwnode_handle = handle;
5372
5373         return its;
5374
5375 out_unmap:
5376         iounmap(its_base);
5377         return NULL;
5378 }
5379
5380 static void its_node_destroy(struct its_node *its)
5381 {
5382         iounmap(its->base);
5383         kfree(its);
5384 }
5385
5386 static int __init its_of_probe(struct device_node *node)
5387 {
5388         struct device_node *np;
5389         struct resource res;
5390         int err;
5391
5392         /*
5393          * Make sure *all* the ITS are reset before we probe any, as
5394          * they may be sharing memory. If any of the ITS fails to
5395          * reset, don't even try to go any further, as this could
5396          * result in something even worse.
5397          */
5398         for (np = of_find_matching_node(node, its_device_id); np;
5399              np = of_find_matching_node(np, its_device_id)) {
5400                 if (!of_device_is_available(np) ||
5401                     !of_property_read_bool(np, "msi-controller") ||
5402                     of_address_to_resource(np, 0, &res))
5403                         continue;
5404
5405                 err = its_reset_one(&res);
5406                 if (err)
5407                         return err;
5408         }
5409
5410         for (np = of_find_matching_node(node, its_device_id); np;
5411              np = of_find_matching_node(np, its_device_id)) {
5412                 struct its_node *its;
5413
5414                 if (!of_device_is_available(np))
5415                         continue;
5416                 if (!of_property_read_bool(np, "msi-controller")) {
5417                         pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5418                                 np);
5419                         continue;
5420                 }
5421
5422                 if (of_address_to_resource(np, 0, &res)) {
5423                         pr_warn("%pOF: no regs?\n", np);
5424                         continue;
5425                 }
5426
5427
5428                 its = its_node_init(&res, &np->fwnode, of_node_to_nid(np));
5429                 if (!its)
5430                         return -ENOMEM;
5431
5432                 its_enable_quirks(its);
5433                 err = its_probe_one(its);
5434                 if (err)  {
5435                         its_node_destroy(its);
5436                         return err;
5437                 }
5438         }
5439         return 0;
5440 }
5441
5442 #ifdef CONFIG_ACPI
5443
5444 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
5445
5446 #ifdef CONFIG_ACPI_NUMA
5447 struct its_srat_map {
5448         /* numa node id */
5449         u32     numa_node;
5450         /* GIC ITS ID */
5451         u32     its_id;
5452 };
5453
5454 static struct its_srat_map *its_srat_maps __initdata;
5455 static int its_in_srat __initdata;
5456
5457 static int __init acpi_get_its_numa_node(u32 its_id)
5458 {
5459         int i;
5460
5461         for (i = 0; i < its_in_srat; i++) {
5462                 if (its_id == its_srat_maps[i].its_id)
5463                         return its_srat_maps[i].numa_node;
5464         }
5465         return NUMA_NO_NODE;
5466 }
5467
5468 static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
5469                                           const unsigned long end)
5470 {
5471         return 0;
5472 }
5473
5474 static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
5475                          const unsigned long end)
5476 {
5477         int node;
5478         struct acpi_srat_gic_its_affinity *its_affinity;
5479
5480         its_affinity = (struct acpi_srat_gic_its_affinity *)header;
5481         if (!its_affinity)
5482                 return -EINVAL;
5483
5484         if (its_affinity->header.length < sizeof(*its_affinity)) {
5485                 pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5486                         its_affinity->header.length);
5487                 return -EINVAL;
5488         }
5489
5490         /*
5491          * Note that in theory a new proximity node could be created by this
5492          * entry as it is an SRAT resource allocation structure.
5493          * We do not currently support doing so.
5494          */
5495         node = pxm_to_node(its_affinity->proximity_domain);
5496
5497         if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
5498                 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
5499                 return 0;
5500         }
5501
5502         its_srat_maps[its_in_srat].numa_node = node;
5503         its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
5504         its_in_srat++;
5505         pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5506                 its_affinity->proximity_domain, its_affinity->its_id, node);
5507
5508         return 0;
5509 }
5510
5511 static void __init acpi_table_parse_srat_its(void)
5512 {
5513         int count;
5514
5515         count = acpi_table_parse_entries(ACPI_SIG_SRAT,
5516                         sizeof(struct acpi_table_srat),
5517                         ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5518                         gic_acpi_match_srat_its, 0);
5519         if (count <= 0)
5520                 return;
5521
5522         its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
5523                                       GFP_KERNEL);
5524         if (!its_srat_maps)
5525                 return;
5526
5527         acpi_table_parse_entries(ACPI_SIG_SRAT,
5528                         sizeof(struct acpi_table_srat),
5529                         ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5530                         gic_acpi_parse_srat_its, 0);
5531 }
5532
5533 /* free the its_srat_maps after ITS probing */
5534 static void __init acpi_its_srat_maps_free(void)
5535 {
5536         kfree(its_srat_maps);
5537 }
5538 #else
5539 static void __init acpi_table_parse_srat_its(void)      { }
5540 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
5541 static void __init acpi_its_srat_maps_free(void) { }
5542 #endif
5543
5544 static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
5545                                           const unsigned long end)
5546 {
5547         struct acpi_madt_generic_translator *its_entry;
5548         struct fwnode_handle *dom_handle;
5549         struct its_node *its;
5550         struct resource res;
5551         int err;
5552
5553         its_entry = (struct acpi_madt_generic_translator *)header;
5554         memset(&res, 0, sizeof(res));
5555         res.start = its_entry->base_address;
5556         res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
5557         res.flags = IORESOURCE_MEM;
5558
5559         dom_handle = irq_domain_alloc_fwnode(&res.start);
5560         if (!dom_handle) {
5561                 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
5562                        &res.start);
5563                 return -ENOMEM;
5564         }
5565
5566         err = iort_register_domain_token(its_entry->translation_id, res.start,
5567                                          dom_handle);
5568         if (err) {
5569                 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
5570                        &res.start, its_entry->translation_id);
5571                 goto dom_err;
5572         }
5573
5574         its = its_node_init(&res, dom_handle,
5575                             acpi_get_its_numa_node(its_entry->translation_id));
5576         if (!its) {
5577                 err = -ENOMEM;
5578                 goto node_err;
5579         }
5580
5581         err = its_probe_one(its);
5582         if (!err)
5583                 return 0;
5584
5585 node_err:
5586         iort_deregister_domain_token(its_entry->translation_id);
5587 dom_err:
5588         irq_domain_free_fwnode(dom_handle);
5589         return err;
5590 }
5591
5592 static int __init its_acpi_reset(union acpi_subtable_headers *header,
5593                                  const unsigned long end)
5594 {
5595         struct acpi_madt_generic_translator *its_entry;
5596         struct resource res;
5597
5598         its_entry = (struct acpi_madt_generic_translator *)header;
5599         res = (struct resource) {
5600                 .start  = its_entry->base_address,
5601                 .end    = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1,
5602                 .flags  = IORESOURCE_MEM,
5603         };
5604
5605         return its_reset_one(&res);
5606 }
5607
5608 static void __init its_acpi_probe(void)
5609 {
5610         acpi_table_parse_srat_its();
5611         /*
5612          * Make sure *all* the ITS are reset before we probe any, as
5613          * they may be sharing memory. If any of the ITS fails to
5614          * reset, don't even try to go any further, as this could
5615          * result in something even worse.
5616          */
5617         if (acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5618                                   its_acpi_reset, 0) > 0)
5619                 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5620                                       gic_acpi_parse_madt_its, 0);
5621         acpi_its_srat_maps_free();
5622 }
5623 #else
5624 static void __init its_acpi_probe(void) { }
5625 #endif
5626
5627 int __init its_lpi_memreserve_init(void)
5628 {
5629         int state;
5630
5631         if (!efi_enabled(EFI_CONFIG_TABLES))
5632                 return 0;
5633
5634         if (list_empty(&its_nodes))
5635                 return 0;
5636
5637         gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5638         state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
5639                                   "irqchip/arm/gicv3/memreserve:online",
5640                                   its_cpu_memreserve_lpi,
5641                                   NULL);
5642         if (state < 0)
5643                 return state;
5644
5645         gic_rdists->cpuhp_memreserve_state = state;
5646
5647         return 0;
5648 }
5649
5650 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
5651                     struct irq_domain *parent_domain)
5652 {
5653         struct device_node *of_node;
5654         struct its_node *its;
5655         bool has_v4 = false;
5656         bool has_v4_1 = false;
5657         int err;
5658
5659         gic_rdists = rdists;
5660
5661         its_parent = parent_domain;
5662         of_node = to_of_node(handle);
5663         if (of_node)
5664                 its_of_probe(of_node);
5665         else
5666                 its_acpi_probe();
5667
5668         if (list_empty(&its_nodes)) {
5669                 pr_warn("ITS: No ITS available, not enabling LPIs\n");
5670                 return -ENXIO;
5671         }
5672
5673         err = allocate_lpi_tables();
5674         if (err)
5675                 return err;
5676
5677         list_for_each_entry(its, &its_nodes, entry) {
5678                 has_v4 |= is_v4(its);
5679                 has_v4_1 |= is_v4_1(its);
5680         }
5681
5682         /* Don't bother with inconsistent systems */
5683         if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
5684                 rdists->has_rvpeid = false;
5685
5686         if (has_v4 & rdists->has_vlpis) {
5687                 const struct irq_domain_ops *sgi_ops;
5688
5689                 if (has_v4_1)
5690                         sgi_ops = &its_sgi_domain_ops;
5691                 else
5692                         sgi_ops = NULL;
5693
5694                 if (its_init_vpe_domain() ||
5695                     its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
5696                         rdists->has_vlpis = false;
5697                         pr_err("ITS: Disabling GICv4 support\n");
5698                 }
5699         }
5700
5701         register_syscore_ops(&its_syscore_ops);
5702
5703         return 0;
5704 }