drivers/irqchip/irq-gic-v3-its.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/acpi.h>
   8 #include <linux/acpi_iort.h>
   9 #include <linux/bitfield.h>
  10 #include <linux/bitmap.h>
  11 #include <linux/cpu.h>
  12 #include <linux/crash_dump.h>
  13 #include <linux/delay.h>
  14 #include <linux/dma-iommu.h>
  15 #include <linux/efi.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/iopoll.h>
  18 #include <linux/irqdomain.h>
  19 #include <linux/list.h>
  20 #include <linux/log2.h>
  21 #include <linux/memblock.h>
  22 #include <linux/mm.h>
  23 #include <linux/msi.h>
  24 #include <linux/of.h>
  25 #include <linux/of_address.h>
  26 #include <linux/of_irq.h>
  27 #include <linux/of_pci.h>
  28 #include <linux/of_platform.h>
  29 #include <linux/percpu.h>
  30 #include <linux/slab.h>
  31 #include <linux/syscore_ops.h>
  32
  33 #include <linux/irqchip.h>
  34 #include <linux/irqchip/arm-gic-v3.h>
  35 #include <linux/irqchip/arm-gic-v4.h>
  36
  37 #include <asm/cputype.h>
  38 #include <asm/exception.h>
  39
  40 #include "irq-gic-common.h"
  41
  42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING           (1ULL << 0)
  43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375       (1ULL << 1)
  44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144       (1ULL << 2)
  45
  46 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING     (1 << 0)
  47 #define RDIST_FLAGS_RD_TABLES_PREALLOCATED      (1 << 1)
  48
  49 static u32 lpi_id_bits;
  50
  51 /*
  52  * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
  53  * deal with (one configuration byte per interrupt). PENDBASE has to
  54  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
  55  */
  56 #define LPI_NRBITS              lpi_id_bits
  57 #define LPI_PROPBASE_SZ         ALIGN(BIT(LPI_NRBITS), SZ_64K)
  58 #define LPI_PENDBASE_SZ         ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
  59
  60 #define LPI_PROP_DEFAULT_PRIO   GICD_INT_DEF_PRI
  61
  62 /*
  63  * Collection structure - just an ID, and a redistributor address to
  64  * ping. We use one per CPU as a bag of interrupts assigned to this
  65  * CPU.
  66  */
  67 struct its_collection {
  68         u64                     target_address;
  69         u16                     col_id;
  70 };
  71
  72 /*
  73  * The ITS_BASER structure - contains memory information, cached
  74  * value of BASER register configuration and ITS page size.
  75  */
  76 struct its_baser {
  77         void            *base;
  78         u64             val;
  79         u32             order;
  80         u32             psz;
  81 };
  82
  83 struct its_device;
  84
  85 /*
  86  * The ITS structure - contains most of the infrastructure, with the
  87  * top-level MSI domain, the command queue, the collections, and the
  88  * list of devices writing to it.
  89  *
  90  * dev_alloc_lock has to be taken for device allocations, while the
  91  * spinlock must be taken to parse data structures such as the device
  92  * list.
  93  */
  94 struct its_node {
  95         raw_spinlock_t          lock;
  96         struct mutex            dev_alloc_lock;
  97         struct list_head        entry;
  98         void __iomem            *base;
  99         void __iomem            *sgir_base;
 100         phys_addr_t             phys_base;
 101         struct its_cmd_block    *cmd_base;
 102         struct its_cmd_block    *cmd_write;
 103         struct its_baser        tables[GITS_BASER_NR_REGS];
 104         struct its_collection   *collections;
 105         struct fwnode_handle    *fwnode_handle;
 106         u64                     (*get_msi_base)(struct its_device *its_dev);
 107         u64                     typer;
 108         u64                     cbaser_save;
 109         u32                     ctlr_save;
 110         u32                     mpidr;
 111         struct list_head        its_device_list;
 112         u64                     flags;
 113         unsigned long           list_nr;
 114         int                     numa_node;
 115         unsigned int            msi_domain_flags;
 116         u32                     pre_its_base; /* for Socionext Synquacer */
 117         int                     vlpi_redist_offset;
 118 };
 119
 120 #define is_v4(its)              (!!((its)->typer & GITS_TYPER_VLPIS))
 121 #define is_v4_1(its)            (!!((its)->typer & GITS_TYPER_VMAPP))
 122 #define device_ids(its)         (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
 123
 124 #define ITS_ITT_ALIGN           SZ_256
 125
 126 /* The maximum number of VPEID bits supported by VLPI commands */
 127 #define ITS_MAX_VPEID_BITS                                              \
 128         ({                                                              \
 129                 int nvpeid = 16;                                        \
 130                 if (gic_rdists->has_rvpeid &&                           \
 131                     gic_rdists->gicd_typer2 & GICD_TYPER2_VIL)          \
 132                         nvpeid = 1 + (gic_rdists->gicd_typer2 &         \
 133                                       GICD_TYPER2_VID);                 \
 134                                                                         \
 135                 nvpeid;                                                 \
 136         })
 137 #define ITS_MAX_VPEID           (1 << (ITS_MAX_VPEID_BITS))
 138
 139 /* Convert page order to size in bytes */
 140 #define PAGE_ORDER_TO_SIZE(o)   (PAGE_SIZE << (o))
 141
 142 struct event_lpi_map {
 143         unsigned long           *lpi_map;
 144         u16                     *col_map;
 145         irq_hw_number_t         lpi_base;
 146         int                     nr_lpis;
 147         raw_spinlock_t          vlpi_lock;
 148         struct its_vm           *vm;
 149         struct its_vlpi_map     *vlpi_maps;
 150         int                     nr_vlpis;
 151 };
 152
 153 /*
 154  * The ITS view of a device - belongs to an ITS, owns an interrupt
 155  * translation table, and a list of interrupts.  If it some of its
 156  * LPIs are injected into a guest (GICv4), the event_map.vm field
 157  * indicates which one.
 158  */
 159 struct its_device {
 160         struct list_head        entry;
 161         struct its_node         *its;
 162         struct event_lpi_map    event_map;
 163         void                    *itt;
 164         u32                     nr_ites;
 165         u32                     device_id;
 166         bool                    shared;
 167 };
 168
 169 static struct {
 170         raw_spinlock_t          lock;
 171         struct its_device       *dev;
 172         struct its_vpe          **vpes;
 173         int                     next_victim;
 174 } vpe_proxy;
 175
 176 struct cpu_lpi_count {
 177         atomic_t        managed;
 178         atomic_t        unmanaged;
 179 };
 180
 181 static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
 182
 183 static LIST_HEAD(its_nodes);
 184 static DEFINE_RAW_SPINLOCK(its_lock);
 185 static struct rdists *gic_rdists;
 186 static struct irq_domain *its_parent;
 187
 188 static unsigned long its_list_map;
 189 static u16 vmovp_seq_num;
 190 static DEFINE_RAW_SPINLOCK(vmovp_lock);
 191
 192 static DEFINE_IDA(its_vpeid_ida);
 193
 194 #define gic_data_rdist()                (raw_cpu_ptr(gic_rdists->rdist))
 195 #define gic_data_rdist_cpu(cpu)         (per_cpu_ptr(gic_rdists->rdist, cpu))
 196 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
 197 #define gic_data_rdist_vlpi_base()      (gic_data_rdist_rd_base() + SZ_128K)
 198
 199 /*
 200  * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
 201  * always have vSGIs mapped.
 202  */
 203 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
 204 {
 205         return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
 206 }
 207
 208 static u16 get_its_list(struct its_vm *vm)
 209 {
 210         struct its_node *its;
 211         unsigned long its_list = 0;
 212
 213         list_for_each_entry(its, &its_nodes, entry) {
 214                 if (!is_v4(its))
 215                         continue;
 216
 217                 if (require_its_list_vmovp(vm, its))
 218                         __set_bit(its->list_nr, &its_list);
 219         }
 220
 221         return (u16)its_list;
 222 }
 223
 224 static inline u32 its_get_event_id(struct irq_data *d)
 225 {
 226         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 227         return d->hwirq - its_dev->event_map.lpi_base;
 228 }
 229
 230 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
 231                                                u32 event)
 232 {
 233         struct its_node *its = its_dev->its;
 234
 235         return its->collections + its_dev->event_map.col_map[event];
 236 }
 237
 238 static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
 239                                                u32 event)
 240 {
 241         if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
 242                 return NULL;
 243
 244         return &its_dev->event_map.vlpi_maps[event];
 245 }
 246
 247 static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
 248 {
 249         if (irqd_is_forwarded_to_vcpu(d)) {
 250                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 251                 u32 event = its_get_event_id(d);
 252
 253                 return dev_event_to_vlpi_map(its_dev, event);
 254         }
 255
 256         return NULL;
 257 }
 258
 259 static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
 260 {
 261         raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
 262         return vpe->col_idx;
 263 }
 264
 265 static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
 266 {
 267         raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
 268 }
 269
 270 static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
 271 {
 272         struct its_vlpi_map *map = get_vlpi_map(d);
 273         int cpu;
 274
 275         if (map) {
 276                 cpu = vpe_to_cpuid_lock(map->vpe, flags);
 277         } else {
 278                 /* Physical LPIs are already locked via the irq_desc lock */
 279                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 280                 cpu = its_dev->event_map.col_map[its_get_event_id(d)];
 281                 /* Keep GCC quiet... */
 282                 *flags = 0;
 283         }
 284
 285         return cpu;
 286 }
 287
 288 static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
 289 {
 290         struct its_vlpi_map *map = get_vlpi_map(d);
 291
 292         if (map)
 293                 vpe_to_cpuid_unlock(map->vpe, flags);
 294 }
 295
 296 static struct its_collection *valid_col(struct its_collection *col)
 297 {
 298         if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
 299                 return NULL;
 300
 301         return col;
 302 }
 303
 304 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
 305 {
 306         if (valid_col(its->collections + vpe->col_idx))
 307                 return vpe;
 308
 309         return NULL;
 310 }
 311
 312 /*
 313  * ITS command descriptors - parameters to be encoded in a command
 314  * block.
 315  */
 316 struct its_cmd_desc {
 317         union {
 318                 struct {
 319                         struct its_device *dev;
 320                         u32 event_id;
 321                 } its_inv_cmd;
 322
 323                 struct {
 324                         struct its_device *dev;
 325                         u32 event_id;
 326                 } its_clear_cmd;
 327
 328                 struct {
 329                         struct its_device *dev;
 330                         u32 event_id;
 331                 } its_int_cmd;
 332
 333                 struct {
 334                         struct its_device *dev;
 335                         int valid;
 336                 } its_mapd_cmd;
 337
 338                 struct {
 339                         struct its_collection *col;
 340                         int valid;
 341                 } its_mapc_cmd;
 342
 343                 struct {
 344                         struct its_device *dev;
 345                         u32 phys_id;
 346                         u32 event_id;
 347                 } its_mapti_cmd;
 348
 349                 struct {
 350                         struct its_device *dev;
 351                         struct its_collection *col;
 352                         u32 event_id;
 353                 } its_movi_cmd;
 354
 355                 struct {
 356                         struct its_device *dev;
 357                         u32 event_id;
 358                 } its_discard_cmd;
 359
 360                 struct {
 361                         struct its_collection *col;
 362                 } its_invall_cmd;
 363
 364                 struct {
 365                         struct its_vpe *vpe;
 366                 } its_vinvall_cmd;
 367
 368                 struct {
 369                         struct its_vpe *vpe;
 370                         struct its_collection *col;
 371                         bool valid;
 372                 } its_vmapp_cmd;
 373
 374                 struct {
 375                         struct its_vpe *vpe;
 376                         struct its_device *dev;
 377                         u32 virt_id;
 378                         u32 event_id;
 379                         bool db_enabled;
 380                 } its_vmapti_cmd;
 381
 382                 struct {
 383                         struct its_vpe *vpe;
 384                         struct its_device *dev;
 385                         u32 event_id;
 386                         bool db_enabled;
 387                 } its_vmovi_cmd;
 388
 389                 struct {
 390                         struct its_vpe *vpe;
 391                         struct its_collection *col;
 392                         u16 seq_num;
 393                         u16 its_list;
 394                 } its_vmovp_cmd;
 395
 396                 struct {
 397                         struct its_vpe *vpe;
 398                 } its_invdb_cmd;
 399
 400                 struct {
 401                         struct its_vpe *vpe;
 402                         u8 sgi;
 403                         u8 priority;
 404                         bool enable;
 405                         bool group;
 406                         bool clear;
 407                 } its_vsgi_cmd;
 408         };
 409 };
 410
 411 /*
 412  * The ITS command block, which is what the ITS actually parses.
 413  */
 414 struct its_cmd_block {
 415         union {
 416                 u64     raw_cmd[4];
 417                 __le64  raw_cmd_le[4];
 418         };
 419 };
 420
 421 #define ITS_CMD_QUEUE_SZ                SZ_64K
 422 #define ITS_CMD_QUEUE_NR_ENTRIES        (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
 423
 424 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
 425                                                     struct its_cmd_block *,
 426                                                     struct its_cmd_desc *);
 427
 428 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
 429                                               struct its_cmd_block *,
 430                                               struct its_cmd_desc *);
 431
 432 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
 433 {
 434         u64 mask = GENMASK_ULL(h, l);
 435         *raw_cmd &= ~mask;
 436         *raw_cmd |= (val << l) & mask;
 437 }
 438
 439 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
 440 {
 441         its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
 442 }
 443
 444 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
 445 {
 446         its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
 447 }
 448
 449 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
 450 {
 451         its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
 452 }
 453
 454 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
 455 {
 456         its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
 457 }
 458
 459 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
 460 {
 461         its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
 462 }
 463
 464 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
 465 {
 466         its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
 467 }
 468
 469 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
 470 {
 471         its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
 472 }
 473
 474 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
 475 {
 476         its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
 477 }
 478
 479 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
 480 {
 481         its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
 482 }
 483
 484 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
 485 {
 486         its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
 487 }
 488
 489 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
 490 {
 491         its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
 492 }
 493
 494 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
 495 {
 496         its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
 497 }
 498
 499 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
 500 {
 501         its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
 502 }
 503
 504 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
 505 {
 506         its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
 507 }
 508
 509 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
 510 {
 511         its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
 512 }
 513
 514 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
 515 {
 516         its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
 517 }
 518
 519 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
 520 {
 521         its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
 522 }
 523
 524 static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
 525 {
 526         its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
 527 }
 528
 529 static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
 530 {
 531         its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
 532 }
 533
 534 static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
 535 {
 536         its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
 537 }
 538
 539 static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
 540                                         u32 vpe_db_lpi)
 541 {
 542         its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
 543 }
 544
 545 static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
 546                                         u32 vpe_db_lpi)
 547 {
 548         its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
 549 }
 550
 551 static void its_encode_db(struct its_cmd_block *cmd, bool db)
 552 {
 553         its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
 554 }
 555
 556 static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
 557 {
 558         its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
 559 }
 560
 561 static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
 562 {
 563         its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
 564 }
 565
 566 static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
 567 {
 568         its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
 569 }
 570
 571 static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
 572 {
 573         its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
 574 }
 575
 576 static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
 577 {
 578         its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
 579 }
 580
 581 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
 582 {
 583         /* Let's fixup BE commands */
 584         cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
 585         cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
 586         cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
 587         cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
 588 }
 589
 590 static struct its_collection *its_build_mapd_cmd(struct its_node *its,
 591                                                  struct its_cmd_block *cmd,
 592                                                  struct its_cmd_desc *desc)
 593 {
 594         unsigned long itt_addr;
 595         u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
 596
 597         itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
 598         itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
 599
 600         its_encode_cmd(cmd, GITS_CMD_MAPD);
 601         its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
 602         its_encode_size(cmd, size - 1);
 603         its_encode_itt(cmd, itt_addr);
 604         its_encode_valid(cmd, desc->its_mapd_cmd.valid);
 605
 606         its_fixup_cmd(cmd);
 607
 608         return NULL;
 609 }
 610
 611 static struct its_collection *its_build_mapc_cmd(struct its_node *its,
 612                                                  struct its_cmd_block *cmd,
 613                                                  struct its_cmd_desc *desc)
 614 {
 615         its_encode_cmd(cmd, GITS_CMD_MAPC);
 616         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
 617         its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
 618         its_encode_valid(cmd, desc->its_mapc_cmd.valid);
 619
 620         its_fixup_cmd(cmd);
 621
 622         return desc->its_mapc_cmd.col;
 623 }
 624
 625 static struct its_collection *its_build_mapti_cmd(struct its_node *its,
 626                                                   struct its_cmd_block *cmd,
 627                                                   struct its_cmd_desc *desc)
 628 {
 629         struct its_collection *col;
 630
 631         col = dev_event_to_col(desc->its_mapti_cmd.dev,
 632                                desc->its_mapti_cmd.event_id);
 633
 634         its_encode_cmd(cmd, GITS_CMD_MAPTI);
 635         its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
 636         its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
 637         its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
 638         its_encode_collection(cmd, col->col_id);
 639
 640         its_fixup_cmd(cmd);
 641
 642         return valid_col(col);
 643 }
 644
 645 static struct its_collection *its_build_movi_cmd(struct its_node *its,
 646                                                  struct its_cmd_block *cmd,
 647                                                  struct its_cmd_desc *desc)
 648 {
 649         struct its_collection *col;
 650
 651         col = dev_event_to_col(desc->its_movi_cmd.dev,
 652                                desc->its_movi_cmd.event_id);
 653
 654         its_encode_cmd(cmd, GITS_CMD_MOVI);
 655         its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
 656         its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
 657         its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
 658
 659         its_fixup_cmd(cmd);
 660
 661         return valid_col(col);
 662 }
 663
 664 static struct its_collection *its_build_discard_cmd(struct its_node *its,
 665                                                     struct its_cmd_block *cmd,
 666                                                     struct its_cmd_desc *desc)
 667 {
 668         struct its_collection *col;
 669
 670         col = dev_event_to_col(desc->its_discard_cmd.dev,
 671                                desc->its_discard_cmd.event_id);
 672
 673         its_encode_cmd(cmd, GITS_CMD_DISCARD);
 674         its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
 675         its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
 676
 677         its_fixup_cmd(cmd);
 678
 679         return valid_col(col);
 680 }
 681
 682 static struct its_collection *its_build_inv_cmd(struct its_node *its,
 683                                                 struct its_cmd_block *cmd,
 684                                                 struct its_cmd_desc *desc)
 685 {
 686         struct its_collection *col;
 687
 688         col = dev_event_to_col(desc->its_inv_cmd.dev,
 689                                desc->its_inv_cmd.event_id);
 690
 691         its_encode_cmd(cmd, GITS_CMD_INV);
 692         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 693         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 694
 695         its_fixup_cmd(cmd);
 696
 697         return valid_col(col);
 698 }
 699
 700 static struct its_collection *its_build_int_cmd(struct its_node *its,
 701                                                 struct its_cmd_block *cmd,
 702                                                 struct its_cmd_desc *desc)
 703 {
 704         struct its_collection *col;
 705
 706         col = dev_event_to_col(desc->its_int_cmd.dev,
 707                                desc->its_int_cmd.event_id);
 708
 709         its_encode_cmd(cmd, GITS_CMD_INT);
 710         its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
 711         its_encode_event_id(cmd, desc->its_int_cmd.event_id);
 712
 713         its_fixup_cmd(cmd);
 714
 715         return valid_col(col);
 716 }
 717
 718 static struct its_collection *its_build_clear_cmd(struct its_node *its,
 719                                                   struct its_cmd_block *cmd,
 720                                                   struct its_cmd_desc *desc)
 721 {
 722         struct its_collection *col;
 723
 724         col = dev_event_to_col(desc->its_clear_cmd.dev,
 725                                desc->its_clear_cmd.event_id);
 726
 727         its_encode_cmd(cmd, GITS_CMD_CLEAR);
 728         its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
 729         its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
 730
 731         its_fixup_cmd(cmd);
 732
 733         return valid_col(col);
 734 }
 735
 736 static struct its_collection *its_build_invall_cmd(struct its_node *its,
 737                                                    struct its_cmd_block *cmd,
 738                                                    struct its_cmd_desc *desc)
 739 {
 740         its_encode_cmd(cmd, GITS_CMD_INVALL);
 741         its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
 742
 743         its_fixup_cmd(cmd);
 744
 745         return NULL;
 746 }
 747
 748 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
 749                                              struct its_cmd_block *cmd,
 750                                              struct its_cmd_desc *desc)
 751 {
 752         its_encode_cmd(cmd, GITS_CMD_VINVALL);
 753         its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
 754
 755         its_fixup_cmd(cmd);
 756
 757         return valid_vpe(its, desc->its_vinvall_cmd.vpe);
 758 }
 759
 760 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
 761                                            struct its_cmd_block *cmd,
 762                                            struct its_cmd_desc *desc)
 763 {
 764         unsigned long vpt_addr, vconf_addr;
 765         u64 target;
 766         bool alloc;
 767
 768         its_encode_cmd(cmd, GITS_CMD_VMAPP);
 769         its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
 770         its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
 771
 772         if (!desc->its_vmapp_cmd.valid) {
 773                 if (is_v4_1(its)) {
 774                         alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
 775                         its_encode_alloc(cmd, alloc);
 776                 }
 777
 778                 goto out;
 779         }
 780
 781         vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
 782         target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
 783
 784         its_encode_target(cmd, target);
 785         its_encode_vpt_addr(cmd, vpt_addr);
 786         its_encode_vpt_size(cmd, LPI_NRBITS - 1);
 787
 788         if (!is_v4_1(its))
 789                 goto out;
 790
 791         vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
 792
 793         alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
 794
 795         its_encode_alloc(cmd, alloc);
 796
 797         /* We can only signal PTZ when alloc==1. Why do we have two bits? */
 798         its_encode_ptz(cmd, alloc);
 799         its_encode_vconf_addr(cmd, vconf_addr);
 800         its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
 801
 802 out:
 803         its_fixup_cmd(cmd);
 804
 805         return valid_vpe(its, desc->its_vmapp_cmd.vpe);
 806 }
 807
 808 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
 809                                             struct its_cmd_block *cmd,
 810                                             struct its_cmd_desc *desc)
 811 {
 812         u32 db;
 813
 814         if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
 815                 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
 816         else
 817                 db = 1023;
 818
 819         its_encode_cmd(cmd, GITS_CMD_VMAPTI);
 820         its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
 821         its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
 822         its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
 823         its_encode_db_phys_id(cmd, db);
 824         its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
 825
 826         its_fixup_cmd(cmd);
 827
 828         return valid_vpe(its, desc->its_vmapti_cmd.vpe);
 829 }
 830
 831 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
 832                                            struct its_cmd_block *cmd,
 833                                            struct its_cmd_desc *desc)
 834 {
 835         u32 db;
 836
 837         if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
 838                 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
 839         else
 840                 db = 1023;
 841
 842         its_encode_cmd(cmd, GITS_CMD_VMOVI);
 843         its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
 844         its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
 845         its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
 846         its_encode_db_phys_id(cmd, db);
 847         its_encode_db_valid(cmd, true);
 848
 849         its_fixup_cmd(cmd);
 850
 851         return valid_vpe(its, desc->its_vmovi_cmd.vpe);
 852 }
 853
 854 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
 855                                            struct its_cmd_block *cmd,
 856                                            struct its_cmd_desc *desc)
 857 {
 858         u64 target;
 859
 860         target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
 861         its_encode_cmd(cmd, GITS_CMD_VMOVP);
 862         its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
 863         its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
 864         its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
 865         its_encode_target(cmd, target);
 866
 867         if (is_v4_1(its)) {
 868                 its_encode_db(cmd, true);
 869                 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
 870         }
 871
 872         its_fixup_cmd(cmd);
 873
 874         return valid_vpe(its, desc->its_vmovp_cmd.vpe);
 875 }
 876
 877 static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
 878                                           struct its_cmd_block *cmd,
 879                                           struct its_cmd_desc *desc)
 880 {
 881         struct its_vlpi_map *map;
 882
 883         map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
 884                                     desc->its_inv_cmd.event_id);
 885
 886         its_encode_cmd(cmd, GITS_CMD_INV);
 887         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 888         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 889
 890         its_fixup_cmd(cmd);
 891
 892         return valid_vpe(its, map->vpe);
 893 }
 894
 895 static struct its_vpe *its_build_vint_cmd(struct its_node *its,
 896                                           struct its_cmd_block *cmd,
 897                                           struct its_cmd_desc *desc)
 898 {
 899         struct its_vlpi_map *map;
 900
 901         map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
 902                                     desc->its_int_cmd.event_id);
 903
 904         its_encode_cmd(cmd, GITS_CMD_INT);
 905         its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
 906         its_encode_event_id(cmd, desc->its_int_cmd.event_id);
 907
 908         its_fixup_cmd(cmd);
 909
 910         return valid_vpe(its, map->vpe);
 911 }
 912
 913 static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
 914                                             struct its_cmd_block *cmd,
 915                                             struct its_cmd_desc *desc)
 916 {
 917         struct its_vlpi_map *map;
 918
 919         map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
 920                                     desc->its_clear_cmd.event_id);
 921
 922         its_encode_cmd(cmd, GITS_CMD_CLEAR);
 923         its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
 924         its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
 925
 926         its_fixup_cmd(cmd);
 927
 928         return valid_vpe(its, map->vpe);
 929 }
 930
 931 static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
 932                                            struct its_cmd_block *cmd,
 933                                            struct its_cmd_desc *desc)
 934 {
 935         if (WARN_ON(!is_v4_1(its)))
 936                 return NULL;
 937
 938         its_encode_cmd(cmd, GITS_CMD_INVDB);
 939         its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
 940
 941         its_fixup_cmd(cmd);
 942
 943         return valid_vpe(its, desc->its_invdb_cmd.vpe);
 944 }
 945
 946 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
 947                                           struct its_cmd_block *cmd,
 948                                           struct its_cmd_desc *desc)
 949 {
 950         if (WARN_ON(!is_v4_1(its)))
 951                 return NULL;
 952
 953         its_encode_cmd(cmd, GITS_CMD_VSGI);
 954         its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
 955         its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
 956         its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
 957         its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
 958         its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
 959         its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
 960
 961         its_fixup_cmd(cmd);
 962
 963         return valid_vpe(its, desc->its_vsgi_cmd.vpe);
 964 }
 965
 966 static u64 its_cmd_ptr_to_offset(struct its_node *its,
 967                                  struct its_cmd_block *ptr)
 968 {
 969         return (ptr - its->cmd_base) * sizeof(*ptr);
 970 }
 971
 972 static int its_queue_full(struct its_node *its)
 973 {
 974         int widx;
 975         int ridx;
 976
 977         widx = its->cmd_write - its->cmd_base;
 978         ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
 979
 980         /* This is incredibly unlikely to happen, unless the ITS locks up. */
 981         if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
 982                 return 1;
 983
 984         return 0;
 985 }
 986
 987 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
 988 {
 989         struct its_cmd_block *cmd;
 990         u32 count = 1000000;    /* 1s! */
 991
 992         while (its_queue_full(its)) {
 993                 count--;
 994                 if (!count) {
 995                         pr_err_ratelimited("ITS queue not draining\n");
 996                         return NULL;
 997                 }
 998                 cpu_relax();
 999                 udelay(1);
1000         }
1001
1002         cmd = its->cmd_write++;
1003
1004         /* Handle queue wrapping */
1005         if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
1006                 its->cmd_write = its->cmd_base;
1007
1008         /* Clear command  */
1009         cmd->raw_cmd[0] = 0;
1010         cmd->raw_cmd[1] = 0;
1011         cmd->raw_cmd[2] = 0;
1012         cmd->raw_cmd[3] = 0;
1013
1014         return cmd;
1015 }
1016
1017 static struct its_cmd_block *its_post_commands(struct its_node *its)
1018 {
1019         u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
1020
1021         writel_relaxed(wr, its->base + GITS_CWRITER);
1022
1023         return its->cmd_write;
1024 }
1025
1026 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
1027 {
1028         /*
1029          * Make sure the commands written to memory are observable by
1030          * the ITS.
1031          */
1032         if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
1033                 gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
1034         else
1035                 dsb(ishst);
1036 }
1037
1038 static int its_wait_for_range_completion(struct its_node *its,
1039                                          u64    prev_idx,
1040                                          struct its_cmd_block *to)
1041 {
1042         u64 rd_idx, to_idx, linear_idx;
1043         u32 count = 1000000;    /* 1s! */
1044
1045         /* Linearize to_idx if the command set has wrapped around */
1046         to_idx = its_cmd_ptr_to_offset(its, to);
1047         if (to_idx < prev_idx)
1048                 to_idx += ITS_CMD_QUEUE_SZ;
1049
1050         linear_idx = prev_idx;
1051
1052         while (1) {
1053                 s64 delta;
1054
1055                 rd_idx = readl_relaxed(its->base + GITS_CREADR);
1056
1057                 /*
1058                  * Compute the read pointer progress, taking the
1059                  * potential wrap-around into account.
1060                  */
1061                 delta = rd_idx - prev_idx;
1062                 if (rd_idx < prev_idx)
1063                         delta += ITS_CMD_QUEUE_SZ;
1064
1065                 linear_idx += delta;
1066                 if (linear_idx >= to_idx)
1067                         break;
1068
1069                 count--;
1070                 if (!count) {
1071                         pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1072                                            to_idx, linear_idx);
1073                         return -1;
1074                 }
1075                 prev_idx = rd_idx;
1076                 cpu_relax();
1077                 udelay(1);
1078         }
1079
1080         return 0;
1081 }
1082
1083 /* Warning, macro hell follows */
1084 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn)       \
1085 void name(struct its_node *its,                                         \
1086           buildtype builder,                                            \
1087           struct its_cmd_desc *desc)                                    \
1088 {                                                                       \
1089         struct its_cmd_block *cmd, *sync_cmd, *next_cmd;                \
1090         synctype *sync_obj;                                             \
1091         unsigned long flags;                                            \
1092         u64 rd_idx;                                                     \
1093                                                                         \
1094         raw_spin_lock_irqsave(&its->lock, flags);                       \
1095                                                                         \
1096         cmd = its_allocate_entry(its);                                  \
1097         if (!cmd) {             /* We're soooooo screewed... */         \
1098                 raw_spin_unlock_irqrestore(&its->lock, flags);          \
1099                 return;                                                 \
1100         }                                                               \
1101         sync_obj = builder(its, cmd, desc);                             \
1102         its_flush_cmd(its, cmd);                                        \
1103                                                                         \
1104         if (sync_obj) {                                                 \
1105                 sync_cmd = its_allocate_entry(its);                     \
1106                 if (!sync_cmd)                                          \
1107                         goto post;                                      \
1108                                                                         \
1109                 buildfn(its, sync_cmd, sync_obj);                       \
1110                 its_flush_cmd(its, sync_cmd);                           \
1111         }                                                               \
1112                                                                         \
1113 post:                                                                   \
1114         rd_idx = readl_relaxed(its->base + GITS_CREADR);                \
1115         next_cmd = its_post_commands(its);                              \
1116         raw_spin_unlock_irqrestore(&its->lock, flags);                  \
1117                                                                         \
1118         if (its_wait_for_range_completion(its, rd_idx, next_cmd))       \
1119                 pr_err_ratelimited("ITS cmd %ps failed\n", builder);    \
1120 }
1121
1122 static void its_build_sync_cmd(struct its_node *its,
1123                                struct its_cmd_block *sync_cmd,
1124                                struct its_collection *sync_col)
1125 {
1126         its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
1127         its_encode_target(sync_cmd, sync_col->target_address);
1128
1129         its_fixup_cmd(sync_cmd);
1130 }
1131
1132 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
1133                              struct its_collection, its_build_sync_cmd)
1134
1135 static void its_build_vsync_cmd(struct its_node *its,
1136                                 struct its_cmd_block *sync_cmd,
1137                                 struct its_vpe *sync_vpe)
1138 {
1139         its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
1140         its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
1141
1142         its_fixup_cmd(sync_cmd);
1143 }
1144
1145 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
1146                              struct its_vpe, its_build_vsync_cmd)
1147
1148 static void its_send_int(struct its_device *dev, u32 event_id)
1149 {
1150         struct its_cmd_desc desc;
1151
1152         desc.its_int_cmd.dev = dev;
1153         desc.its_int_cmd.event_id = event_id;
1154
1155         its_send_single_command(dev->its, its_build_int_cmd, &desc);
1156 }
1157
1158 static void its_send_clear(struct its_device *dev, u32 event_id)
1159 {
1160         struct its_cmd_desc desc;
1161
1162         desc.its_clear_cmd.dev = dev;
1163         desc.its_clear_cmd.event_id = event_id;
1164
1165         its_send_single_command(dev->its, its_build_clear_cmd, &desc);
1166 }
1167
1168 static void its_send_inv(struct its_device *dev, u32 event_id)
1169 {
1170         struct its_cmd_desc desc;
1171
1172         desc.its_inv_cmd.dev = dev;
1173         desc.its_inv_cmd.event_id = event_id;
1174
1175         its_send_single_command(dev->its, its_build_inv_cmd, &desc);
1176 }
1177
1178 static void its_send_mapd(struct its_device *dev, int valid)
1179 {
1180         struct its_cmd_desc desc;
1181
1182         desc.its_mapd_cmd.dev = dev;
1183         desc.its_mapd_cmd.valid = !!valid;
1184
1185         its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
1186 }
1187
1188 static void its_send_mapc(struct its_node *its, struct its_collection *col,
1189                           int valid)
1190 {
1191         struct its_cmd_desc desc;
1192
1193         desc.its_mapc_cmd.col = col;
1194         desc.its_mapc_cmd.valid = !!valid;
1195
1196         its_send_single_command(its, its_build_mapc_cmd, &desc);
1197 }
1198
1199 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
1200 {
1201         struct its_cmd_desc desc;
1202
1203         desc.its_mapti_cmd.dev = dev;
1204         desc.its_mapti_cmd.phys_id = irq_id;
1205         desc.its_mapti_cmd.event_id = id;
1206
1207         its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
1208 }
1209
1210 static void its_send_movi(struct its_device *dev,
1211                           struct its_collection *col, u32 id)
1212 {
1213         struct its_cmd_desc desc;
1214
1215         desc.its_movi_cmd.dev = dev;
1216         desc.its_movi_cmd.col = col;
1217         desc.its_movi_cmd.event_id = id;
1218
1219         its_send_single_command(dev->its, its_build_movi_cmd, &desc);
1220 }
1221
1222 static void its_send_discard(struct its_device *dev, u32 id)
1223 {
1224         struct its_cmd_desc desc;
1225
1226         desc.its_discard_cmd.dev = dev;
1227         desc.its_discard_cmd.event_id = id;
1228
1229         its_send_single_command(dev->its, its_build_discard_cmd, &desc);
1230 }
1231
1232 static void its_send_invall(struct its_node *its, struct its_collection *col)
1233 {
1234         struct its_cmd_desc desc;
1235
1236         desc.its_invall_cmd.col = col;
1237
1238         its_send_single_command(its, its_build_invall_cmd, &desc);
1239 }
1240
1241 static void its_send_vmapti(struct its_device *dev, u32 id)
1242 {
1243         struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1244         struct its_cmd_desc desc;
1245
1246         desc.its_vmapti_cmd.vpe = map->vpe;
1247         desc.its_vmapti_cmd.dev = dev;
1248         desc.its_vmapti_cmd.virt_id = map->vintid;
1249         desc.its_vmapti_cmd.event_id = id;
1250         desc.its_vmapti_cmd.db_enabled = map->db_enabled;
1251
1252         its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
1253 }
1254
1255 static void its_send_vmovi(struct its_device *dev, u32 id)
1256 {
1257         struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1258         struct its_cmd_desc desc;
1259
1260         desc.its_vmovi_cmd.vpe = map->vpe;
1261         desc.its_vmovi_cmd.dev = dev;
1262         desc.its_vmovi_cmd.event_id = id;
1263         desc.its_vmovi_cmd.db_enabled = map->db_enabled;
1264
1265         its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
1266 }
1267
1268 static void its_send_vmapp(struct its_node *its,
1269                            struct its_vpe *vpe, bool valid)
1270 {
1271         struct its_cmd_desc desc;
1272
1273         desc.its_vmapp_cmd.vpe = vpe;
1274         desc.its_vmapp_cmd.valid = valid;
1275         desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
1276
1277         its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
1278 }
1279
1280 static void its_send_vmovp(struct its_vpe *vpe)
1281 {
1282         struct its_cmd_desc desc = {};
1283         struct its_node *its;
1284         unsigned long flags;
1285         int col_id = vpe->col_idx;
1286
1287         desc.its_vmovp_cmd.vpe = vpe;
1288
1289         if (!its_list_map) {
1290                 its = list_first_entry(&its_nodes, struct its_node, entry);
1291                 desc.its_vmovp_cmd.col = &its->collections[col_id];
1292                 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1293                 return;
1294         }
1295
1296         /*
1297          * Yet another marvel of the architecture. If using the
1298          * its_list "feature", we need to make sure that all ITSs
1299          * receive all VMOVP commands in the same order. The only way
1300          * to guarantee this is to make vmovp a serialization point.
1301          *
1302          * Wall <-- Head.
1303          */
1304         raw_spin_lock_irqsave(&vmovp_lock, flags);
1305
1306         desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
1307         desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
1308
1309         /* Emit VMOVPs */
1310         list_for_each_entry(its, &its_nodes, entry) {
1311                 if (!is_v4(its))
1312                         continue;
1313
1314                 if (!require_its_list_vmovp(vpe->its_vm, its))
1315                         continue;
1316
1317                 desc.its_vmovp_cmd.col = &its->collections[col_id];
1318                 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1319         }
1320
1321         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1322 }
1323
1324 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1325 {
1326         struct its_cmd_desc desc;
1327
1328         desc.its_vinvall_cmd.vpe = vpe;
1329         its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1330 }
1331
1332 static void its_send_vinv(struct its_device *dev, u32 event_id)
1333 {
1334         struct its_cmd_desc desc;
1335
1336         /*
1337          * There is no real VINV command. This is just a normal INV,
1338          * with a VSYNC instead of a SYNC.
1339          */
1340         desc.its_inv_cmd.dev = dev;
1341         desc.its_inv_cmd.event_id = event_id;
1342
1343         its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
1344 }
1345
1346 static void its_send_vint(struct its_device *dev, u32 event_id)
1347 {
1348         struct its_cmd_desc desc;
1349
1350         /*
1351          * There is no real VINT command. This is just a normal INT,
1352          * with a VSYNC instead of a SYNC.
1353          */
1354         desc.its_int_cmd.dev = dev;
1355         desc.its_int_cmd.event_id = event_id;
1356
1357         its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
1358 }
1359
1360 static void its_send_vclear(struct its_device *dev, u32 event_id)
1361 {
1362         struct its_cmd_desc desc;
1363
1364         /*
1365          * There is no real VCLEAR command. This is just a normal CLEAR,
1366          * with a VSYNC instead of a SYNC.
1367          */
1368         desc.its_clear_cmd.dev = dev;
1369         desc.its_clear_cmd.event_id = event_id;
1370
1371         its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
1372 }
1373
1374 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1375 {
1376         struct its_cmd_desc desc;
1377
1378         desc.its_invdb_cmd.vpe = vpe;
1379         its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
1380 }
1381
1382 /*
1383  * irqchip functions - assumes MSI, mostly.
1384  */
1385 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1386 {
1387         struct its_vlpi_map *map = get_vlpi_map(d);
1388         irq_hw_number_t hwirq;
1389         void *va;
1390         u8 *cfg;
1391
1392         if (map) {
1393                 va = page_address(map->vm->vprop_page);
1394                 hwirq = map->vintid;
1395
1396                 /* Remember the updated property */
1397                 map->properties &= ~clr;
1398                 map->properties |= set | LPI_PROP_GROUP1;
1399         } else {
1400                 va = gic_rdists->prop_table_va;
1401                 hwirq = d->hwirq;
1402         }
1403
1404         cfg = va + hwirq - 8192;
1405         *cfg &= ~clr;
1406         *cfg |= set | LPI_PROP_GROUP1;
1407
1408         /*
1409          * Make the above write visible to the redistributors.
1410          * And yes, we're flushing exactly: One. Single. Byte.
1411          * Humpf...
1412          */
1413         if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
1414                 gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
1415         else
1416                 dsb(ishst);
1417 }
1418
1419 static void wait_for_syncr(void __iomem *rdbase)
1420 {
1421         while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
1422                 cpu_relax();
1423 }
1424
1425 static void direct_lpi_inv(struct irq_data *d)
1426 {
1427         struct its_vlpi_map *map = get_vlpi_map(d);
1428         void __iomem *rdbase;
1429         unsigned long flags;
1430         u64 val;
1431         int cpu;
1432
1433         if (map) {
1434                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1435
1436                 WARN_ON(!is_v4_1(its_dev->its));
1437
1438                 val  = GICR_INVLPIR_V;
1439                 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1440                 val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
1441         } else {
1442                 val = d->hwirq;
1443         }
1444
1445         /* Target the redistributor this LPI is currently routed to */
1446         cpu = irq_to_cpuid_lock(d, &flags);
1447         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
1448         rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
1449         gic_write_lpir(val, rdbase + GICR_INVLPIR);
1450
1451         wait_for_syncr(rdbase);
1452         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
1453         irq_to_cpuid_unlock(d, flags);
1454 }
1455
1456 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1457 {
1458         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1459
1460         lpi_write_config(d, clr, set);
1461         if (gic_rdists->has_direct_lpi &&
1462             (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
1463                 direct_lpi_inv(d);
1464         else if (!irqd_is_forwarded_to_vcpu(d))
1465                 its_send_inv(its_dev, its_get_event_id(d));
1466         else
1467                 its_send_vinv(its_dev, its_get_event_id(d));
1468 }
1469
1470 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1471 {
1472         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1473         u32 event = its_get_event_id(d);
1474         struct its_vlpi_map *map;
1475
1476         /*
1477          * GICv4.1 does away with the per-LPI nonsense, nothing to do
1478          * here.
1479          */
1480         if (is_v4_1(its_dev->its))
1481                 return;
1482
1483         map = dev_event_to_vlpi_map(its_dev, event);
1484
1485         if (map->db_enabled == enable)
1486                 return;
1487
1488         map->db_enabled = enable;
1489
1490         /*
1491          * More fun with the architecture:
1492          *
1493          * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1494          * value or to 1023, depending on the enable bit. But that
1495          * would be issuing a mapping for an /existing/ DevID+EventID
1496          * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1497          * to the /same/ vPE, using this opportunity to adjust the
1498          * doorbell. Mouahahahaha. We loves it, Precious.
1499          */
1500         its_send_vmovi(its_dev, event);
1501 }
1502
1503 static void its_mask_irq(struct irq_data *d)
1504 {
1505         if (irqd_is_forwarded_to_vcpu(d))
1506                 its_vlpi_set_doorbell(d, false);
1507
1508         lpi_update_config(d, LPI_PROP_ENABLED, 0);
1509 }
1510
1511 static void its_unmask_irq(struct irq_data *d)
1512 {
1513         if (irqd_is_forwarded_to_vcpu(d))
1514                 its_vlpi_set_doorbell(d, true);
1515
1516         lpi_update_config(d, 0, LPI_PROP_ENABLED);
1517 }
1518
1519 static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
1520 {
1521         if (irqd_affinity_is_managed(d))
1522                 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1523
1524         return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1525 }
1526
1527 static void its_inc_lpi_count(struct irq_data *d, int cpu)
1528 {
1529         if (irqd_affinity_is_managed(d))
1530                 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1531         else
1532                 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1533 }
1534
1535 static void its_dec_lpi_count(struct irq_data *d, int cpu)
1536 {
1537         if (irqd_affinity_is_managed(d))
1538                 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1539         else
1540                 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1541 }
1542
1543 static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
1544                                               const struct cpumask *cpu_mask)
1545 {
1546         unsigned int cpu = nr_cpu_ids, tmp;
1547         int count = S32_MAX;
1548
1549         for_each_cpu(tmp, cpu_mask) {
1550                 int this_count = its_read_lpi_count(d, tmp);
1551                 if (this_count < count) {
1552                         cpu = tmp;
1553                         count = this_count;
1554                 }
1555         }
1556
1557         return cpu;
1558 }
1559
1560 /*
1561  * As suggested by Thomas Gleixner in:
1562  * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1563  */
1564 static int its_select_cpu(struct irq_data *d,
1565                           const struct cpumask *aff_mask)
1566 {
1567         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1568         cpumask_var_t tmpmask;
1569         int cpu, node;
1570
1571         if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
1572                 return -ENOMEM;
1573
1574         node = its_dev->its->numa_node;
1575
1576         if (!irqd_affinity_is_managed(d)) {
1577                 /* First try the NUMA node */
1578                 if (node != NUMA_NO_NODE) {
1579                         /*
1580                          * Try the intersection of the affinity mask and the
1581                          * node mask (and the online mask, just to be safe).
1582                          */
1583                         cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
1584                         cpumask_and(tmpmask, tmpmask, cpu_online_mask);
1585
1586                         /*
1587                          * Ideally, we would check if the mask is empty, and
1588                          * try again on the full node here.
1589                          *
1590                          * But it turns out that the way ACPI describes the
1591                          * affinity for ITSs only deals about memory, and
1592                          * not target CPUs, so it cannot describe a single
1593                          * ITS placed next to two NUMA nodes.
1594                          *
1595                          * Instead, just fallback on the online mask. This
1596                          * diverges from Thomas' suggestion above.
1597                          */
1598                         cpu = cpumask_pick_least_loaded(d, tmpmask);
1599                         if (cpu < nr_cpu_ids)
1600                                 goto out;
1601
1602                         /* If we can't cross sockets, give up */
1603                         if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
1604                                 goto out;
1605
1606                         /* If the above failed, expand the search */
1607                 }
1608
1609                 /* Try the intersection of the affinity and online masks */
1610                 cpumask_and(tmpmask, aff_mask, cpu_online_mask);
1611
1612                 /* If that doesn't fly, the online mask is the last resort */
1613                 if (cpumask_empty(tmpmask))
1614                         cpumask_copy(tmpmask, cpu_online_mask);
1615
1616                 cpu = cpumask_pick_least_loaded(d, tmpmask);
1617         } else {
1618                 cpumask_and(tmpmask, irq_data_get_affinity_mask(d), cpu_online_mask);
1619
1620                 /* If we cannot cross sockets, limit the search to that node */
1621                 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
1622                     node != NUMA_NO_NODE)
1623                         cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
1624
1625                 cpu = cpumask_pick_least_loaded(d, tmpmask);
1626         }
1627 out:
1628         free_cpumask_var(tmpmask);
1629
1630         pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
1631         return cpu;
1632 }
1633
1634 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1635                             bool force)
1636 {
1637         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1638         struct its_collection *target_col;
1639         u32 id = its_get_event_id(d);
1640         int cpu, prev_cpu;
1641
1642         /* A forwarded interrupt should use irq_set_vcpu_affinity */
1643         if (irqd_is_forwarded_to_vcpu(d))
1644                 return -EINVAL;
1645
1646         prev_cpu = its_dev->event_map.col_map[id];
1647         its_dec_lpi_count(d, prev_cpu);
1648
1649         if (!force)
1650                 cpu = its_select_cpu(d, mask_val);
1651         else
1652                 cpu = cpumask_pick_least_loaded(d, mask_val);
1653
1654         if (cpu < 0 || cpu >= nr_cpu_ids)
1655                 goto err;
1656
1657         /* don't set the affinity when the target cpu is same as current one */
1658         if (cpu != prev_cpu) {
1659                 target_col = &its_dev->its->collections[cpu];
1660                 its_send_movi(its_dev, target_col, id);
1661                 its_dev->event_map.col_map[id] = cpu;
1662                 irq_data_update_effective_affinity(d, cpumask_of(cpu));
1663         }
1664
1665         its_inc_lpi_count(d, cpu);
1666
1667         return IRQ_SET_MASK_OK_DONE;
1668
1669 err:
1670         its_inc_lpi_count(d, prev_cpu);
1671         return -EINVAL;
1672 }
1673
1674 static u64 its_irq_get_msi_base(struct its_device *its_dev)
1675 {
1676         struct its_node *its = its_dev->its;
1677
1678         return its->phys_base + GITS_TRANSLATER;
1679 }
1680
1681 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1682 {
1683         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1684         struct its_node *its;
1685         u64 addr;
1686
1687         its = its_dev->its;
1688         addr = its->get_msi_base(its_dev);
1689
1690         msg->address_lo         = lower_32_bits(addr);
1691         msg->address_hi         = upper_32_bits(addr);
1692         msg->data               = its_get_event_id(d);
1693
1694         iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1695 }
1696
1697 static int its_irq_set_irqchip_state(struct irq_data *d,
1698                                      enum irqchip_irq_state which,
1699                                      bool state)
1700 {
1701         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1702         u32 event = its_get_event_id(d);
1703
1704         if (which != IRQCHIP_STATE_PENDING)
1705                 return -EINVAL;
1706
1707         if (irqd_is_forwarded_to_vcpu(d)) {
1708                 if (state)
1709                         its_send_vint(its_dev, event);
1710                 else
1711                         its_send_vclear(its_dev, event);
1712         } else {
1713                 if (state)
1714                         its_send_int(its_dev, event);
1715                 else
1716                         its_send_clear(its_dev, event);
1717         }
1718
1719         return 0;
1720 }
1721
1722 static int its_irq_retrigger(struct irq_data *d)
1723 {
1724         return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
1725 }
1726
1727 /*
1728  * Two favourable cases:
1729  *
1730  * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1731  *     for vSGI delivery
1732  *
1733  * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1734  *     and we're better off mapping all VPEs always
1735  *
1736  * If neither (a) nor (b) is true, then we map vPEs on demand.
1737  *
1738  */
1739 static bool gic_requires_eager_mapping(void)
1740 {
1741         if (!its_list_map || gic_rdists->has_rvpeid)
1742                 return true;
1743
1744         return false;
1745 }
1746
1747 static void its_map_vm(struct its_node *its, struct its_vm *vm)
1748 {
1749         unsigned long flags;
1750
1751         if (gic_requires_eager_mapping())
1752                 return;
1753
1754         raw_spin_lock_irqsave(&vmovp_lock, flags);
1755
1756         /*
1757          * If the VM wasn't mapped yet, iterate over the vpes and get
1758          * them mapped now.
1759          */
1760         vm->vlpi_count[its->list_nr]++;
1761
1762         if (vm->vlpi_count[its->list_nr] == 1) {
1763                 int i;
1764
1765                 for (i = 0; i < vm->nr_vpes; i++) {
1766                         struct its_vpe *vpe = vm->vpes[i];
1767                         struct irq_data *d = irq_get_irq_data(vpe->irq);
1768
1769                         /* Map the VPE to the first possible CPU */
1770                         vpe->col_idx = cpumask_first(cpu_online_mask);
1771                         its_send_vmapp(its, vpe, true);
1772                         its_send_vinvall(its, vpe);
1773                         irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
1774                 }
1775         }
1776
1777         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1778 }
1779
1780 static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1781 {
1782         unsigned long flags;
1783
1784         /* Not using the ITS list? Everything is always mapped. */
1785         if (gic_requires_eager_mapping())
1786                 return;
1787
1788         raw_spin_lock_irqsave(&vmovp_lock, flags);
1789
1790         if (!--vm->vlpi_count[its->list_nr]) {
1791                 int i;
1792
1793                 for (i = 0; i < vm->nr_vpes; i++)
1794                         its_send_vmapp(its, vm->vpes[i], false);
1795         }
1796
1797         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1798 }
1799
1800 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1801 {
1802         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1803         u32 event = its_get_event_id(d);
1804         int ret = 0;
1805
1806         if (!info->map)
1807                 return -EINVAL;
1808
1809         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1810
1811         if (!its_dev->event_map.vm) {
1812                 struct its_vlpi_map *maps;
1813
1814                 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1815                                GFP_ATOMIC);
1816                 if (!maps) {
1817                         ret = -ENOMEM;
1818                         goto out;
1819                 }
1820
1821                 its_dev->event_map.vm = info->map->vm;
1822                 its_dev->event_map.vlpi_maps = maps;
1823         } else if (its_dev->event_map.vm != info->map->vm) {
1824                 ret = -EINVAL;
1825                 goto out;
1826         }
1827
1828         /* Get our private copy of the mapping information */
1829         its_dev->event_map.vlpi_maps[event] = *info->map;
1830
1831         if (irqd_is_forwarded_to_vcpu(d)) {
1832                 /* Already mapped, move it around */
1833                 its_send_vmovi(its_dev, event);
1834         } else {
1835                 /* Ensure all the VPEs are mapped on this ITS */
1836                 its_map_vm(its_dev->its, info->map->vm);
1837
1838                 /*
1839                  * Flag the interrupt as forwarded so that we can
1840                  * start poking the virtual property table.
1841                  */
1842                 irqd_set_forwarded_to_vcpu(d);
1843
1844                 /* Write out the property to the prop table */
1845                 lpi_write_config(d, 0xff, info->map->properties);
1846
1847                 /* Drop the physical mapping */
1848                 its_send_discard(its_dev, event);
1849
1850                 /* and install the virtual one */
1851                 its_send_vmapti(its_dev, event);
1852
1853                 /* Increment the number of VLPIs */
1854                 its_dev->event_map.nr_vlpis++;
1855         }
1856
1857 out:
1858         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1859         return ret;
1860 }
1861
1862 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1863 {
1864         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1865         struct its_vlpi_map *map;
1866         int ret = 0;
1867
1868         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1869
1870         map = get_vlpi_map(d);
1871
1872         if (!its_dev->event_map.vm || !map) {
1873                 ret = -EINVAL;
1874                 goto out;
1875         }
1876
1877         /* Copy our mapping information to the incoming request */
1878         *info->map = *map;
1879
1880 out:
1881         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1882         return ret;
1883 }
1884
1885 static int its_vlpi_unmap(struct irq_data *d)
1886 {
1887         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1888         u32 event = its_get_event_id(d);
1889         int ret = 0;
1890
1891         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1892
1893         if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) {
1894                 ret = -EINVAL;
1895                 goto out;
1896         }
1897
1898         /* Drop the virtual mapping */
1899         its_send_discard(its_dev, event);
1900
1901         /* and restore the physical one */
1902         irqd_clr_forwarded_to_vcpu(d);
1903         its_send_mapti(its_dev, d->hwirq, event);
1904         lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
1905                                     LPI_PROP_ENABLED |
1906                                     LPI_PROP_GROUP1));
1907
1908         /* Potentially unmap the VM from this ITS */
1909         its_unmap_vm(its_dev->its, its_dev->event_map.vm);
1910
1911         /*
1912          * Drop the refcount and make the device available again if
1913          * this was the last VLPI.
1914          */
1915         if (!--its_dev->event_map.nr_vlpis) {
1916                 its_dev->event_map.vm = NULL;
1917                 kfree(its_dev->event_map.vlpi_maps);
1918         }
1919
1920 out:
1921         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1922         return ret;
1923 }
1924
1925 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
1926 {
1927         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1928
1929         if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1930                 return -EINVAL;
1931
1932         if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
1933                 lpi_update_config(d, 0xff, info->config);
1934         else
1935                 lpi_write_config(d, 0xff, info->config);
1936         its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
1937
1938         return 0;
1939 }
1940
1941 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
1942 {
1943         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1944         struct its_cmd_info *info = vcpu_info;
1945
1946         /* Need a v4 ITS */
1947         if (!is_v4(its_dev->its))
1948                 return -EINVAL;
1949
1950         /* Unmap request? */
1951         if (!info)
1952                 return its_vlpi_unmap(d);
1953
1954         switch (info->cmd_type) {
1955         case MAP_VLPI:
1956                 return its_vlpi_map(d, info);
1957
1958         case GET_VLPI:
1959                 return its_vlpi_get(d, info);
1960
1961         case PROP_UPDATE_VLPI:
1962         case PROP_UPDATE_AND_INV_VLPI:
1963                 return its_vlpi_prop_update(d, info);
1964
1965         default:
1966                 return -EINVAL;
1967         }
1968 }
1969
1970 static struct irq_chip its_irq_chip = {
1971         .name                   = "ITS",
1972         .irq_mask               = its_mask_irq,
1973         .irq_unmask             = its_unmask_irq,
1974         .irq_eoi                = irq_chip_eoi_parent,
1975         .irq_set_affinity       = its_set_affinity,
1976         .irq_compose_msi_msg    = its_irq_compose_msi_msg,
1977         .irq_set_irqchip_state  = its_irq_set_irqchip_state,
1978         .irq_retrigger          = its_irq_retrigger,
1979         .irq_set_vcpu_affinity  = its_irq_set_vcpu_affinity,
1980 };
1981
1982
1983 /*
1984  * How we allocate LPIs:
1985  *
1986  * lpi_range_list contains ranges of LPIs that are to available to
1987  * allocate from. To allocate LPIs, just pick the first range that
1988  * fits the required allocation, and reduce it by the required
1989  * amount. Once empty, remove the range from the list.
1990  *
1991  * To free a range of LPIs, add a free range to the list, sort it and
1992  * merge the result if the new range happens to be adjacent to an
1993  * already free block.
1994  *
1995  * The consequence of the above is that allocation is cost is low, but
1996  * freeing is expensive. We assumes that freeing rarely occurs.
1997  */
1998 #define ITS_MAX_LPI_NRBITS      16 /* 64K LPIs */
1999
2000 static DEFINE_MUTEX(lpi_range_lock);
2001 static LIST_HEAD(lpi_range_list);
2002
2003 struct lpi_range {
2004         struct list_head        entry;
2005         u32                     base_id;
2006         u32                     span;
2007 };
2008
2009 static struct lpi_range *mk_lpi_range(u32 base, u32 span)
2010 {
2011         struct lpi_range *range;
2012
2013         range = kmalloc(sizeof(*range), GFP_KERNEL);
2014         if (range) {
2015                 range->base_id = base;
2016                 range->span = span;
2017         }
2018
2019         return range;
2020 }
2021
2022 static int alloc_lpi_range(u32 nr_lpis, u32 *base)
2023 {
2024         struct lpi_range *range, *tmp;
2025         int err = -ENOSPC;
2026
2027         mutex_lock(&lpi_range_lock);
2028
2029         list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
2030                 if (range->span >= nr_lpis) {
2031                         *base = range->base_id;
2032                         range->base_id += nr_lpis;
2033                         range->span -= nr_lpis;
2034
2035                         if (range->span == 0) {
2036                                 list_del(&range->entry);
2037                                 kfree(range);
2038                         }
2039
2040                         err = 0;
2041                         break;
2042                 }
2043         }
2044
2045         mutex_unlock(&lpi_range_lock);
2046
2047         pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
2048         return err;
2049 }
2050
2051 static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
2052 {
2053         if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
2054                 return;
2055         if (a->base_id + a->span != b->base_id)
2056                 return;
2057         b->base_id = a->base_id;
2058         b->span += a->span;
2059         list_del(&a->entry);
2060         kfree(a);
2061 }
2062
2063 static int free_lpi_range(u32 base, u32 nr_lpis)
2064 {
2065         struct lpi_range *new, *old;
2066
2067         new = mk_lpi_range(base, nr_lpis);
2068         if (!new)
2069                 return -ENOMEM;
2070
2071         mutex_lock(&lpi_range_lock);
2072
2073         list_for_each_entry_reverse(old, &lpi_range_list, entry) {
2074                 if (old->base_id < base)
2075                         break;
2076         }
2077         /*
2078          * old is the last element with ->base_id smaller than base,
2079          * so new goes right after it. If there are no elements with
2080          * ->base_id smaller than base, &old->entry ends up pointing
2081          * at the head of the list, and inserting new it the start of
2082          * the list is the right thing to do in that case as well.
2083          */
2084         list_add(&new->entry, &old->entry);
2085         /*
2086          * Now check if we can merge with the preceding and/or
2087          * following ranges.
2088          */
2089         merge_lpi_ranges(old, new);
2090         merge_lpi_ranges(new, list_next_entry(new, entry));
2091
2092         mutex_unlock(&lpi_range_lock);
2093         return 0;
2094 }
2095
2096 static int __init its_lpi_init(u32 id_bits)
2097 {
2098         u32 lpis = (1UL << id_bits) - 8192;
2099         u32 numlpis;
2100         int err;
2101
2102         numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
2103
2104         if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
2105                 lpis = numlpis;
2106                 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
2107                         lpis);
2108         }
2109
2110         /*
2111          * Initializing the allocator is just the same as freeing the
2112          * full range of LPIs.
2113          */
2114         err = free_lpi_range(8192, lpis);
2115         pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
2116         return err;
2117 }
2118
2119 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
2120 {
2121         unsigned long *bitmap = NULL;
2122         int err = 0;
2123
2124         do {
2125                 err = alloc_lpi_range(nr_irqs, base);
2126                 if (!err)
2127                         break;
2128
2129                 nr_irqs /= 2;
2130         } while (nr_irqs > 0);
2131
2132         if (!nr_irqs)
2133                 err = -ENOSPC;
2134
2135         if (err)
2136                 goto out;
2137
2138         bitmap = kcalloc(BITS_TO_LONGS(nr_irqs), sizeof (long), GFP_ATOMIC);
2139         if (!bitmap)
2140                 goto out;
2141
2142         *nr_ids = nr_irqs;
2143
2144 out:
2145         if (!bitmap)
2146                 *base = *nr_ids = 0;
2147
2148         return bitmap;
2149 }
2150
2151 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
2152 {
2153         WARN_ON(free_lpi_range(base, nr_ids));
2154         kfree(bitmap);
2155 }
2156
2157 static void gic_reset_prop_table(void *va)
2158 {
2159         /* Priority 0xa0, Group-1, disabled */
2160         memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
2161
2162         /* Make sure the GIC will observe the written configuration */
2163         gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
2164 }
2165
2166 static struct page *its_allocate_prop_table(gfp_t gfp_flags)
2167 {
2168         struct page *prop_page;
2169
2170         prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
2171         if (!prop_page)
2172                 return NULL;
2173
2174         gic_reset_prop_table(page_address(prop_page));
2175
2176         return prop_page;
2177 }
2178
2179 static void its_free_prop_table(struct page *prop_page)
2180 {
2181         free_pages((unsigned long)page_address(prop_page),
2182                    get_order(LPI_PROPBASE_SZ));
2183 }
2184
2185 static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
2186 {
2187         phys_addr_t start, end, addr_end;
2188         u64 i;
2189
2190         /*
2191          * We don't bother checking for a kdump kernel as by
2192          * construction, the LPI tables are out of this kernel's
2193          * memory map.
2194          */
2195         if (is_kdump_kernel())
2196                 return true;
2197
2198         addr_end = addr + size - 1;
2199
2200         for_each_reserved_mem_range(i, &start, &end) {
2201                 if (addr >= start && addr_end <= end)
2202                         return true;
2203         }
2204
2205         /* Not found, not a good sign... */
2206         pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
2207                 &addr, &addr_end);
2208         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2209         return false;
2210 }
2211
2212 static int gic_reserve_range(phys_addr_t addr, unsigned long size)
2213 {
2214         if (efi_enabled(EFI_CONFIG_TABLES))
2215                 return efi_mem_reserve_persistent(addr, size);
2216
2217         return 0;
2218 }
2219
2220 static int __init its_setup_lpi_prop_table(void)
2221 {
2222         if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
2223                 u64 val;
2224
2225                 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2226                 lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
2227
2228                 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
2229                 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
2230                                                      LPI_PROPBASE_SZ,
2231                                                      MEMREMAP_WB);
2232                 gic_reset_prop_table(gic_rdists->prop_table_va);
2233         } else {
2234                 struct page *page;
2235
2236                 lpi_id_bits = min_t(u32,
2237                                     GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
2238                                     ITS_MAX_LPI_NRBITS);
2239                 page = its_allocate_prop_table(GFP_NOWAIT);
2240                 if (!page) {
2241                         pr_err("Failed to allocate PROPBASE\n");
2242                         return -ENOMEM;
2243                 }
2244
2245                 gic_rdists->prop_table_pa = page_to_phys(page);
2246                 gic_rdists->prop_table_va = page_address(page);
2247                 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
2248                                           LPI_PROPBASE_SZ));
2249         }
2250
2251         pr_info("GICv3: using LPI property table @%pa\n",
2252                 &gic_rdists->prop_table_pa);
2253
2254         return its_lpi_init(lpi_id_bits);
2255 }
2256
2257 static const char *its_base_type_string[] = {
2258         [GITS_BASER_TYPE_DEVICE]        = "Devices",
2259         [GITS_BASER_TYPE_VCPU]          = "Virtual CPUs",
2260         [GITS_BASER_TYPE_RESERVED3]     = "Reserved (3)",
2261         [GITS_BASER_TYPE_COLLECTION]    = "Interrupt Collections",
2262         [GITS_BASER_TYPE_RESERVED5]     = "Reserved (5)",
2263         [GITS_BASER_TYPE_RESERVED6]     = "Reserved (6)",
2264         [GITS_BASER_TYPE_RESERVED7]     = "Reserved (7)",
2265 };
2266
2267 static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
2268 {
2269         u32 idx = baser - its->tables;
2270
2271         return gits_read_baser(its->base + GITS_BASER + (idx << 3));
2272 }
2273
2274 static void its_write_baser(struct its_node *its, struct its_baser *baser,
2275                             u64 val)
2276 {
2277         u32 idx = baser - its->tables;
2278
2279         gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
2280         baser->val = its_read_baser(its, baser);
2281 }
2282
2283 static int its_setup_baser(struct its_node *its, struct its_baser *baser,
2284                            u64 cache, u64 shr, u32 order, bool indirect)
2285 {
2286         u64 val = its_read_baser(its, baser);
2287         u64 esz = GITS_BASER_ENTRY_SIZE(val);
2288         u64 type = GITS_BASER_TYPE(val);
2289         u64 baser_phys, tmp;
2290         u32 alloc_pages, psz;
2291         struct page *page;
2292         void *base;
2293
2294         psz = baser->psz;
2295         alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
2296         if (alloc_pages > GITS_BASER_PAGES_MAX) {
2297                 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
2298                         &its->phys_base, its_base_type_string[type],
2299                         alloc_pages, GITS_BASER_PAGES_MAX);
2300                 alloc_pages = GITS_BASER_PAGES_MAX;
2301                 order = get_order(GITS_BASER_PAGES_MAX * psz);
2302         }
2303
2304         page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
2305         if (!page)
2306                 return -ENOMEM;
2307
2308         base = (void *)page_address(page);
2309         baser_phys = virt_to_phys(base);
2310
2311         /* Check if the physical address of the memory is above 48bits */
2312         if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
2313
2314                 /* 52bit PA is supported only when PageSize=64K */
2315                 if (psz != SZ_64K) {
2316                         pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
2317                         free_pages((unsigned long)base, order);
2318                         return -ENXIO;
2319                 }
2320
2321                 /* Convert 52bit PA to 48bit field */
2322                 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
2323         }
2324
2325 retry_baser:
2326         val = (baser_phys                                        |
2327                 (type << GITS_BASER_TYPE_SHIFT)                  |
2328                 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT)       |
2329                 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT)    |
2330                 cache                                            |
2331                 shr                                              |
2332                 GITS_BASER_VALID);
2333
2334         val |=  indirect ? GITS_BASER_INDIRECT : 0x0;
2335
2336         switch (psz) {
2337         case SZ_4K:
2338                 val |= GITS_BASER_PAGE_SIZE_4K;
2339                 break;
2340         case SZ_16K:
2341                 val |= GITS_BASER_PAGE_SIZE_16K;
2342                 break;
2343         case SZ_64K:
2344                 val |= GITS_BASER_PAGE_SIZE_64K;
2345                 break;
2346         }
2347
2348         its_write_baser(its, baser, val);
2349         tmp = baser->val;
2350
2351         if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
2352                 /*
2353                  * Shareability didn't stick. Just use
2354                  * whatever the read reported, which is likely
2355                  * to be the only thing this redistributor
2356                  * supports. If that's zero, make it
2357                  * non-cacheable as well.
2358                  */
2359                 shr = tmp & GITS_BASER_SHAREABILITY_MASK;
2360                 if (!shr) {
2361                         cache = GITS_BASER_nC;
2362                         gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
2363                 }
2364                 goto retry_baser;
2365         }
2366
2367         if (val != tmp) {
2368                 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
2369                        &its->phys_base, its_base_type_string[type],
2370                        val, tmp);
2371                 free_pages((unsigned long)base, order);
2372                 return -ENXIO;
2373         }
2374
2375         baser->order = order;
2376         baser->base = base;
2377         baser->psz = psz;
2378         tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
2379
2380         pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
2381                 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
2382                 its_base_type_string[type],
2383                 (unsigned long)virt_to_phys(base),
2384                 indirect ? "indirect" : "flat", (int)esz,
2385                 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
2386
2387         return 0;
2388 }
2389
2390 static bool its_parse_indirect_baser(struct its_node *its,
2391                                      struct its_baser *baser,
2392                                      u32 *order, u32 ids)
2393 {
2394         u64 tmp = its_read_baser(its, baser);
2395         u64 type = GITS_BASER_TYPE(tmp);
2396         u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
2397         u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
2398         u32 new_order = *order;
2399         u32 psz = baser->psz;
2400         bool indirect = false;
2401
2402         /* No need to enable Indirection if memory requirement < (psz*2)bytes */
2403         if ((esz << ids) > (psz * 2)) {
2404                 /*
2405                  * Find out whether hw supports a single or two-level table by
2406                  * table by reading bit at offset '62' after writing '1' to it.
2407                  */
2408                 its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
2409                 indirect = !!(baser->val & GITS_BASER_INDIRECT);
2410
2411                 if (indirect) {
2412                         /*
2413                          * The size of the lvl2 table is equal to ITS page size
2414                          * which is 'psz'. For computing lvl1 table size,
2415                          * subtract ID bits that sparse lvl2 table from 'ids'
2416                          * which is reported by ITS hardware times lvl1 table
2417                          * entry size.
2418                          */
2419                         ids -= ilog2(psz / (int)esz);
2420                         esz = GITS_LVL1_ENTRY_SIZE;
2421                 }
2422         }
2423
2424         /*
2425          * Allocate as many entries as required to fit the
2426          * range of device IDs that the ITS can grok... The ID
2427          * space being incredibly sparse, this results in a
2428          * massive waste of memory if two-level device table
2429          * feature is not supported by hardware.
2430          */
2431         new_order = max_t(u32, get_order(esz << ids), new_order);
2432         if (new_order >= MAX_ORDER) {
2433                 new_order = MAX_ORDER - 1;
2434                 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
2435                 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
2436                         &its->phys_base, its_base_type_string[type],
2437                         device_ids(its), ids);
2438         }
2439
2440         *order = new_order;
2441
2442         return indirect;
2443 }
2444
2445 static u32 compute_common_aff(u64 val)
2446 {
2447         u32 aff, clpiaff;
2448
2449         aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
2450         clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
2451
2452         return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
2453 }
2454
2455 static u32 compute_its_aff(struct its_node *its)
2456 {
2457         u64 val;
2458         u32 svpet;
2459
2460         /*
2461          * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
2462          * the resulting affinity. We then use that to see if this match
2463          * our own affinity.
2464          */
2465         svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
2466         val  = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
2467         val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
2468         return compute_common_aff(val);
2469 }
2470
2471 static struct its_node *find_sibling_its(struct its_node *cur_its)
2472 {
2473         struct its_node *its;
2474         u32 aff;
2475
2476         if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
2477                 return NULL;
2478
2479         aff = compute_its_aff(cur_its);
2480
2481         list_for_each_entry(its, &its_nodes, entry) {
2482                 u64 baser;
2483
2484                 if (!is_v4_1(its) || its == cur_its)
2485                         continue;
2486
2487                 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2488                         continue;
2489
2490                 if (aff != compute_its_aff(its))
2491                         continue;
2492
2493                 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2494                 baser = its->tables[2].val;
2495                 if (!(baser & GITS_BASER_VALID))
2496                         continue;
2497
2498                 return its;
2499         }
2500
2501         return NULL;
2502 }
2503
2504 static void its_free_tables(struct its_node *its)
2505 {
2506         int i;
2507
2508         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2509                 if (its->tables[i].base) {
2510                         free_pages((unsigned long)its->tables[i].base,
2511                                    its->tables[i].order);
2512                         its->tables[i].base = NULL;
2513                 }
2514         }
2515 }
2516
2517 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
2518 {
2519         u64 psz = SZ_64K;
2520
2521         while (psz) {
2522                 u64 val, gpsz;
2523
2524                 val = its_read_baser(its, baser);
2525                 val &= ~GITS_BASER_PAGE_SIZE_MASK;
2526
2527                 switch (psz) {
2528                 case SZ_64K:
2529                         gpsz = GITS_BASER_PAGE_SIZE_64K;
2530                         break;
2531                 case SZ_16K:
2532                         gpsz = GITS_BASER_PAGE_SIZE_16K;
2533                         break;
2534                 case SZ_4K:
2535                 default:
2536                         gpsz = GITS_BASER_PAGE_SIZE_4K;
2537                         break;
2538                 }
2539
2540                 gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
2541
2542                 val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
2543                 its_write_baser(its, baser, val);
2544
2545                 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
2546                         break;
2547
2548                 switch (psz) {
2549                 case SZ_64K:
2550                         psz = SZ_16K;
2551                         break;
2552                 case SZ_16K:
2553                         psz = SZ_4K;
2554                         break;
2555                 case SZ_4K:
2556                 default:
2557                         return -1;
2558                 }
2559         }
2560
2561         baser->psz = psz;
2562         return 0;
2563 }
2564
2565 static int its_alloc_tables(struct its_node *its)
2566 {
2567         u64 shr = GITS_BASER_InnerShareable;
2568         u64 cache = GITS_BASER_RaWaWb;
2569         int err, i;
2570
2571         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
2572                 /* erratum 24313: ignore memory access type */
2573                 cache = GITS_BASER_nCnB;
2574
2575         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2576                 struct its_baser *baser = its->tables + i;
2577                 u64 val = its_read_baser(its, baser);
2578                 u64 type = GITS_BASER_TYPE(val);
2579                 bool indirect = false;
2580                 u32 order;
2581
2582                 if (type == GITS_BASER_TYPE_NONE)
2583                         continue;
2584
2585                 if (its_probe_baser_psz(its, baser)) {
2586                         its_free_tables(its);
2587                         return -ENXIO;
2588                 }
2589
2590                 order = get_order(baser->psz);
2591
2592                 switch (type) {
2593                 case GITS_BASER_TYPE_DEVICE:
2594                         indirect = its_parse_indirect_baser(its, baser, &order,
2595                                                             device_ids(its));
2596                         break;
2597
2598                 case GITS_BASER_TYPE_VCPU:
2599                         if (is_v4_1(its)) {
2600                                 struct its_node *sibling;
2601
2602                                 WARN_ON(i != 2);
2603                                 if ((sibling = find_sibling_its(its))) {
2604                                         *baser = sibling->tables[2];
2605                                         its_write_baser(its, baser, baser->val);
2606                                         continue;
2607                                 }
2608                         }
2609
2610                         indirect = its_parse_indirect_baser(its, baser, &order,
2611                                                             ITS_MAX_VPEID_BITS);
2612                         break;
2613                 }
2614
2615                 err = its_setup_baser(its, baser, cache, shr, order, indirect);
2616                 if (err < 0) {
2617                         its_free_tables(its);
2618                         return err;
2619                 }
2620
2621                 /* Update settings which will be used for next BASERn */
2622                 cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
2623                 shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
2624         }
2625
2626         return 0;
2627 }
2628
2629 static u64 inherit_vpe_l1_table_from_its(void)
2630 {
2631         struct its_node *its;
2632         u64 val;
2633         u32 aff;
2634
2635         val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2636         aff = compute_common_aff(val);
2637
2638         list_for_each_entry(its, &its_nodes, entry) {
2639                 u64 baser, addr;
2640
2641                 if (!is_v4_1(its))
2642                         continue;
2643
2644                 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2645                         continue;
2646
2647                 if (aff != compute_its_aff(its))
2648                         continue;
2649
2650                 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2651                 baser = its->tables[2].val;
2652                 if (!(baser & GITS_BASER_VALID))
2653                         continue;
2654
2655                 /* We have a winner! */
2656                 gic_data_rdist()->vpe_l1_base = its->tables[2].base;
2657
2658                 val  = GICR_VPROPBASER_4_1_VALID;
2659                 if (baser & GITS_BASER_INDIRECT)
2660                         val |= GICR_VPROPBASER_4_1_INDIRECT;
2661                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
2662                                   FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
2663                 switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
2664                 case GIC_PAGE_SIZE_64K:
2665                         addr = GITS_BASER_ADDR_48_to_52(baser);
2666                         break;
2667                 default:
2668                         addr = baser & GENMASK_ULL(47, 12);
2669                         break;
2670                 }
2671                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
2672                 val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
2673                                   FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
2674                 val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
2675                                   FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
2676                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
2677
2678                 return val;
2679         }
2680
2681         return 0;
2682 }
2683
2684 static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
2685 {
2686         u32 aff;
2687         u64 val;
2688         int cpu;
2689
2690         val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2691         aff = compute_common_aff(val);
2692
2693         for_each_possible_cpu(cpu) {
2694                 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2695
2696                 if (!base || cpu == smp_processor_id())
2697                         continue;
2698
2699                 val = gic_read_typer(base + GICR_TYPER);
2700                 if (aff != compute_common_aff(val))
2701                         continue;
2702
2703                 /*
2704                  * At this point, we have a victim. This particular CPU
2705                  * has already booted, and has an affinity that matches
2706                  * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
2707                  * Make sure we don't write the Z bit in that case.
2708                  */
2709                 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2710                 val &= ~GICR_VPROPBASER_4_1_Z;
2711
2712                 gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2713                 *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
2714
2715                 return val;
2716         }
2717
2718         return 0;
2719 }
2720
2721 static bool allocate_vpe_l2_table(int cpu, u32 id)
2722 {
2723         void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2724         unsigned int psz, esz, idx, npg, gpsz;
2725         u64 val;
2726         struct page *page;
2727         __le64 *table;
2728
2729         if (!gic_rdists->has_rvpeid)
2730                 return true;
2731
2732         /* Skip non-present CPUs */
2733         if (!base)
2734                 return true;
2735
2736         val  = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2737
2738         esz  = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
2739         gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2740         npg  = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
2741
2742         switch (gpsz) {
2743         default:
2744                 WARN_ON(1);
2745                 fallthrough;
2746         case GIC_PAGE_SIZE_4K:
2747                 psz = SZ_4K;
2748                 break;
2749         case GIC_PAGE_SIZE_16K:
2750                 psz = SZ_16K;
2751                 break;
2752         case GIC_PAGE_SIZE_64K:
2753                 psz = SZ_64K;
2754                 break;
2755         }
2756
2757         /* Don't allow vpe_id that exceeds single, flat table limit */
2758         if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
2759                 return (id < (npg * psz / (esz * SZ_8)));
2760
2761         /* Compute 1st level table index & check if that exceeds table limit */
2762         idx = id >> ilog2(psz / (esz * SZ_8));
2763         if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
2764                 return false;
2765
2766         table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2767
2768         /* Allocate memory for 2nd level table */
2769         if (!table[idx]) {
2770                 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
2771                 if (!page)
2772                         return false;
2773
2774                 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
2775                 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2776                         gic_flush_dcache_to_poc(page_address(page), psz);
2777
2778                 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
2779
2780                 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2781                 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2782                         gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
2783
2784                 /* Ensure updated table contents are visible to RD hardware */
2785                 dsb(sy);
2786         }
2787
2788         return true;
2789 }
2790
2791 static int allocate_vpe_l1_table(void)
2792 {
2793         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2794         u64 val, gpsz, npg, pa;
2795         unsigned int psz = SZ_64K;
2796         unsigned int np, epp, esz;
2797         struct page *page;
2798
2799         if (!gic_rdists->has_rvpeid)
2800                 return 0;
2801
2802         /*
2803          * if VPENDBASER.Valid is set, disable any previously programmed
2804          * VPE by setting PendingLast while clearing Valid. This has the
2805          * effect of making sure no doorbell will be generated and we can
2806          * then safely clear VPROPBASER.Valid.
2807          */
2808         if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
2809                 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
2810                                       vlpi_base + GICR_VPENDBASER);
2811
2812         /*
2813          * If we can inherit the configuration from another RD, let's do
2814          * so. Otherwise, we have to go through the allocation process. We
2815          * assume that all RDs have the exact same requirements, as
2816          * nothing will work otherwise.
2817          */
2818         val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
2819         if (val & GICR_VPROPBASER_4_1_VALID)
2820                 goto out;
2821
2822         gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
2823         if (!gic_data_rdist()->vpe_table_mask)
2824                 return -ENOMEM;
2825
2826         val = inherit_vpe_l1_table_from_its();
2827         if (val & GICR_VPROPBASER_4_1_VALID)
2828                 goto out;
2829
2830         /* First probe the page size */
2831         val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
2832         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2833         val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
2834         gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2835         esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
2836
2837         switch (gpsz) {
2838         default:
2839                 gpsz = GIC_PAGE_SIZE_4K;
2840                 fallthrough;
2841         case GIC_PAGE_SIZE_4K:
2842                 psz = SZ_4K;
2843                 break;
2844         case GIC_PAGE_SIZE_16K:
2845                 psz = SZ_16K;
2846                 break;
2847         case GIC_PAGE_SIZE_64K:
2848                 psz = SZ_64K;
2849                 break;
2850         }
2851
2852         /*
2853          * Start populating the register from scratch, including RO fields
2854          * (which we want to print in debug cases...)
2855          */
2856         val = 0;
2857         val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
2858         val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
2859
2860         /* How many entries per GIC page? */
2861         esz++;
2862         epp = psz / (esz * SZ_8);
2863
2864         /*
2865          * If we need more than just a single L1 page, flag the table
2866          * as indirect and compute the number of required L1 pages.
2867          */
2868         if (epp < ITS_MAX_VPEID) {
2869                 int nl2;
2870
2871                 val |= GICR_VPROPBASER_4_1_INDIRECT;
2872
2873                 /* Number of L2 pages required to cover the VPEID space */
2874                 nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
2875
2876                 /* Number of L1 pages to point to the L2 pages */
2877                 npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
2878         } else {
2879                 npg = 1;
2880         }
2881
2882         val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
2883
2884         /* Right, that's the number of CPU pages we need for L1 */
2885         np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
2886
2887         pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
2888                  np, npg, psz, epp, esz);
2889         page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
2890         if (!page)
2891                 return -ENOMEM;
2892
2893         gic_data_rdist()->vpe_l1_base = page_address(page);
2894         pa = virt_to_phys(page_address(page));
2895         WARN_ON(!IS_ALIGNED(pa, psz));
2896
2897         val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
2898         val |= GICR_VPROPBASER_RaWb;
2899         val |= GICR_VPROPBASER_InnerShareable;
2900         val |= GICR_VPROPBASER_4_1_Z;
2901         val |= GICR_VPROPBASER_4_1_VALID;
2902
2903 out:
2904         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2905         cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
2906
2907         pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
2908                  smp_processor_id(), val,
2909                  cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
2910
2911         return 0;
2912 }
2913
2914 static int its_alloc_collections(struct its_node *its)
2915 {
2916         int i;
2917
2918         its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
2919                                    GFP_KERNEL);
2920         if (!its->collections)
2921                 return -ENOMEM;
2922
2923         for (i = 0; i < nr_cpu_ids; i++)
2924                 its->collections[i].target_address = ~0ULL;
2925
2926         return 0;
2927 }
2928
2929 static struct page *its_allocate_pending_table(gfp_t gfp_flags)
2930 {
2931         struct page *pend_page;
2932
2933         pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
2934                                 get_order(LPI_PENDBASE_SZ));
2935         if (!pend_page)
2936                 return NULL;
2937
2938         /* Make sure the GIC will observe the zero-ed page */
2939         gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
2940
2941         return pend_page;
2942 }
2943
2944 static void its_free_pending_table(struct page *pt)
2945 {
2946         free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
2947 }
2948
2949 /*
2950  * Booting with kdump and LPIs enabled is generally fine. Any other
2951  * case is wrong in the absence of firmware/EFI support.
2952  */
2953 static bool enabled_lpis_allowed(void)
2954 {
2955         phys_addr_t addr;
2956         u64 val;
2957
2958         /* Check whether the property table is in a reserved region */
2959         val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2960         addr = val & GENMASK_ULL(51, 12);
2961
2962         return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
2963 }
2964
2965 static int __init allocate_lpi_tables(void)
2966 {
2967         u64 val;
2968         int err, cpu;
2969
2970         /*
2971          * If LPIs are enabled while we run this from the boot CPU,
2972          * flag the RD tables as pre-allocated if the stars do align.
2973          */
2974         val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
2975         if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
2976                 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
2977                                       RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
2978                 pr_info("GICv3: Using preallocated redistributor tables\n");
2979         }
2980
2981         err = its_setup_lpi_prop_table();
2982         if (err)
2983                 return err;
2984
2985         /*
2986          * We allocate all the pending tables anyway, as we may have a
2987          * mix of RDs that have had LPIs enabled, and some that
2988          * don't. We'll free the unused ones as each CPU comes online.
2989          */
2990         for_each_possible_cpu(cpu) {
2991                 struct page *pend_page;
2992
2993                 pend_page = its_allocate_pending_table(GFP_NOWAIT);
2994                 if (!pend_page) {
2995                         pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
2996                         return -ENOMEM;
2997                 }
2998
2999                 gic_data_rdist_cpu(cpu)->pend_page = pend_page;
3000         }
3001
3002         return 0;
3003 }
3004
3005 static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
3006 {
3007         u32 count = 1000000;    /* 1s! */
3008         bool clean;
3009         u64 val;
3010
3011         val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3012         val &= ~GICR_VPENDBASER_Valid;
3013         val &= ~clr;
3014         val |= set;
3015         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3016
3017         do {
3018                 val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3019                 clean = !(val & GICR_VPENDBASER_Dirty);
3020                 if (!clean) {
3021                         count--;
3022                         cpu_relax();
3023                         udelay(1);
3024                 }
3025         } while (!clean && count);
3026
3027         if (unlikely(val & GICR_VPENDBASER_Dirty)) {
3028                 pr_err_ratelimited("ITS virtual pending table not cleaning\n");
3029                 val |= GICR_VPENDBASER_PendingLast;
3030         }
3031
3032         return val;
3033 }
3034
3035 static void its_cpu_init_lpis(void)
3036 {
3037         void __iomem *rbase = gic_data_rdist_rd_base();
3038         struct page *pend_page;
3039         phys_addr_t paddr;
3040         u64 val, tmp;
3041
3042         if (gic_data_rdist()->lpi_enabled)
3043                 return;
3044
3045         val = readl_relaxed(rbase + GICR_CTLR);
3046         if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
3047             (val & GICR_CTLR_ENABLE_LPIS)) {
3048                 /*
3049                  * Check that we get the same property table on all
3050                  * RDs. If we don't, this is hopeless.
3051                  */
3052                 paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
3053                 paddr &= GENMASK_ULL(51, 12);
3054                 if (WARN_ON(gic_rdists->prop_table_pa != paddr))
3055                         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3056
3057                 paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3058                 paddr &= GENMASK_ULL(51, 16);
3059
3060                 WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
3061                 its_free_pending_table(gic_data_rdist()->pend_page);
3062                 gic_data_rdist()->pend_page = NULL;
3063
3064                 goto out;
3065         }
3066
3067         pend_page = gic_data_rdist()->pend_page;
3068         paddr = page_to_phys(pend_page);
3069         WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
3070
3071         /* set PROPBASE */
3072         val = (gic_rdists->prop_table_pa |
3073                GICR_PROPBASER_InnerShareable |
3074                GICR_PROPBASER_RaWaWb |
3075                ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
3076
3077         gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3078         tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
3079
3080         if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
3081                 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
3082                         /*
3083                          * The HW reports non-shareable, we must
3084                          * remove the cacheability attributes as
3085                          * well.
3086                          */
3087                         val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
3088                                  GICR_PROPBASER_CACHEABILITY_MASK);
3089                         val |= GICR_PROPBASER_nC;
3090                         gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3091                 }
3092                 pr_info_once("GIC: using cache flushing for LPI property table\n");
3093                 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
3094         }
3095
3096         /* set PENDBASE */
3097         val = (page_to_phys(pend_page) |
3098                GICR_PENDBASER_InnerShareable |
3099                GICR_PENDBASER_RaWaWb);
3100
3101         gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3102         tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3103
3104         if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
3105                 /*
3106                  * The HW reports non-shareable, we must remove the
3107                  * cacheability attributes as well.
3108                  */
3109                 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
3110                          GICR_PENDBASER_CACHEABILITY_MASK);
3111                 val |= GICR_PENDBASER_nC;
3112                 gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3113         }
3114
3115         /* Enable LPIs */
3116         val = readl_relaxed(rbase + GICR_CTLR);
3117         val |= GICR_CTLR_ENABLE_LPIS;
3118         writel_relaxed(val, rbase + GICR_CTLR);
3119
3120         if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
3121                 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3122
3123                 /*
3124                  * It's possible for CPU to receive VLPIs before it is
3125                  * scheduled as a vPE, especially for the first CPU, and the
3126                  * VLPI with INTID larger than 2^(IDbits+1) will be considered
3127                  * as out of range and dropped by GIC.
3128                  * So we initialize IDbits to known value to avoid VLPI drop.
3129                  */
3130                 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3131                 pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
3132                         smp_processor_id(), val);
3133                 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3134
3135                 /*
3136                  * Also clear Valid bit of GICR_VPENDBASER, in case some
3137                  * ancient programming gets left in and has possibility of
3138                  * corrupting memory.
3139                  */
3140                 val = its_clear_vpend_valid(vlpi_base, 0, 0);
3141         }
3142
3143         if (allocate_vpe_l1_table()) {
3144                 /*
3145                  * If the allocation has failed, we're in massive trouble.
3146                  * Disable direct injection, and pray that no VM was
3147                  * already running...
3148                  */
3149                 gic_rdists->has_rvpeid = false;
3150                 gic_rdists->has_vlpis = false;
3151         }
3152
3153         /* Make sure the GIC has seen the above */
3154         dsb(sy);
3155 out:
3156         gic_data_rdist()->lpi_enabled = true;
3157         pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
3158                 smp_processor_id(),
3159                 gic_data_rdist()->pend_page ? "allocated" : "reserved",
3160                 &paddr);
3161 }
3162
3163 static void its_cpu_init_collection(struct its_node *its)
3164 {
3165         int cpu = smp_processor_id();
3166         u64 target;
3167
3168         /* avoid cross node collections and its mapping */
3169         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
3170                 struct device_node *cpu_node;
3171
3172                 cpu_node = of_get_cpu_node(cpu, NULL);
3173                 if (its->numa_node != NUMA_NO_NODE &&
3174                         its->numa_node != of_node_to_nid(cpu_node))
3175                         return;
3176         }
3177
3178         /*
3179          * We now have to bind each collection to its target
3180          * redistributor.
3181          */
3182         if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
3183                 /*
3184                  * This ITS wants the physical address of the
3185                  * redistributor.
3186                  */
3187                 target = gic_data_rdist()->phys_base;
3188         } else {
3189                 /* This ITS wants a linear CPU number. */
3190                 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
3191                 target = GICR_TYPER_CPU_NUMBER(target) << 16;
3192         }
3193
3194         /* Perform collection mapping */
3195         its->collections[cpu].target_address = target;
3196         its->collections[cpu].col_id = cpu;
3197
3198         its_send_mapc(its, &its->collections[cpu], 1);
3199         its_send_invall(its, &its->collections[cpu]);
3200 }
3201
3202 static void its_cpu_init_collections(void)
3203 {
3204         struct its_node *its;
3205
3206         raw_spin_lock(&its_lock);
3207
3208         list_for_each_entry(its, &its_nodes, entry)
3209                 its_cpu_init_collection(its);
3210
3211         raw_spin_unlock(&its_lock);
3212 }
3213
3214 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
3215 {
3216         struct its_device *its_dev = NULL, *tmp;
3217         unsigned long flags;
3218
3219         raw_spin_lock_irqsave(&its->lock, flags);
3220
3221         list_for_each_entry(tmp, &its->its_device_list, entry) {
3222                 if (tmp->device_id == dev_id) {
3223                         its_dev = tmp;
3224                         break;
3225                 }
3226         }
3227
3228         raw_spin_unlock_irqrestore(&its->lock, flags);
3229
3230         return its_dev;
3231 }
3232
3233 static struct its_baser *its_get_baser(struct its_node *its, u32 type)
3234 {
3235         int i;
3236
3237         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3238                 if (GITS_BASER_TYPE(its->tables[i].val) == type)
3239                         return &its->tables[i];
3240         }
3241
3242         return NULL;
3243 }
3244
3245 static bool its_alloc_table_entry(struct its_node *its,
3246                                   struct its_baser *baser, u32 id)
3247 {
3248         struct page *page;
3249         u32 esz, idx;
3250         __le64 *table;
3251
3252         /* Don't allow device id that exceeds single, flat table limit */
3253         esz = GITS_BASER_ENTRY_SIZE(baser->val);
3254         if (!(baser->val & GITS_BASER_INDIRECT))
3255                 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
3256
3257         /* Compute 1st level table index & check if that exceeds table limit */
3258         idx = id >> ilog2(baser->psz / esz);
3259         if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
3260                 return false;
3261
3262         table = baser->base;
3263
3264         /* Allocate memory for 2nd level table */
3265         if (!table[idx]) {
3266                 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
3267                                         get_order(baser->psz));
3268                 if (!page)
3269                         return false;
3270
3271                 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
3272                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3273                         gic_flush_dcache_to_poc(page_address(page), baser->psz);
3274
3275                 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
3276
3277                 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
3278                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3279                         gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
3280
3281                 /* Ensure updated table contents are visible to ITS hardware */
3282                 dsb(sy);
3283         }
3284
3285         return true;
3286 }
3287
3288 static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
3289 {
3290         struct its_baser *baser;
3291
3292         baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
3293
3294         /* Don't allow device id that exceeds ITS hardware limit */
3295         if (!baser)
3296                 return (ilog2(dev_id) < device_ids(its));
3297
3298         return its_alloc_table_entry(its, baser, dev_id);
3299 }
3300
3301 static bool its_alloc_vpe_table(u32 vpe_id)
3302 {
3303         struct its_node *its;
3304         int cpu;
3305
3306         /*
3307          * Make sure the L2 tables are allocated on *all* v4 ITSs. We
3308          * could try and only do it on ITSs corresponding to devices
3309          * that have interrupts targeted at this VPE, but the
3310          * complexity becomes crazy (and you have tons of memory
3311          * anyway, right?).
3312          */
3313         list_for_each_entry(its, &its_nodes, entry) {
3314                 struct its_baser *baser;
3315
3316                 if (!is_v4(its))
3317                         continue;
3318
3319                 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
3320                 if (!baser)
3321                         return false;
3322
3323                 if (!its_alloc_table_entry(its, baser, vpe_id))
3324                         return false;
3325         }
3326
3327         /* Non v4.1? No need to iterate RDs and go back early. */
3328         if (!gic_rdists->has_rvpeid)
3329                 return true;
3330
3331         /*
3332          * Make sure the L2 tables are allocated for all copies of
3333          * the L1 table on *all* v4.1 RDs.
3334          */
3335         for_each_possible_cpu(cpu) {
3336                 if (!allocate_vpe_l2_table(cpu, vpe_id))
3337                         return false;
3338         }
3339
3340         return true;
3341 }
3342
3343 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
3344                                             int nvecs, bool alloc_lpis)
3345 {
3346         struct its_device *dev;
3347         unsigned long *lpi_map = NULL;
3348         unsigned long flags;
3349         u16 *col_map = NULL;
3350         void *itt;
3351         int lpi_base;
3352         int nr_lpis;
3353         int nr_ites;
3354         int sz;
3355
3356         if (!its_alloc_device_table(its, dev_id))
3357                 return NULL;
3358
3359         if (WARN_ON(!is_power_of_2(nvecs)))
3360                 nvecs = roundup_pow_of_two(nvecs);
3361
3362         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
3363         /*
3364          * Even if the device wants a single LPI, the ITT must be
3365          * sized as a power of two (and you need at least one bit...).
3366          */
3367         nr_ites = max(2, nvecs);
3368         sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
3369         sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
3370         itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
3371         if (alloc_lpis) {
3372                 lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
3373                 if (lpi_map)
3374                         col_map = kcalloc(nr_lpis, sizeof(*col_map),
3375                                           GFP_KERNEL);
3376         } else {
3377                 col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
3378                 nr_lpis = 0;
3379                 lpi_base = 0;
3380         }
3381
3382         if (!dev || !itt ||  !col_map || (!lpi_map && alloc_lpis)) {
3383                 kfree(dev);
3384                 kfree(itt);
3385                 kfree(lpi_map);
3386                 kfree(col_map);
3387                 return NULL;
3388         }
3389
3390         gic_flush_dcache_to_poc(itt, sz);
3391
3392         dev->its = its;
3393         dev->itt = itt;
3394         dev->nr_ites = nr_ites;
3395         dev->event_map.lpi_map = lpi_map;
3396         dev->event_map.col_map = col_map;
3397         dev->event_map.lpi_base = lpi_base;
3398         dev->event_map.nr_lpis = nr_lpis;
3399         raw_spin_lock_init(&dev->event_map.vlpi_lock);
3400         dev->device_id = dev_id;
3401         INIT_LIST_HEAD(&dev->entry);
3402
3403         raw_spin_lock_irqsave(&its->lock, flags);
3404         list_add(&dev->entry, &its->its_device_list);
3405         raw_spin_unlock_irqrestore(&its->lock, flags);
3406
3407         /* Map device to its ITT */
3408         its_send_mapd(dev, 1);
3409
3410         return dev;
3411 }
3412
3413 static void its_free_device(struct its_device *its_dev)
3414 {
3415         unsigned long flags;
3416
3417         raw_spin_lock_irqsave(&its_dev->its->lock, flags);
3418         list_del(&its_dev->entry);
3419         raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
3420         kfree(its_dev->event_map.col_map);
3421         kfree(its_dev->itt);
3422         kfree(its_dev);
3423 }
3424
3425 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
3426 {
3427         int idx;
3428
3429         /* Find a free LPI region in lpi_map and allocate them. */
3430         idx = bitmap_find_free_region(dev->event_map.lpi_map,
3431                                       dev->event_map.nr_lpis,
3432                                       get_count_order(nvecs));
3433         if (idx < 0)
3434                 return -ENOSPC;
3435
3436         *hwirq = dev->event_map.lpi_base + idx;
3437
3438         return 0;
3439 }
3440
3441 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
3442                            int nvec, msi_alloc_info_t *info)
3443 {
3444         struct its_node *its;
3445         struct its_device *its_dev;
3446         struct msi_domain_info *msi_info;
3447         u32 dev_id;
3448         int err = 0;
3449
3450         /*
3451          * We ignore "dev" entirely, and rely on the dev_id that has
3452          * been passed via the scratchpad. This limits this domain's
3453          * usefulness to upper layers that definitely know that they
3454          * are built on top of the ITS.
3455          */
3456         dev_id = info->scratchpad[0].ul;
3457
3458         msi_info = msi_get_domain_info(domain);
3459         its = msi_info->data;
3460
3461         if (!gic_rdists->has_direct_lpi &&
3462             vpe_proxy.dev &&
3463             vpe_proxy.dev->its == its &&
3464             dev_id == vpe_proxy.dev->device_id) {
3465                 /* Bad luck. Get yourself a better implementation */
3466                 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
3467                           dev_id);
3468                 return -EINVAL;
3469         }
3470
3471         mutex_lock(&its->dev_alloc_lock);
3472         its_dev = its_find_device(its, dev_id);
3473         if (its_dev) {
3474                 /*
3475                  * We already have seen this ID, probably through
3476                  * another alias (PCI bridge of some sort). No need to
3477                  * create the device.
3478                  */
3479                 its_dev->shared = true;
3480                 pr_debug("Reusing ITT for devID %x\n", dev_id);
3481                 goto out;
3482         }
3483
3484         its_dev = its_create_device(its, dev_id, nvec, true);
3485         if (!its_dev) {
3486                 err = -ENOMEM;
3487                 goto out;
3488         }
3489
3490         if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE)
3491                 its_dev->shared = true;
3492
3493         pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
3494 out:
3495         mutex_unlock(&its->dev_alloc_lock);
3496         info->scratchpad[0].ptr = its_dev;
3497         return err;
3498 }
3499
3500 static struct msi_domain_ops its_msi_domain_ops = {
3501         .msi_prepare    = its_msi_prepare,
3502 };
3503
3504 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
3505                                     unsigned int virq,
3506                                     irq_hw_number_t hwirq)
3507 {
3508         struct irq_fwspec fwspec;
3509
3510         if (irq_domain_get_of_node(domain->parent)) {
3511                 fwspec.fwnode = domain->parent->fwnode;
3512                 fwspec.param_count = 3;
3513                 fwspec.param[0] = GIC_IRQ_TYPE_LPI;
3514                 fwspec.param[1] = hwirq;
3515                 fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
3516         } else if (is_fwnode_irqchip(domain->parent->fwnode)) {
3517                 fwspec.fwnode = domain->parent->fwnode;
3518                 fwspec.param_count = 2;
3519                 fwspec.param[0] = hwirq;
3520                 fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
3521         } else {
3522                 return -EINVAL;
3523         }
3524
3525         return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
3526 }
3527
3528 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3529                                 unsigned int nr_irqs, void *args)
3530 {
3531         msi_alloc_info_t *info = args;
3532         struct its_device *its_dev = info->scratchpad[0].ptr;
3533         struct its_node *its = its_dev->its;
3534         struct irq_data *irqd;
3535         irq_hw_number_t hwirq;
3536         int err;
3537         int i;
3538
3539         err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
3540         if (err)
3541                 return err;
3542
3543         err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
3544         if (err)
3545                 return err;
3546
3547         for (i = 0; i < nr_irqs; i++) {
3548                 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
3549                 if (err)
3550                         return err;
3551
3552                 irq_domain_set_hwirq_and_chip(domain, virq + i,
3553                                               hwirq + i, &its_irq_chip, its_dev);
3554                 irqd = irq_get_irq_data(virq + i);
3555                 irqd_set_single_target(irqd);
3556                 irqd_set_affinity_on_activate(irqd);
3557                 pr_debug("ID:%d pID:%d vID:%d\n",
3558                          (int)(hwirq + i - its_dev->event_map.lpi_base),
3559                          (int)(hwirq + i), virq + i);
3560         }
3561
3562         return 0;
3563 }
3564
3565 static int its_irq_domain_activate(struct irq_domain *domain,
3566                                    struct irq_data *d, bool reserve)
3567 {
3568         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3569         u32 event = its_get_event_id(d);
3570         int cpu;
3571
3572         cpu = its_select_cpu(d, cpu_online_mask);
3573         if (cpu < 0 || cpu >= nr_cpu_ids)
3574                 return -EINVAL;
3575
3576         its_inc_lpi_count(d, cpu);
3577         its_dev->event_map.col_map[event] = cpu;
3578         irq_data_update_effective_affinity(d, cpumask_of(cpu));
3579
3580         /* Map the GIC IRQ and event to the device */
3581         its_send_mapti(its_dev, d->hwirq, event);
3582         return 0;
3583 }
3584
3585 static void its_irq_domain_deactivate(struct irq_domain *domain,
3586                                       struct irq_data *d)
3587 {
3588         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3589         u32 event = its_get_event_id(d);
3590
3591         its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
3592         /* Stop the delivery of interrupts */
3593         its_send_discard(its_dev, event);
3594 }
3595
3596 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
3597                                 unsigned int nr_irqs)
3598 {
3599         struct irq_data *d = irq_domain_get_irq_data(domain, virq);
3600         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3601         struct its_node *its = its_dev->its;
3602         int i;
3603
3604         bitmap_release_region(its_dev->event_map.lpi_map,
3605                               its_get_event_id(irq_domain_get_irq_data(domain, virq)),
3606                               get_count_order(nr_irqs));
3607
3608         for (i = 0; i < nr_irqs; i++) {
3609                 struct irq_data *data = irq_domain_get_irq_data(domain,
3610                                                                 virq + i);
3611                 /* Nuke the entry in the domain */
3612                 irq_domain_reset_irq_data(data);
3613         }
3614
3615         mutex_lock(&its->dev_alloc_lock);
3616
3617         /*
3618          * If all interrupts have been freed, start mopping the
3619          * floor. This is conditioned on the device not being shared.
3620          */
3621         if (!its_dev->shared &&
3622             bitmap_empty(its_dev->event_map.lpi_map,
3623                          its_dev->event_map.nr_lpis)) {
3624                 its_lpi_free(its_dev->event_map.lpi_map,
3625                              its_dev->event_map.lpi_base,
3626                              its_dev->event_map.nr_lpis);
3627
3628                 /* Unmap device/itt */
3629                 its_send_mapd(its_dev, 0);
3630                 its_free_device(its_dev);
3631         }
3632
3633         mutex_unlock(&its->dev_alloc_lock);
3634
3635         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
3636 }
3637
3638 static const struct irq_domain_ops its_domain_ops = {
3639         .alloc                  = its_irq_domain_alloc,
3640         .free                   = its_irq_domain_free,
3641         .activate               = its_irq_domain_activate,
3642         .deactivate             = its_irq_domain_deactivate,
3643 };
3644
3645 /*
3646  * This is insane.
3647  *
3648  * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
3649  * likely), the only way to perform an invalidate is to use a fake
3650  * device to issue an INV command, implying that the LPI has first
3651  * been mapped to some event on that device. Since this is not exactly
3652  * cheap, we try to keep that mapping around as long as possible, and
3653  * only issue an UNMAP if we're short on available slots.
3654  *
3655  * Broken by design(tm).
3656  *
3657  * GICv4.1, on the other hand, mandates that we're able to invalidate
3658  * by writing to a MMIO register. It doesn't implement the whole of
3659  * DirectLPI, but that's good enough. And most of the time, we don't
3660  * even have to invalidate anything, as the redistributor can be told
3661  * whether to generate a doorbell or not (we thus leave it enabled,
3662  * always).
3663  */
3664 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
3665 {
3666         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3667         if (gic_rdists->has_rvpeid)
3668                 return;
3669
3670         /* Already unmapped? */
3671         if (vpe->vpe_proxy_event == -1)
3672                 return;
3673
3674         its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
3675         vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
3676
3677         /*
3678          * We don't track empty slots at all, so let's move the
3679          * next_victim pointer if we can quickly reuse that slot
3680          * instead of nuking an existing entry. Not clear that this is
3681          * always a win though, and this might just generate a ripple
3682          * effect... Let's just hope VPEs don't migrate too often.
3683          */
3684         if (vpe_proxy.vpes[vpe_proxy.next_victim])
3685                 vpe_proxy.next_victim = vpe->vpe_proxy_event;
3686
3687         vpe->vpe_proxy_event = -1;
3688 }
3689
3690 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
3691 {
3692         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3693         if (gic_rdists->has_rvpeid)
3694                 return;
3695
3696         if (!gic_rdists->has_direct_lpi) {
3697                 unsigned long flags;
3698
3699                 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3700                 its_vpe_db_proxy_unmap_locked(vpe);
3701                 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3702         }
3703 }
3704
3705 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
3706 {
3707         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3708         if (gic_rdists->has_rvpeid)
3709                 return;
3710
3711         /* Already mapped? */
3712         if (vpe->vpe_proxy_event != -1)
3713                 return;
3714
3715         /* This slot was already allocated. Kick the other VPE out. */
3716         if (vpe_proxy.vpes[vpe_proxy.next_victim])
3717                 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
3718
3719         /* Map the new VPE instead */
3720         vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
3721         vpe->vpe_proxy_event = vpe_proxy.next_victim;
3722         vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
3723
3724         vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
3725         its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
3726 }
3727
3728 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3729 {
3730         unsigned long flags;
3731         struct its_collection *target_col;
3732
3733         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3734         if (gic_rdists->has_rvpeid)
3735                 return;
3736
3737         if (gic_rdists->has_direct_lpi) {
3738                 void __iomem *rdbase;
3739
3740                 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
3741                 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3742                 wait_for_syncr(rdbase);
3743
3744                 return;
3745         }
3746
3747         raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3748
3749         its_vpe_db_proxy_map_locked(vpe);
3750
3751         target_col = &vpe_proxy.dev->its->collections[to];
3752         its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3753         vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3754
3755         raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3756 }
3757
3758 static int its_vpe_set_affinity(struct irq_data *d,
3759                                 const struct cpumask *mask_val,
3760                                 bool force)
3761 {
3762         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3763         int from, cpu = cpumask_first(mask_val);
3764         unsigned long flags;
3765
3766         /*
3767          * Changing affinity is mega expensive, so let's be as lazy as
3768          * we can and only do it if we really have to. Also, if mapped
3769          * into the proxy device, we need to move the doorbell
3770          * interrupt to its new location.
3771          *
3772          * Another thing is that changing the affinity of a vPE affects
3773          * *other interrupts* such as all the vLPIs that are routed to
3774          * this vPE. This means that the irq_desc lock is not enough to
3775          * protect us, and that we must ensure nobody samples vpe->col_idx
3776          * during the update, hence the lock below which must also be
3777          * taken on any vLPI handling path that evaluates vpe->col_idx.
3778          */
3779         from = vpe_to_cpuid_lock(vpe, &flags);
3780         if (from == cpu)
3781                 goto out;
3782
3783         vpe->col_idx = cpu;
3784
3785         /*
3786          * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD
3787          * is sharing its VPE table with the current one.
3788          */
3789         if (gic_data_rdist_cpu(cpu)->vpe_table_mask &&
3790             cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask))
3791                 goto out;
3792
3793         its_send_vmovp(vpe);
3794         its_vpe_db_proxy_move(vpe, from, cpu);
3795
3796 out:
3797         irq_data_update_effective_affinity(d, cpumask_of(cpu));
3798         vpe_to_cpuid_unlock(vpe, flags);
3799
3800         return IRQ_SET_MASK_OK_DONE;
3801 }
3802
3803 static void its_wait_vpt_parse_complete(void)
3804 {
3805         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3806         u64 val;
3807
3808         if (!gic_rdists->has_vpend_valid_dirty)
3809                 return;
3810
3811         WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
3812                                                        val,
3813                                                        !(val & GICR_VPENDBASER_Dirty),
3814                                                        1, 500));
3815 }
3816
3817 static void its_vpe_schedule(struct its_vpe *vpe)
3818 {
3819         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3820         u64 val;
3821
3822         /* Schedule the VPE */
3823         val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
3824                 GENMASK_ULL(51, 12);
3825         val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3826         val |= GICR_VPROPBASER_RaWb;
3827         val |= GICR_VPROPBASER_InnerShareable;
3828         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3829
3830         val  = virt_to_phys(page_address(vpe->vpt_page)) &
3831                 GENMASK_ULL(51, 16);
3832         val |= GICR_VPENDBASER_RaWaWb;
3833         val |= GICR_VPENDBASER_InnerShareable;
3834         /*
3835          * There is no good way of finding out if the pending table is
3836          * empty as we can race against the doorbell interrupt very
3837          * easily. So in the end, vpe->pending_last is only an
3838          * indication that the vcpu has something pending, not one
3839          * that the pending table is empty. A good implementation
3840          * would be able to read its coarse map pretty quickly anyway,
3841          * making this a tolerable issue.
3842          */
3843         val |= GICR_VPENDBASER_PendingLast;
3844         val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
3845         val |= GICR_VPENDBASER_Valid;
3846         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3847 }
3848
3849 static void its_vpe_deschedule(struct its_vpe *vpe)
3850 {
3851         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3852         u64 val;
3853
3854         val = its_clear_vpend_valid(vlpi_base, 0, 0);
3855
3856         vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
3857         vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
3858 }
3859
3860 static void its_vpe_invall(struct its_vpe *vpe)
3861 {
3862         struct its_node *its;
3863
3864         list_for_each_entry(its, &its_nodes, entry) {
3865                 if (!is_v4(its))
3866                         continue;
3867
3868                 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
3869                         continue;
3870
3871                 /*
3872                  * Sending a VINVALL to a single ITS is enough, as all
3873                  * we need is to reach the redistributors.
3874                  */
3875                 its_send_vinvall(its, vpe);
3876                 return;
3877         }
3878 }
3879
3880 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
3881 {
3882         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3883         struct its_cmd_info *info = vcpu_info;
3884
3885         switch (info->cmd_type) {
3886         case SCHEDULE_VPE:
3887                 its_vpe_schedule(vpe);
3888                 return 0;
3889
3890         case DESCHEDULE_VPE:
3891                 its_vpe_deschedule(vpe);
3892                 return 0;
3893
3894         case COMMIT_VPE:
3895                 its_wait_vpt_parse_complete();
3896                 return 0;
3897
3898         case INVALL_VPE:
3899                 its_vpe_invall(vpe);
3900                 return 0;
3901
3902         default:
3903                 return -EINVAL;
3904         }
3905 }
3906
3907 static void its_vpe_send_cmd(struct its_vpe *vpe,
3908                              void (*cmd)(struct its_device *, u32))
3909 {
3910         unsigned long flags;
3911
3912         raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3913
3914         its_vpe_db_proxy_map_locked(vpe);
3915         cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
3916
3917         raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3918 }
3919
3920 static void its_vpe_send_inv(struct irq_data *d)
3921 {
3922         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3923
3924         if (gic_rdists->has_direct_lpi) {
3925                 void __iomem *rdbase;
3926
3927                 /* Target the redistributor this VPE is currently known on */
3928                 raw_spin_lock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock);
3929                 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
3930                 gic_write_lpir(d->parent_data->hwirq, rdbase + GICR_INVLPIR);
3931                 wait_for_syncr(rdbase);
3932                 raw_spin_unlock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock);
3933         } else {
3934                 its_vpe_send_cmd(vpe, its_send_inv);
3935         }
3936 }
3937
3938 static void its_vpe_mask_irq(struct irq_data *d)
3939 {
3940         /*
3941          * We need to unmask the LPI, which is described by the parent
3942          * irq_data. Instead of calling into the parent (which won't
3943          * exactly do the right thing, let's simply use the
3944          * parent_data pointer. Yes, I'm naughty.
3945          */
3946         lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
3947         its_vpe_send_inv(d);
3948 }
3949
3950 static void its_vpe_unmask_irq(struct irq_data *d)
3951 {
3952         /* Same hack as above... */
3953         lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
3954         its_vpe_send_inv(d);
3955 }
3956
3957 static int its_vpe_set_irqchip_state(struct irq_data *d,
3958                                      enum irqchip_irq_state which,
3959                                      bool state)
3960 {
3961         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3962
3963         if (which != IRQCHIP_STATE_PENDING)
3964                 return -EINVAL;
3965
3966         if (gic_rdists->has_direct_lpi) {
3967                 void __iomem *rdbase;
3968
3969                 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
3970                 if (state) {
3971                         gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
3972                 } else {
3973                         gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3974                         wait_for_syncr(rdbase);
3975                 }
3976         } else {
3977                 if (state)
3978                         its_vpe_send_cmd(vpe, its_send_int);
3979                 else
3980                         its_vpe_send_cmd(vpe, its_send_clear);
3981         }
3982
3983         return 0;
3984 }
3985
3986 static int its_vpe_retrigger(struct irq_data *d)
3987 {
3988         return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
3989 }
3990
3991 static struct irq_chip its_vpe_irq_chip = {
3992         .name                   = "GICv4-vpe",
3993         .irq_mask               = its_vpe_mask_irq,
3994         .irq_unmask             = its_vpe_unmask_irq,
3995         .irq_eoi                = irq_chip_eoi_parent,
3996         .irq_set_affinity       = its_vpe_set_affinity,
3997         .irq_retrigger          = its_vpe_retrigger,
3998         .irq_set_irqchip_state  = its_vpe_set_irqchip_state,
3999         .irq_set_vcpu_affinity  = its_vpe_set_vcpu_affinity,
4000 };
4001
4002 static struct its_node *find_4_1_its(void)
4003 {
4004         static struct its_node *its = NULL;
4005
4006         if (!its) {
4007                 list_for_each_entry(its, &its_nodes, entry) {
4008                         if (is_v4_1(its))
4009                                 return its;
4010                 }
4011
4012                 /* Oops? */
4013                 its = NULL;
4014         }
4015
4016         return its;
4017 }
4018
4019 static void its_vpe_4_1_send_inv(struct irq_data *d)
4020 {
4021         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4022         struct its_node *its;
4023
4024         /*
4025          * GICv4.1 wants doorbells to be invalidated using the
4026          * INVDB command in order to be broadcast to all RDs. Send
4027          * it to the first valid ITS, and let the HW do its magic.
4028          */
4029         its = find_4_1_its();
4030         if (its)
4031                 its_send_invdb(its, vpe);
4032 }
4033
4034 static void its_vpe_4_1_mask_irq(struct irq_data *d)
4035 {
4036         lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4037         its_vpe_4_1_send_inv(d);
4038 }
4039
4040 static void its_vpe_4_1_unmask_irq(struct irq_data *d)
4041 {
4042         lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4043         its_vpe_4_1_send_inv(d);
4044 }
4045
4046 static void its_vpe_4_1_schedule(struct its_vpe *vpe,
4047                                  struct its_cmd_info *info)
4048 {
4049         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4050         u64 val = 0;
4051
4052         /* Schedule the VPE */
4053         val |= GICR_VPENDBASER_Valid;
4054         val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
4055         val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
4056         val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
4057
4058         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
4059 }
4060
4061 static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4062                                    struct its_cmd_info *info)
4063 {
4064         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4065         u64 val;
4066
4067         if (info->req_db) {
4068                 unsigned long flags;
4069
4070                 /*
4071                  * vPE is going to block: make the vPE non-resident with
4072                  * PendingLast clear and DB set. The GIC guarantees that if
4073                  * we read-back PendingLast clear, then a doorbell will be
4074                  * delivered when an interrupt comes.
4075                  *
4076                  * Note the locking to deal with the concurrent update of
4077                  * pending_last from the doorbell interrupt handler that can
4078                  * run concurrently.
4079                  */
4080                 raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
4081                 val = its_clear_vpend_valid(vlpi_base,
4082                                             GICR_VPENDBASER_PendingLast,
4083                                             GICR_VPENDBASER_4_1_DB);
4084                 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4085                 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
4086         } else {
4087                 /*
4088                  * We're not blocking, so just make the vPE non-resident
4089                  * with PendingLast set, indicating that we'll be back.
4090                  */
4091                 val = its_clear_vpend_valid(vlpi_base,
4092                                             0,
4093                                             GICR_VPENDBASER_PendingLast);
4094                 vpe->pending_last = true;
4095         }
4096 }
4097
4098 static void its_vpe_4_1_invall(struct its_vpe *vpe)
4099 {
4100         void __iomem *rdbase;
4101         unsigned long flags;
4102         u64 val;
4103         int cpu;
4104
4105         val  = GICR_INVALLR_V;
4106         val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
4107
4108         /* Target the redistributor this vPE is currently known on */
4109         cpu = vpe_to_cpuid_lock(vpe, &flags);
4110         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4111         rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
4112         gic_write_lpir(val, rdbase + GICR_INVALLR);
4113
4114         wait_for_syncr(rdbase);
4115         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4116         vpe_to_cpuid_unlock(vpe, flags);
4117 }
4118
4119 static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4120 {
4121         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4122         struct its_cmd_info *info = vcpu_info;
4123
4124         switch (info->cmd_type) {
4125         case SCHEDULE_VPE:
4126                 its_vpe_4_1_schedule(vpe, info);
4127                 return 0;
4128
4129         case DESCHEDULE_VPE:
4130                 its_vpe_4_1_deschedule(vpe, info);
4131                 return 0;
4132
4133         case COMMIT_VPE:
4134                 its_wait_vpt_parse_complete();
4135                 return 0;
4136
4137         case INVALL_VPE:
4138                 its_vpe_4_1_invall(vpe);
4139                 return 0;
4140
4141         default:
4142                 return -EINVAL;
4143         }
4144 }
4145
4146 static struct irq_chip its_vpe_4_1_irq_chip = {
4147         .name                   = "GICv4.1-vpe",
4148         .irq_mask               = its_vpe_4_1_mask_irq,
4149         .irq_unmask             = its_vpe_4_1_unmask_irq,
4150         .irq_eoi                = irq_chip_eoi_parent,
4151         .irq_set_affinity       = its_vpe_set_affinity,
4152         .irq_set_vcpu_affinity  = its_vpe_4_1_set_vcpu_affinity,
4153 };
4154
4155 static void its_configure_sgi(struct irq_data *d, bool clear)
4156 {
4157         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4158         struct its_cmd_desc desc;
4159
4160         desc.its_vsgi_cmd.vpe = vpe;
4161         desc.its_vsgi_cmd.sgi = d->hwirq;
4162         desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4163         desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4164         desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4165         desc.its_vsgi_cmd.clear = clear;
4166
4167         /*
4168          * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4169          * destination VPE is mapped there. Since we map them eagerly at
4170          * activation time, we're pretty sure the first GICv4.1 ITS will do.
4171          */
4172         its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
4173 }
4174
4175 static void its_sgi_mask_irq(struct irq_data *d)
4176 {
4177         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4178
4179         vpe->sgi_config[d->hwirq].enabled = false;
4180         its_configure_sgi(d, false);
4181 }
4182
4183 static void its_sgi_unmask_irq(struct irq_data *d)
4184 {
4185         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4186
4187         vpe->sgi_config[d->hwirq].enabled = true;
4188         its_configure_sgi(d, false);
4189 }
4190
4191 static int its_sgi_set_affinity(struct irq_data *d,
4192                                 const struct cpumask *mask_val,
4193                                 bool force)
4194 {
4195         /*
4196          * There is no notion of affinity for virtual SGIs, at least
4197          * not on the host (since they can only be targeting a vPE).
4198          * Tell the kernel we've done whatever it asked for.
4199          */
4200         irq_data_update_effective_affinity(d, mask_val);
4201         return IRQ_SET_MASK_OK;
4202 }
4203
4204 static int its_sgi_set_irqchip_state(struct irq_data *d,
4205                                      enum irqchip_irq_state which,
4206                                      bool state)
4207 {
4208         if (which != IRQCHIP_STATE_PENDING)
4209                 return -EINVAL;
4210
4211         if (state) {
4212                 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4213                 struct its_node *its = find_4_1_its();
4214                 u64 val;
4215
4216                 val  = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
4217                 val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
4218                 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
4219         } else {
4220                 its_configure_sgi(d, true);
4221         }
4222
4223         return 0;
4224 }
4225
4226 static int its_sgi_get_irqchip_state(struct irq_data *d,
4227                                      enum irqchip_irq_state which, bool *val)
4228 {
4229         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4230         void __iomem *base;
4231         unsigned long flags;
4232         u32 count = 1000000;    /* 1s! */
4233         u32 status;
4234         int cpu;
4235
4236         if (which != IRQCHIP_STATE_PENDING)
4237                 return -EINVAL;
4238
4239         /*
4240          * Locking galore! We can race against two different events:
4241          *
4242          * - Concurrent vPE affinity change: we must make sure it cannot
4243          *   happen, or we'll talk to the wrong redistributor. This is
4244          *   identical to what happens with vLPIs.
4245          *
4246          * - Concurrent VSGIPENDR access: As it involves accessing two
4247          *   MMIO registers, this must be made atomic one way or another.
4248          */
4249         cpu = vpe_to_cpuid_lock(vpe, &flags);
4250         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4251         base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
4252         writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
4253         do {
4254                 status = readl_relaxed(base + GICR_VSGIPENDR);
4255                 if (!(status & GICR_VSGIPENDR_BUSY))
4256                         goto out;
4257
4258                 count--;
4259                 if (!count) {
4260                         pr_err_ratelimited("Unable to get SGI status\n");
4261                         goto out;
4262                 }
4263                 cpu_relax();
4264                 udelay(1);
4265         } while (count);
4266
4267 out:
4268         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4269         vpe_to_cpuid_unlock(vpe, flags);
4270
4271         if (!count)
4272                 return -ENXIO;
4273
4274         *val = !!(status & (1 << d->hwirq));
4275
4276         return 0;
4277 }
4278
4279 static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4280 {
4281         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4282         struct its_cmd_info *info = vcpu_info;
4283
4284         switch (info->cmd_type) {
4285         case PROP_UPDATE_VSGI:
4286                 vpe->sgi_config[d->hwirq].priority = info->priority;
4287                 vpe->sgi_config[d->hwirq].group = info->group;
4288                 its_configure_sgi(d, false);
4289                 return 0;
4290
4291         default:
4292                 return -EINVAL;
4293         }
4294 }
4295
4296 static struct irq_chip its_sgi_irq_chip = {
4297         .name                   = "GICv4.1-sgi",
4298         .irq_mask               = its_sgi_mask_irq,
4299         .irq_unmask             = its_sgi_unmask_irq,
4300         .irq_set_affinity       = its_sgi_set_affinity,
4301         .irq_set_irqchip_state  = its_sgi_set_irqchip_state,
4302         .irq_get_irqchip_state  = its_sgi_get_irqchip_state,
4303         .irq_set_vcpu_affinity  = its_sgi_set_vcpu_affinity,
4304 };
4305
4306 static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
4307                                     unsigned int virq, unsigned int nr_irqs,
4308                                     void *args)
4309 {
4310         struct its_vpe *vpe = args;
4311         int i;
4312
4313         /* Yes, we do want 16 SGIs */
4314         WARN_ON(nr_irqs != 16);
4315
4316         for (i = 0; i < 16; i++) {
4317                 vpe->sgi_config[i].priority = 0;
4318                 vpe->sgi_config[i].enabled = false;
4319                 vpe->sgi_config[i].group = false;
4320
4321                 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4322                                               &its_sgi_irq_chip, vpe);
4323                 irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
4324         }
4325
4326         return 0;
4327 }
4328
4329 static void its_sgi_irq_domain_free(struct irq_domain *domain,
4330                                     unsigned int virq,
4331                                     unsigned int nr_irqs)
4332 {
4333         /* Nothing to do */
4334 }
4335
4336 static int its_sgi_irq_domain_activate(struct irq_domain *domain,
4337                                        struct irq_data *d, bool reserve)
4338 {
4339         /* Write out the initial SGI configuration */
4340         its_configure_sgi(d, false);
4341         return 0;
4342 }
4343
4344 static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
4345                                           struct irq_data *d)
4346 {
4347         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4348
4349         /*
4350          * The VSGI command is awkward:
4351          *
4352          * - To change the configuration, CLEAR must be set to false,
4353          *   leaving the pending bit unchanged.
4354          * - To clear the pending bit, CLEAR must be set to true, leaving
4355          *   the configuration unchanged.
4356          *
4357          * You just can't do both at once, hence the two commands below.
4358          */
4359         vpe->sgi_config[d->hwirq].enabled = false;
4360         its_configure_sgi(d, false);
4361         its_configure_sgi(d, true);
4362 }
4363
4364 static const struct irq_domain_ops its_sgi_domain_ops = {
4365         .alloc          = its_sgi_irq_domain_alloc,
4366         .free           = its_sgi_irq_domain_free,
4367         .activate       = its_sgi_irq_domain_activate,
4368         .deactivate     = its_sgi_irq_domain_deactivate,
4369 };
4370
4371 static int its_vpe_id_alloc(void)
4372 {
4373         return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
4374 }
4375
4376 static void its_vpe_id_free(u16 id)
4377 {
4378         ida_simple_remove(&its_vpeid_ida, id);
4379 }
4380
4381 static int its_vpe_init(struct its_vpe *vpe)
4382 {
4383         struct page *vpt_page;
4384         int vpe_id;
4385
4386         /* Allocate vpe_id */
4387         vpe_id = its_vpe_id_alloc();
4388         if (vpe_id < 0)
4389                 return vpe_id;
4390
4391         /* Allocate VPT */
4392         vpt_page = its_allocate_pending_table(GFP_KERNEL);
4393         if (!vpt_page) {
4394                 its_vpe_id_free(vpe_id);
4395                 return -ENOMEM;
4396         }
4397
4398         if (!its_alloc_vpe_table(vpe_id)) {
4399                 its_vpe_id_free(vpe_id);
4400                 its_free_pending_table(vpt_page);
4401                 return -ENOMEM;
4402         }
4403
4404         raw_spin_lock_init(&vpe->vpe_lock);
4405         vpe->vpe_id = vpe_id;
4406         vpe->vpt_page = vpt_page;
4407         if (gic_rdists->has_rvpeid)
4408                 atomic_set(&vpe->vmapp_count, 0);
4409         else
4410                 vpe->vpe_proxy_event = -1;
4411
4412         return 0;
4413 }
4414
4415 static void its_vpe_teardown(struct its_vpe *vpe)
4416 {
4417         its_vpe_db_proxy_unmap(vpe);
4418         its_vpe_id_free(vpe->vpe_id);
4419         its_free_pending_table(vpe->vpt_page);
4420 }
4421
4422 static void its_vpe_irq_domain_free(struct irq_domain *domain,
4423                                     unsigned int virq,
4424                                     unsigned int nr_irqs)
4425 {
4426         struct its_vm *vm = domain->host_data;
4427         int i;
4428
4429         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
4430
4431         for (i = 0; i < nr_irqs; i++) {
4432                 struct irq_data *data = irq_domain_get_irq_data(domain,
4433                                                                 virq + i);
4434                 struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
4435
4436                 BUG_ON(vm != vpe->its_vm);
4437
4438                 clear_bit(data->hwirq, vm->db_bitmap);
4439                 its_vpe_teardown(vpe);
4440                 irq_domain_reset_irq_data(data);
4441         }
4442
4443         if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
4444                 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
4445                 its_free_prop_table(vm->vprop_page);
4446         }
4447 }
4448
4449 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
4450                                     unsigned int nr_irqs, void *args)
4451 {
4452         struct irq_chip *irqchip = &its_vpe_irq_chip;
4453         struct its_vm *vm = args;
4454         unsigned long *bitmap;
4455         struct page *vprop_page;
4456         int base, nr_ids, i, err = 0;
4457
4458         BUG_ON(!vm);
4459
4460         bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
4461         if (!bitmap)
4462                 return -ENOMEM;
4463
4464         if (nr_ids < nr_irqs) {
4465                 its_lpi_free(bitmap, base, nr_ids);
4466                 return -ENOMEM;
4467         }
4468
4469         vprop_page = its_allocate_prop_table(GFP_KERNEL);
4470         if (!vprop_page) {
4471                 its_lpi_free(bitmap, base, nr_ids);
4472                 return -ENOMEM;
4473         }
4474
4475         vm->db_bitmap = bitmap;
4476         vm->db_lpi_base = base;
4477         vm->nr_db_lpis = nr_ids;
4478         vm->vprop_page = vprop_page;
4479
4480         if (gic_rdists->has_rvpeid)
4481                 irqchip = &its_vpe_4_1_irq_chip;
4482
4483         for (i = 0; i < nr_irqs; i++) {
4484                 vm->vpes[i]->vpe_db_lpi = base + i;
4485                 err = its_vpe_init(vm->vpes[i]);
4486                 if (err)
4487                         break;
4488                 err = its_irq_gic_domain_alloc(domain, virq + i,
4489                                                vm->vpes[i]->vpe_db_lpi);
4490                 if (err)
4491                         break;
4492                 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4493                                               irqchip, vm->vpes[i]);
4494                 set_bit(i, bitmap);
4495         }
4496
4497         if (err) {
4498                 if (i > 0)
4499                         its_vpe_irq_domain_free(domain, virq, i - 1);
4500
4501                 its_lpi_free(bitmap, base, nr_ids);
4502                 its_free_prop_table(vprop_page);
4503         }
4504
4505         return err;
4506 }
4507
4508 static int its_vpe_irq_domain_activate(struct irq_domain *domain,
4509                                        struct irq_data *d, bool reserve)
4510 {
4511         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4512         struct its_node *its;
4513
4514         /*
4515          * If we use the list map, we issue VMAPP on demand... Unless
4516          * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4517          * so that VSGIs can work.
4518          */
4519         if (!gic_requires_eager_mapping())
4520                 return 0;
4521
4522         /* Map the VPE to the first possible CPU */
4523         vpe->col_idx = cpumask_first(cpu_online_mask);
4524
4525         list_for_each_entry(its, &its_nodes, entry) {
4526                 if (!is_v4(its))
4527                         continue;
4528
4529                 its_send_vmapp(its, vpe, true);
4530                 its_send_vinvall(its, vpe);
4531         }
4532
4533         irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
4534
4535         return 0;
4536 }
4537
4538 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
4539                                           struct irq_data *d)
4540 {
4541         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4542         struct its_node *its;
4543
4544         /*
4545          * If we use the list map on GICv4.0, we unmap the VPE once no
4546          * VLPIs are associated with the VM.
4547          */
4548         if (!gic_requires_eager_mapping())
4549                 return;
4550
4551         list_for_each_entry(its, &its_nodes, entry) {
4552                 if (!is_v4(its))
4553                         continue;
4554
4555                 its_send_vmapp(its, vpe, false);
4556         }
4557 }
4558
4559 static const struct irq_domain_ops its_vpe_domain_ops = {
4560         .alloc                  = its_vpe_irq_domain_alloc,
4561         .free                   = its_vpe_irq_domain_free,
4562         .activate               = its_vpe_irq_domain_activate,
4563         .deactivate             = its_vpe_irq_domain_deactivate,
4564 };
4565
4566 static int its_force_quiescent(void __iomem *base)
4567 {
4568         u32 count = 1000000;    /* 1s */
4569         u32 val;
4570
4571         val = readl_relaxed(base + GITS_CTLR);
4572         /*
4573          * GIC architecture specification requires the ITS to be both
4574          * disabled and quiescent for writes to GITS_BASER<n> or
4575          * GITS_CBASER to not have UNPREDICTABLE results.
4576          */
4577         if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
4578                 return 0;
4579
4580         /* Disable the generation of all interrupts to this ITS */
4581         val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
4582         writel_relaxed(val, base + GITS_CTLR);
4583
4584         /* Poll GITS_CTLR and wait until ITS becomes quiescent */
4585         while (1) {
4586                 val = readl_relaxed(base + GITS_CTLR);
4587                 if (val & GITS_CTLR_QUIESCENT)
4588                         return 0;
4589
4590                 count--;
4591                 if (!count)
4592                         return -EBUSY;
4593
4594                 cpu_relax();
4595                 udelay(1);
4596         }
4597 }
4598
4599 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
4600 {
4601         struct its_node *its = data;
4602
4603         /* erratum 22375: only alloc 8MB table size (20 bits) */
4604         its->typer &= ~GITS_TYPER_DEVBITS;
4605         its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
4606         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
4607
4608         return true;
4609 }
4610
4611 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
4612 {
4613         struct its_node *its = data;
4614
4615         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
4616
4617         return true;
4618 }
4619
4620 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
4621 {
4622         struct its_node *its = data;
4623
4624         /* On QDF2400, the size of the ITE is 16Bytes */
4625         its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
4626         its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
4627
4628         return true;
4629 }
4630
4631 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
4632 {
4633         struct its_node *its = its_dev->its;
4634
4635         /*
4636          * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4637          * which maps 32-bit writes targeted at a separate window of
4638          * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4639          * with device ID taken from bits [device_id_bits + 1:2] of
4640          * the window offset.
4641          */
4642         return its->pre_its_base + (its_dev->device_id << 2);
4643 }
4644
4645 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
4646 {
4647         struct its_node *its = data;
4648         u32 pre_its_window[2];
4649         u32 ids;
4650
4651         if (!fwnode_property_read_u32_array(its->fwnode_handle,
4652                                            "socionext,synquacer-pre-its",
4653                                            pre_its_window,
4654                                            ARRAY_SIZE(pre_its_window))) {
4655
4656                 its->pre_its_base = pre_its_window[0];
4657                 its->get_msi_base = its_irq_get_msi_base_pre_its;
4658
4659                 ids = ilog2(pre_its_window[1]) - 2;
4660                 if (device_ids(its) > ids) {
4661                         its->typer &= ~GITS_TYPER_DEVBITS;
4662                         its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
4663                 }
4664
4665                 /* the pre-ITS breaks isolation, so disable MSI remapping */
4666                 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP;
4667                 return true;
4668         }
4669         return false;
4670 }
4671
4672 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
4673 {
4674         struct its_node *its = data;
4675
4676         /*
4677          * Hip07 insists on using the wrong address for the VLPI
4678          * page. Trick it into doing the right thing...
4679          */
4680         its->vlpi_redist_offset = SZ_128K;
4681         return true;
4682 }
4683
4684 static const struct gic_quirk its_quirks[] = {
4685 #ifdef CONFIG_CAVIUM_ERRATUM_22375
4686         {
4687                 .desc   = "ITS: Cavium errata 22375, 24313",
4688                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
4689                 .mask   = 0xffff0fff,
4690                 .init   = its_enable_quirk_cavium_22375,
4691         },
4692 #endif
4693 #ifdef CONFIG_CAVIUM_ERRATUM_23144
4694         {
4695                 .desc   = "ITS: Cavium erratum 23144",
4696                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
4697                 .mask   = 0xffff0fff,
4698                 .init   = its_enable_quirk_cavium_23144,
4699         },
4700 #endif
4701 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
4702         {
4703                 .desc   = "ITS: QDF2400 erratum 0065",
4704                 .iidr   = 0x00001070, /* QDF2400 ITS rev 1.x */
4705                 .mask   = 0xffffffff,
4706                 .init   = its_enable_quirk_qdf2400_e0065,
4707         },
4708 #endif
4709 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4710         {
4711                 /*
4712                  * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4713                  * implementation, but with a 'pre-ITS' added that requires
4714                  * special handling in software.
4715                  */
4716                 .desc   = "ITS: Socionext Synquacer pre-ITS",
4717                 .iidr   = 0x0001143b,
4718                 .mask   = 0xffffffff,
4719                 .init   = its_enable_quirk_socionext_synquacer,
4720         },
4721 #endif
4722 #ifdef CONFIG_HISILICON_ERRATUM_161600802
4723         {
4724                 .desc   = "ITS: Hip07 erratum 161600802",
4725                 .iidr   = 0x00000004,
4726                 .mask   = 0xffffffff,
4727                 .init   = its_enable_quirk_hip07_161600802,
4728         },
4729 #endif
4730         {
4731         }
4732 };
4733
4734 static void its_enable_quirks(struct its_node *its)
4735 {
4736         u32 iidr = readl_relaxed(its->base + GITS_IIDR);
4737
4738         gic_enable_quirks(iidr, its_quirks, its);
4739 }
4740
4741 static int its_save_disable(void)
4742 {
4743         struct its_node *its;
4744         int err = 0;
4745
4746         raw_spin_lock(&its_lock);
4747         list_for_each_entry(its, &its_nodes, entry) {
4748                 void __iomem *base;
4749
4750                 base = its->base;
4751                 its->ctlr_save = readl_relaxed(base + GITS_CTLR);
4752                 err = its_force_quiescent(base);
4753                 if (err) {
4754                         pr_err("ITS@%pa: failed to quiesce: %d\n",
4755                                &its->phys_base, err);
4756                         writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4757                         goto err;
4758                 }
4759
4760                 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
4761         }
4762
4763 err:
4764         if (err) {
4765                 list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
4766                         void __iomem *base;
4767
4768                         base = its->base;
4769                         writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4770                 }
4771         }
4772         raw_spin_unlock(&its_lock);
4773
4774         return err;
4775 }
4776
4777 static void its_restore_enable(void)
4778 {
4779         struct its_node *its;
4780         int ret;
4781
4782         raw_spin_lock(&its_lock);
4783         list_for_each_entry(its, &its_nodes, entry) {
4784                 void __iomem *base;
4785                 int i;
4786
4787                 base = its->base;
4788
4789                 /*
4790                  * Make sure that the ITS is disabled. If it fails to quiesce,
4791                  * don't restore it since writing to CBASER or BASER<n>
4792                  * registers is undefined according to the GIC v3 ITS
4793                  * Specification.
4794                  *
4795                  * Firmware resuming with the ITS enabled is terminally broken.
4796                  */
4797                 WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
4798                 ret = its_force_quiescent(base);
4799                 if (ret) {
4800                         pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
4801                                &its->phys_base, ret);
4802                         continue;
4803                 }
4804
4805                 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
4806
4807                 /*
4808                  * Writing CBASER resets CREADR to 0, so make CWRITER and
4809                  * cmd_write line up with it.
4810                  */
4811                 its->cmd_write = its->cmd_base;
4812                 gits_write_cwriter(0, base + GITS_CWRITER);
4813
4814                 /* Restore GITS_BASER from the value cache. */
4815                 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
4816                         struct its_baser *baser = &its->tables[i];
4817
4818                         if (!(baser->val & GITS_BASER_VALID))
4819                                 continue;
4820
4821                         its_write_baser(its, baser, baser->val);
4822                 }
4823                 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4824
4825                 /*
4826                  * Reinit the collection if it's stored in the ITS. This is
4827                  * indicated by the col_id being less than the HCC field.
4828                  * CID < HCC as specified in the GIC v3 Documentation.
4829                  */
4830                 if (its->collections[smp_processor_id()].col_id <
4831                     GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
4832                         its_cpu_init_collection(its);
4833         }
4834         raw_spin_unlock(&its_lock);
4835 }
4836
4837 static struct syscore_ops its_syscore_ops = {
4838         .suspend = its_save_disable,
4839         .resume = its_restore_enable,
4840 };
4841
4842 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
4843 {
4844         struct irq_domain *inner_domain;
4845         struct msi_domain_info *info;
4846
4847         info = kzalloc(sizeof(*info), GFP_KERNEL);
4848         if (!info)
4849                 return -ENOMEM;
4850
4851         inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its);
4852         if (!inner_domain) {
4853                 kfree(info);
4854                 return -ENOMEM;
4855         }
4856
4857         inner_domain->parent = its_parent;
4858         irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
4859         inner_domain->flags |= its->msi_domain_flags;
4860         info->ops = &its_msi_domain_ops;
4861         info->data = its;
4862         inner_domain->host_data = info;
4863
4864         return 0;
4865 }
4866
4867 static int its_init_vpe_domain(void)
4868 {
4869         struct its_node *its;
4870         u32 devid;
4871         int entries;
4872
4873         if (gic_rdists->has_direct_lpi) {
4874                 pr_info("ITS: Using DirectLPI for VPE invalidation\n");
4875                 return 0;
4876         }
4877
4878         /* Any ITS will do, even if not v4 */
4879         its = list_first_entry(&its_nodes, struct its_node, entry);
4880
4881         entries = roundup_pow_of_two(nr_cpu_ids);
4882         vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
4883                                  GFP_KERNEL);
4884         if (!vpe_proxy.vpes) {
4885                 pr_err("ITS: Can't allocate GICv4 proxy device array\n");
4886                 return -ENOMEM;
4887         }
4888
4889         /* Use the last possible DevID */
4890         devid = GENMASK(device_ids(its) - 1, 0);
4891         vpe_proxy.dev = its_create_device(its, devid, entries, false);
4892         if (!vpe_proxy.dev) {
4893                 kfree(vpe_proxy.vpes);
4894                 pr_err("ITS: Can't allocate GICv4 proxy device\n");
4895                 return -ENOMEM;
4896         }
4897
4898         BUG_ON(entries > vpe_proxy.dev->nr_ites);
4899
4900         raw_spin_lock_init(&vpe_proxy.lock);
4901         vpe_proxy.next_victim = 0;
4902         pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
4903                 devid, vpe_proxy.dev->nr_ites);
4904
4905         return 0;
4906 }
4907
4908 static int __init its_compute_its_list_map(struct resource *res,
4909                                            void __iomem *its_base)
4910 {
4911         int its_number;
4912         u32 ctlr;
4913
4914         /*
4915          * This is assumed to be done early enough that we're
4916          * guaranteed to be single-threaded, hence no
4917          * locking. Should this change, we should address
4918          * this.
4919          */
4920         its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
4921         if (its_number >= GICv4_ITS_LIST_MAX) {
4922                 pr_err("ITS@%pa: No ITSList entry available!\n",
4923                        &res->start);
4924                 return -EINVAL;
4925         }
4926
4927         ctlr = readl_relaxed(its_base + GITS_CTLR);
4928         ctlr &= ~GITS_CTLR_ITS_NUMBER;
4929         ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
4930         writel_relaxed(ctlr, its_base + GITS_CTLR);
4931         ctlr = readl_relaxed(its_base + GITS_CTLR);
4932         if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
4933                 its_number = ctlr & GITS_CTLR_ITS_NUMBER;
4934                 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
4935         }
4936
4937         if (test_and_set_bit(its_number, &its_list_map)) {
4938                 pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
4939                        &res->start, its_number);
4940                 return -EINVAL;
4941         }
4942
4943         return its_number;
4944 }
4945
4946 static int __init its_probe_one(struct resource *res,
4947                                 struct fwnode_handle *handle, int numa_node)
4948 {
4949         struct its_node *its;
4950         void __iomem *its_base;
4951         u32 val, ctlr;
4952         u64 baser, tmp, typer;
4953         struct page *page;
4954         int err;
4955
4956         its_base = ioremap(res->start, SZ_64K);
4957         if (!its_base) {
4958                 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
4959                 return -ENOMEM;
4960         }
4961
4962         val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
4963         if (val != 0x30 && val != 0x40) {
4964                 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
4965                 err = -ENODEV;
4966                 goto out_unmap;
4967         }
4968
4969         err = its_force_quiescent(its_base);
4970         if (err) {
4971                 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
4972                 goto out_unmap;
4973         }
4974
4975         pr_info("ITS %pR\n", res);
4976
4977         its = kzalloc(sizeof(*its), GFP_KERNEL);
4978         if (!its) {
4979                 err = -ENOMEM;
4980                 goto out_unmap;
4981         }
4982
4983         raw_spin_lock_init(&its->lock);
4984         mutex_init(&its->dev_alloc_lock);
4985         INIT_LIST_HEAD(&its->entry);
4986         INIT_LIST_HEAD(&its->its_device_list);
4987         typer = gic_read_typer(its_base + GITS_TYPER);
4988         its->typer = typer;
4989         its->base = its_base;
4990         its->phys_base = res->start;
4991         if (is_v4(its)) {
4992                 if (!(typer & GITS_TYPER_VMOVP)) {
4993                         err = its_compute_its_list_map(res, its_base);
4994                         if (err < 0)
4995                                 goto out_free_its;
4996
4997                         its->list_nr = err;
4998
4999                         pr_info("ITS@%pa: Using ITS number %d\n",
5000                                 &res->start, err);
5001                 } else {
5002                         pr_info("ITS@%pa: Single VMOVP capable\n", &res->start);
5003                 }
5004
5005                 if (is_v4_1(its)) {
5006                         u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer);
5007
5008                         its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K);
5009                         if (!its->sgir_base) {
5010                                 err = -ENOMEM;
5011                                 goto out_free_its;
5012                         }
5013
5014                         its->mpidr = readl_relaxed(its_base + GITS_MPIDR);
5015
5016                         pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
5017                                 &res->start, its->mpidr, svpet);
5018                 }
5019         }
5020
5021         its->numa_node = numa_node;
5022
5023         page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
5024                                 get_order(ITS_CMD_QUEUE_SZ));
5025         if (!page) {
5026                 err = -ENOMEM;
5027                 goto out_unmap_sgir;
5028         }
5029         its->cmd_base = (void *)page_address(page);
5030         its->cmd_write = its->cmd_base;
5031         its->fwnode_handle = handle;
5032         its->get_msi_base = its_irq_get_msi_base;
5033         its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP;
5034
5035         its_enable_quirks(its);
5036
5037         err = its_alloc_tables(its);
5038         if (err)
5039                 goto out_free_cmd;
5040
5041         err = its_alloc_collections(its);
5042         if (err)
5043                 goto out_free_tables;
5044
5045         baser = (virt_to_phys(its->cmd_base)    |
5046                  GITS_CBASER_RaWaWb             |
5047                  GITS_CBASER_InnerShareable     |
5048                  (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
5049                  GITS_CBASER_VALID);
5050
5051         gits_write_cbaser(baser, its->base + GITS_CBASER);
5052         tmp = gits_read_cbaser(its->base + GITS_CBASER);
5053
5054         if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
5055                 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
5056                         /*
5057                          * The HW reports non-shareable, we must
5058                          * remove the cacheability attributes as
5059                          * well.
5060                          */
5061                         baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
5062                                    GITS_CBASER_CACHEABILITY_MASK);
5063                         baser |= GITS_CBASER_nC;
5064                         gits_write_cbaser(baser, its->base + GITS_CBASER);
5065                 }
5066                 pr_info("ITS: using cache flushing for cmd queue\n");
5067                 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
5068         }
5069
5070         gits_write_cwriter(0, its->base + GITS_CWRITER);
5071         ctlr = readl_relaxed(its->base + GITS_CTLR);
5072         ctlr |= GITS_CTLR_ENABLE;
5073         if (is_v4(its))
5074                 ctlr |= GITS_CTLR_ImDe;
5075         writel_relaxed(ctlr, its->base + GITS_CTLR);
5076
5077         err = its_init_domain(handle, its);
5078         if (err)
5079                 goto out_free_tables;
5080
5081         raw_spin_lock(&its_lock);
5082         list_add(&its->entry, &its_nodes);
5083         raw_spin_unlock(&its_lock);
5084
5085         return 0;
5086
5087 out_free_tables:
5088         its_free_tables(its);
5089 out_free_cmd:
5090         free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
5091 out_unmap_sgir:
5092         if (its->sgir_base)
5093                 iounmap(its->sgir_base);
5094 out_free_its:
5095         kfree(its);
5096 out_unmap:
5097         iounmap(its_base);
5098         pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err);
5099         return err;
5100 }
5101
5102 static bool gic_rdists_supports_plpis(void)
5103 {
5104         return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
5105 }
5106
5107 static int redist_disable_lpis(void)
5108 {
5109         void __iomem *rbase = gic_data_rdist_rd_base();
5110         u64 timeout = USEC_PER_SEC;
5111         u64 val;
5112
5113         if (!gic_rdists_supports_plpis()) {
5114                 pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
5115                 return -ENXIO;
5116         }
5117
5118         val = readl_relaxed(rbase + GICR_CTLR);
5119         if (!(val & GICR_CTLR_ENABLE_LPIS))
5120                 return 0;
5121
5122         /*
5123          * If coming via a CPU hotplug event, we don't need to disable
5124          * LPIs before trying to re-enable them. They are already
5125          * configured and all is well in the world.
5126          *
5127          * If running with preallocated tables, there is nothing to do.
5128          */
5129         if (gic_data_rdist()->lpi_enabled ||
5130             (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
5131                 return 0;
5132
5133         /*
5134          * From that point on, we only try to do some damage control.
5135          */
5136         pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
5137                 smp_processor_id());
5138         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
5139
5140         /* Disable LPIs */
5141         val &= ~GICR_CTLR_ENABLE_LPIS;
5142         writel_relaxed(val, rbase + GICR_CTLR);
5143
5144         /* Make sure any change to GICR_CTLR is observable by the GIC */
5145         dsb(sy);
5146
5147         /*
5148          * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
5149          * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
5150          * Error out if we time out waiting for RWP to clear.
5151          */
5152         while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
5153                 if (!timeout) {
5154                         pr_err("CPU%d: Timeout while disabling LPIs\n",
5155                                smp_processor_id());
5156                         return -ETIMEDOUT;
5157                 }
5158                 udelay(1);
5159                 timeout--;
5160         }
5161
5162         /*
5163          * After it has been written to 1, it is IMPLEMENTATION
5164          * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
5165          * cleared to 0. Error out if clearing the bit failed.
5166          */
5167         if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
5168                 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
5169                 return -EBUSY;
5170         }
5171
5172         return 0;
5173 }
5174
5175 int its_cpu_init(void)
5176 {
5177         if (!list_empty(&its_nodes)) {
5178                 int ret;
5179
5180                 ret = redist_disable_lpis();
5181                 if (ret)
5182                         return ret;
5183
5184                 its_cpu_init_lpis();
5185                 its_cpu_init_collections();
5186         }
5187
5188         return 0;
5189 }
5190
5191 static const struct of_device_id its_device_id[] = {
5192         {       .compatible     = "arm,gic-v3-its",     },
5193         {},
5194 };
5195
5196 static int __init its_of_probe(struct device_node *node)
5197 {
5198         struct device_node *np;
5199         struct resource res;
5200
5201         for (np = of_find_matching_node(node, its_device_id); np;
5202              np = of_find_matching_node(np, its_device_id)) {
5203                 if (!of_device_is_available(np))
5204                         continue;
5205                 if (!of_property_read_bool(np, "msi-controller")) {
5206                         pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5207                                 np);
5208                         continue;
5209                 }
5210
5211                 if (of_address_to_resource(np, 0, &res)) {
5212                         pr_warn("%pOF: no regs?\n", np);
5213                         continue;
5214                 }
5215
5216                 its_probe_one(&res, &np->fwnode, of_node_to_nid(np));
5217         }
5218         return 0;
5219 }
5220
5221 #ifdef CONFIG_ACPI
5222
5223 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
5224
5225 #ifdef CONFIG_ACPI_NUMA
5226 struct its_srat_map {
5227         /* numa node id */
5228         u32     numa_node;
5229         /* GIC ITS ID */
5230         u32     its_id;
5231 };
5232
5233 static struct its_srat_map *its_srat_maps __initdata;
5234 static int its_in_srat __initdata;
5235
5236 static int __init acpi_get_its_numa_node(u32 its_id)
5237 {
5238         int i;
5239
5240         for (i = 0; i < its_in_srat; i++) {
5241                 if (its_id == its_srat_maps[i].its_id)
5242                         return its_srat_maps[i].numa_node;
5243         }
5244         return NUMA_NO_NODE;
5245 }
5246
5247 static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
5248                                           const unsigned long end)
5249 {
5250         return 0;
5251 }
5252
5253 static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
5254                          const unsigned long end)
5255 {
5256         int node;
5257         struct acpi_srat_gic_its_affinity *its_affinity;
5258
5259         its_affinity = (struct acpi_srat_gic_its_affinity *)header;
5260         if (!its_affinity)
5261                 return -EINVAL;
5262
5263         if (its_affinity->header.length < sizeof(*its_affinity)) {
5264                 pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5265                         its_affinity->header.length);
5266                 return -EINVAL;
5267         }
5268
5269         /*
5270          * Note that in theory a new proximity node could be created by this
5271          * entry as it is an SRAT resource allocation structure.
5272          * We do not currently support doing so.
5273          */
5274         node = pxm_to_node(its_affinity->proximity_domain);
5275
5276         if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
5277                 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
5278                 return 0;
5279         }
5280
5281         its_srat_maps[its_in_srat].numa_node = node;
5282         its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
5283         its_in_srat++;
5284         pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5285                 its_affinity->proximity_domain, its_affinity->its_id, node);
5286
5287         return 0;
5288 }
5289
5290 static void __init acpi_table_parse_srat_its(void)
5291 {
5292         int count;
5293
5294         count = acpi_table_parse_entries(ACPI_SIG_SRAT,
5295                         sizeof(struct acpi_table_srat),
5296                         ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5297                         gic_acpi_match_srat_its, 0);
5298         if (count <= 0)
5299                 return;
5300
5301         its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
5302                                       GFP_KERNEL);
5303         if (!its_srat_maps) {
5304                 pr_warn("SRAT: Failed to allocate memory for its_srat_maps!\n");
5305                 return;
5306         }
5307
5308         acpi_table_parse_entries(ACPI_SIG_SRAT,
5309                         sizeof(struct acpi_table_srat),
5310                         ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5311                         gic_acpi_parse_srat_its, 0);
5312 }
5313
5314 /* free the its_srat_maps after ITS probing */
5315 static void __init acpi_its_srat_maps_free(void)
5316 {
5317         kfree(its_srat_maps);
5318 }
5319 #else
5320 static void __init acpi_table_parse_srat_its(void)      { }
5321 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
5322 static void __init acpi_its_srat_maps_free(void) { }
5323 #endif
5324
5325 static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
5326                                           const unsigned long end)
5327 {
5328         struct acpi_madt_generic_translator *its_entry;
5329         struct fwnode_handle *dom_handle;
5330         struct resource res;
5331         int err;
5332
5333         its_entry = (struct acpi_madt_generic_translator *)header;
5334         memset(&res, 0, sizeof(res));
5335         res.start = its_entry->base_address;
5336         res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
5337         res.flags = IORESOURCE_MEM;
5338
5339         dom_handle = irq_domain_alloc_fwnode(&res.start);
5340         if (!dom_handle) {
5341                 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
5342                        &res.start);
5343                 return -ENOMEM;
5344         }
5345
5346         err = iort_register_domain_token(its_entry->translation_id, res.start,
5347                                          dom_handle);
5348         if (err) {
5349                 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
5350                        &res.start, its_entry->translation_id);
5351                 goto dom_err;
5352         }
5353
5354         err = its_probe_one(&res, dom_handle,
5355                         acpi_get_its_numa_node(its_entry->translation_id));
5356         if (!err)
5357                 return 0;
5358
5359         iort_deregister_domain_token(its_entry->translation_id);
5360 dom_err:
5361         irq_domain_free_fwnode(dom_handle);
5362         return err;
5363 }
5364
5365 static void __init its_acpi_probe(void)
5366 {
5367         acpi_table_parse_srat_its();
5368         acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5369                               gic_acpi_parse_madt_its, 0);
5370         acpi_its_srat_maps_free();
5371 }
5372 #else
5373 static void __init its_acpi_probe(void) { }
5374 #endif
5375
5376 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
5377                     struct irq_domain *parent_domain)
5378 {
5379         struct device_node *of_node;
5380         struct its_node *its;
5381         bool has_v4 = false;
5382         bool has_v4_1 = false;
5383         int err;
5384
5385         gic_rdists = rdists;
5386
5387         its_parent = parent_domain;
5388         of_node = to_of_node(handle);
5389         if (of_node)
5390                 its_of_probe(of_node);
5391         else
5392                 its_acpi_probe();
5393
5394         if (list_empty(&its_nodes)) {
5395                 pr_warn("ITS: No ITS available, not enabling LPIs\n");
5396                 return -ENXIO;
5397         }
5398
5399         err = allocate_lpi_tables();
5400         if (err)
5401                 return err;
5402
5403         list_for_each_entry(its, &its_nodes, entry) {
5404                 has_v4 |= is_v4(its);
5405                 has_v4_1 |= is_v4_1(its);
5406         }
5407
5408         /* Don't bother with inconsistent systems */
5409         if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
5410                 rdists->has_rvpeid = false;
5411
5412         if (has_v4 & rdists->has_vlpis) {
5413                 const struct irq_domain_ops *sgi_ops;
5414
5415                 if (has_v4_1)
5416                         sgi_ops = &its_sgi_domain_ops;
5417                 else
5418                         sgi_ops = NULL;
5419
5420                 if (its_init_vpe_domain() ||
5421                     its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
5422                         rdists->has_vlpis = false;
5423                         pr_err("ITS: Disabling GICv4 support\n");
5424                 }
5425         }
5426
5427         register_syscore_ops(&its_syscore_ops);
5428
5429         return 0;
5430 }