drivers/pci/controller/vmd.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Volume Management Device driver
   4  * Copyright (c) 2015, Intel Corporation.
   5  */
   6
   7 #include <linux/device.h>
   8 #include <linux/interrupt.h>
   9 #include <linux/irq.h>
  10 #include <linux/kernel.h>
  11 #include <linux/module.h>
  12 #include <linux/msi.h>
  13 #include <linux/pci.h>
  14 #include <linux/srcu.h>
  15 #include <linux/rculist.h>
  16 #include <linux/rcupdate.h>
  17
  18 #include <asm/irqdomain.h>
  19 #include <asm/device.h>
  20 #include <asm/msi.h>
  21 #include <asm/msidef.h>
  22
  23 #define VMD_CFGBAR      0
  24 #define VMD_MEMBAR1     2
  25 #define VMD_MEMBAR2     4
  26
  27 #define PCI_REG_VMCAP           0x40
  28 #define BUS_RESTRICT_CAP(vmcap) (vmcap & 0x1)
  29 #define PCI_REG_VMCONFIG        0x44
  30 #define BUS_RESTRICT_CFG(vmcfg) ((vmcfg >> 8) & 0x3)
  31 #define PCI_REG_VMLOCK          0x70
  32 #define MB2_SHADOW_EN(vmlock)   (vmlock & 0x2)
  33
  34 #define MB2_SHADOW_OFFSET       0x2000
  35 #define MB2_SHADOW_SIZE         16
  36
  37 enum vmd_features {
  38         /*
  39          * Device may contain registers which hint the physical location of the
  40          * membars, in order to allow proper address translation during
  41          * resource assignment to enable guest virtualization
  42          */
  43         VMD_FEAT_HAS_MEMBAR_SHADOW              = (1 << 0),
  44
  45         /*
  46          * Device may provide root port configuration information which limits
  47          * bus numbering
  48          */
  49         VMD_FEAT_HAS_BUS_RESTRICTIONS           = (1 << 1),
  50
  51         /*
  52          * Device contains physical location shadow registers in
  53          * vendor-specific capability space
  54          */
  55         VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP        = (1 << 2),
  56 };
  57
  58 /*
  59  * Lock for manipulating VMD IRQ lists.
  60  */
  61 static DEFINE_RAW_SPINLOCK(list_lock);
  62
  63 /**
  64  * struct vmd_irq - private data to map driver IRQ to the VMD shared vector
  65  * @node:       list item for parent traversal.
  66  * @irq:        back pointer to parent.
  67  * @enabled:    true if driver enabled IRQ
  68  * @virq:       the virtual IRQ value provided to the requesting driver.
  69  *
  70  * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to
  71  * a VMD IRQ using this structure.
  72  */
  73 struct vmd_irq {
  74         struct list_head        node;
  75         struct vmd_irq_list     *irq;
  76         bool                    enabled;
  77         unsigned int            virq;
  78 };
  79
  80 /**
  81  * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector
  82  * @irq_list:   the list of irq's the VMD one demuxes to.
  83  * @srcu:       SRCU struct for local synchronization.
  84  * @count:      number of child IRQs assigned to this vector; used to track
  85  *              sharing.
  86  */
  87 struct vmd_irq_list {
  88         struct list_head        irq_list;
  89         struct srcu_struct      srcu;
  90         unsigned int            count;
  91 };
  92
  93 struct vmd_dev {
  94         struct pci_dev          *dev;
  95
  96         spinlock_t              cfg_lock;
  97         char __iomem            *cfgbar;
  98
  99         int msix_count;
 100         struct vmd_irq_list     *irqs;
 101
 102         struct pci_sysdata      sysdata;
 103         struct resource         resources[3];
 104         struct irq_domain       *irq_domain;
 105         struct pci_bus          *bus;
 106         u8                      busn_start;
 107 };
 108
 109 static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus)
 110 {
 111         return container_of(bus->sysdata, struct vmd_dev, sysdata);
 112 }
 113
 114 static inline unsigned int index_from_irqs(struct vmd_dev *vmd,
 115                                            struct vmd_irq_list *irqs)
 116 {
 117         return irqs - vmd->irqs;
 118 }
 119
 120 /*
 121  * Drivers managing a device in a VMD domain allocate their own IRQs as before,
 122  * but the MSI entry for the hardware it's driving will be programmed with a
 123  * destination ID for the VMD MSI-X table.  The VMD muxes interrupts in its
 124  * domain into one of its own, and the VMD driver de-muxes these for the
 125  * handlers sharing that VMD IRQ.  The vmd irq_domain provides the operations
 126  * and irq_chip to set this up.
 127  */
 128 static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 129 {
 130         struct vmd_irq *vmdirq = data->chip_data;
 131         struct vmd_irq_list *irq = vmdirq->irq;
 132         struct vmd_dev *vmd = irq_data_get_irq_handler_data(data);
 133
 134         msg->address_hi = MSI_ADDR_BASE_HI;
 135         msg->address_lo = MSI_ADDR_BASE_LO |
 136                           MSI_ADDR_DEST_ID(index_from_irqs(vmd, irq));
 137         msg->data = 0;
 138 }
 139
 140 /*
 141  * We rely on MSI_FLAG_USE_DEF_CHIP_OPS to set the IRQ mask/unmask ops.
 142  */
 143 static void vmd_irq_enable(struct irq_data *data)
 144 {
 145         struct vmd_irq *vmdirq = data->chip_data;
 146         unsigned long flags;
 147
 148         raw_spin_lock_irqsave(&list_lock, flags);
 149         WARN_ON(vmdirq->enabled);
 150         list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list);
 151         vmdirq->enabled = true;
 152         raw_spin_unlock_irqrestore(&list_lock, flags);
 153
 154         data->chip->irq_unmask(data);
 155 }
 156
 157 static void vmd_irq_disable(struct irq_data *data)
 158 {
 159         struct vmd_irq *vmdirq = data->chip_data;
 160         unsigned long flags;
 161
 162         data->chip->irq_mask(data);
 163
 164         raw_spin_lock_irqsave(&list_lock, flags);
 165         if (vmdirq->enabled) {
 166                 list_del_rcu(&vmdirq->node);
 167                 vmdirq->enabled = false;
 168         }
 169         raw_spin_unlock_irqrestore(&list_lock, flags);
 170 }
 171
 172 /*
 173  * XXX: Stubbed until we develop acceptable way to not create conflicts with
 174  * other devices sharing the same vector.
 175  */
 176 static int vmd_irq_set_affinity(struct irq_data *data,
 177                                 const struct cpumask *dest, bool force)
 178 {
 179         return -EINVAL;
 180 }
 181
 182 static struct irq_chip vmd_msi_controller = {
 183         .name                   = "VMD-MSI",
 184         .irq_enable             = vmd_irq_enable,
 185         .irq_disable            = vmd_irq_disable,
 186         .irq_compose_msi_msg    = vmd_compose_msi_msg,
 187         .irq_set_affinity       = vmd_irq_set_affinity,
 188 };
 189
 190 static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info,
 191                                      msi_alloc_info_t *arg)
 192 {
 193         return 0;
 194 }
 195
 196 /*
 197  * XXX: We can be even smarter selecting the best IRQ once we solve the
 198  * affinity problem.
 199  */
 200 static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc)
 201 {
 202         int i, best = 1;
 203         unsigned long flags;
 204
 205         if (vmd->msix_count == 1)
 206                 return &vmd->irqs[0];
 207
 208         /*
 209          * White list for fast-interrupt handlers. All others will share the
 210          * "slow" interrupt vector.
 211          */
 212         switch (msi_desc_to_pci_dev(desc)->class) {
 213         case PCI_CLASS_STORAGE_EXPRESS:
 214                 break;
 215         default:
 216                 return &vmd->irqs[0];
 217         }
 218
 219         raw_spin_lock_irqsave(&list_lock, flags);
 220         for (i = 1; i < vmd->msix_count; i++)
 221                 if (vmd->irqs[i].count < vmd->irqs[best].count)
 222                         best = i;
 223         vmd->irqs[best].count++;
 224         raw_spin_unlock_irqrestore(&list_lock, flags);
 225
 226         return &vmd->irqs[best];
 227 }
 228
 229 static int vmd_msi_init(struct irq_domain *domain, struct msi_domain_info *info,
 230                         unsigned int virq, irq_hw_number_t hwirq,
 231                         msi_alloc_info_t *arg)
 232 {
 233         struct msi_desc *desc = arg->desc;
 234         struct vmd_dev *vmd = vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);
 235         struct vmd_irq *vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL);
 236         unsigned int index, vector;
 237
 238         if (!vmdirq)
 239                 return -ENOMEM;
 240
 241         INIT_LIST_HEAD(&vmdirq->node);
 242         vmdirq->irq = vmd_next_irq(vmd, desc);
 243         vmdirq->virq = virq;
 244         index = index_from_irqs(vmd, vmdirq->irq);
 245         vector = pci_irq_vector(vmd->dev, index);
 246
 247         irq_domain_set_info(domain, virq, vector, info->chip, vmdirq,
 248                             handle_untracked_irq, vmd, NULL);
 249         return 0;
 250 }
 251
 252 static void vmd_msi_free(struct irq_domain *domain,
 253                         struct msi_domain_info *info, unsigned int virq)
 254 {
 255         struct vmd_irq *vmdirq = irq_get_chip_data(virq);
 256         unsigned long flags;
 257
 258         synchronize_srcu(&vmdirq->irq->srcu);
 259
 260         /* XXX: Potential optimization to rebalance */
 261         raw_spin_lock_irqsave(&list_lock, flags);
 262         vmdirq->irq->count--;
 263         raw_spin_unlock_irqrestore(&list_lock, flags);
 264
 265         kfree(vmdirq);
 266 }
 267
 268 static int vmd_msi_prepare(struct irq_domain *domain, struct device *dev,
 269                            int nvec, msi_alloc_info_t *arg)
 270 {
 271         struct pci_dev *pdev = to_pci_dev(dev);
 272         struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
 273
 274         if (nvec > vmd->msix_count)
 275                 return vmd->msix_count;
 276
 277         memset(arg, 0, sizeof(*arg));
 278         return 0;
 279 }
 280
 281 static void vmd_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
 282 {
 283         arg->desc = desc;
 284 }
 285
 286 static struct msi_domain_ops vmd_msi_domain_ops = {
 287         .get_hwirq      = vmd_get_hwirq,
 288         .msi_init       = vmd_msi_init,
 289         .msi_free       = vmd_msi_free,
 290         .msi_prepare    = vmd_msi_prepare,
 291         .set_desc       = vmd_set_desc,
 292 };
 293
 294 static struct msi_domain_info vmd_msi_domain_info = {
 295         .flags          = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
 296                           MSI_FLAG_PCI_MSIX,
 297         .ops            = &vmd_msi_domain_ops,
 298         .chip           = &vmd_msi_controller,
 299 };
 300
 301 static int vmd_create_irq_domain(struct vmd_dev *vmd)
 302 {
 303         struct fwnode_handle *fn;
 304
 305         fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain);
 306         if (!fn)
 307                 return -ENODEV;
 308
 309         vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info, NULL);
 310         if (!vmd->irq_domain) {
 311                 irq_domain_free_fwnode(fn);
 312                 return -ENODEV;
 313         }
 314
 315         return 0;
 316 }
 317
 318 static void vmd_remove_irq_domain(struct vmd_dev *vmd)
 319 {
 320         if (vmd->irq_domain) {
 321                 struct fwnode_handle *fn = vmd->irq_domain->fwnode;
 322
 323                 irq_domain_remove(vmd->irq_domain);
 324                 irq_domain_free_fwnode(fn);
 325         }
 326 }
 327
 328 static char __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,
 329                                   unsigned int devfn, int reg, int len)
 330 {
 331         char __iomem *addr = vmd->cfgbar +
 332                              ((bus->number - vmd->busn_start) << 20) +
 333                              (devfn << 12) + reg;
 334
 335         if ((addr - vmd->cfgbar) + len >=
 336             resource_size(&vmd->dev->resource[VMD_CFGBAR]))
 337                 return NULL;
 338
 339         return addr;
 340 }
 341
 342 /*
 343  * CPU may deadlock if config space is not serialized on some versions of this
 344  * hardware, so all config space access is done under a spinlock.
 345  */
 346 static int vmd_pci_read(struct pci_bus *bus, unsigned int devfn, int reg,
 347                         int len, u32 *value)
 348 {
 349         struct vmd_dev *vmd = vmd_from_bus(bus);
 350         char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
 351         unsigned long flags;
 352         int ret = 0;
 353
 354         if (!addr)
 355                 return -EFAULT;
 356
 357         spin_lock_irqsave(&vmd->cfg_lock, flags);
 358         switch (len) {
 359         case 1:
 360                 *value = readb(addr);
 361                 break;
 362         case 2:
 363                 *value = readw(addr);
 364                 break;
 365         case 4:
 366                 *value = readl(addr);
 367                 break;
 368         default:
 369                 ret = -EINVAL;
 370                 break;
 371         }
 372         spin_unlock_irqrestore(&vmd->cfg_lock, flags);
 373         return ret;
 374 }
 375
 376 /*
 377  * VMD h/w converts non-posted config writes to posted memory writes. The
 378  * read-back in this function forces the completion so it returns only after
 379  * the config space was written, as expected.
 380  */
 381 static int vmd_pci_write(struct pci_bus *bus, unsigned int devfn, int reg,
 382                          int len, u32 value)
 383 {
 384         struct vmd_dev *vmd = vmd_from_bus(bus);
 385         char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
 386         unsigned long flags;
 387         int ret = 0;
 388
 389         if (!addr)
 390                 return -EFAULT;
 391
 392         spin_lock_irqsave(&vmd->cfg_lock, flags);
 393         switch (len) {
 394         case 1:
 395                 writeb(value, addr);
 396                 readb(addr);
 397                 break;
 398         case 2:
 399                 writew(value, addr);
 400                 readw(addr);
 401                 break;
 402         case 4:
 403                 writel(value, addr);
 404                 readl(addr);
 405                 break;
 406         default:
 407                 ret = -EINVAL;
 408                 break;
 409         }
 410         spin_unlock_irqrestore(&vmd->cfg_lock, flags);
 411         return ret;
 412 }
 413
 414 static struct pci_ops vmd_ops = {
 415         .read           = vmd_pci_read,
 416         .write          = vmd_pci_write,
 417 };
 418
 419 static void vmd_attach_resources(struct vmd_dev *vmd)
 420 {
 421         vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1];
 422         vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2];
 423 }
 424
 425 static void vmd_detach_resources(struct vmd_dev *vmd)
 426 {
 427         vmd->dev->resource[VMD_MEMBAR1].child = NULL;
 428         vmd->dev->resource[VMD_MEMBAR2].child = NULL;
 429 }
 430
 431 /*
 432  * VMD domains start at 0x10000 to not clash with ACPI _SEG domains.
 433  * Per ACPI r6.0, sec 6.5.6,  _SEG returns an integer, of which the lower
 434  * 16 bits are the PCI Segment Group (domain) number.  Other bits are
 435  * currently reserved.
 436  */
 437 static int vmd_find_free_domain(void)
 438 {
 439         int domain = 0xffff;
 440         struct pci_bus *bus = NULL;
 441
 442         while ((bus = pci_find_next_bus(bus)) != NULL)
 443                 domain = max_t(int, domain, pci_domain_nr(bus));
 444         return domain + 1;
 445 }
 446
 447 static int vmd_get_phys_offsets(struct vmd_dev *vmd, bool native_hint,
 448                                 resource_size_t *offset1,
 449                                 resource_size_t *offset2)
 450 {
 451         struct pci_dev *dev = vmd->dev;
 452         u64 phys1, phys2;
 453
 454         if (native_hint) {
 455                 u32 vmlock;
 456                 int ret;
 457
 458                 ret = pci_read_config_dword(dev, PCI_REG_VMLOCK, &vmlock);
 459                 if (ret || vmlock == ~0)
 460                         return -ENODEV;
 461
 462                 if (MB2_SHADOW_EN(vmlock)) {
 463                         void __iomem *membar2;
 464
 465                         membar2 = pci_iomap(dev, VMD_MEMBAR2, 0);
 466                         if (!membar2)
 467                                 return -ENOMEM;
 468                         phys1 = readq(membar2 + MB2_SHADOW_OFFSET);
 469                         phys2 = readq(membar2 + MB2_SHADOW_OFFSET + 8);
 470                         pci_iounmap(dev, membar2);
 471                 } else
 472                         return 0;
 473         } else {
 474                 /* Hypervisor-Emulated Vendor-Specific Capability */
 475                 int pos = pci_find_capability(dev, PCI_CAP_ID_VNDR);
 476                 u32 reg, regu;
 477
 478                 pci_read_config_dword(dev, pos + 4, &reg);
 479
 480                 /* "SHDW" */
 481                 if (pos && reg == 0x53484457) {
 482                         pci_read_config_dword(dev, pos + 8, &reg);
 483                         pci_read_config_dword(dev, pos + 12, &regu);
 484                         phys1 = (u64) regu << 32 | reg;
 485
 486                         pci_read_config_dword(dev, pos + 16, &reg);
 487                         pci_read_config_dword(dev, pos + 20, &regu);
 488                         phys2 = (u64) regu << 32 | reg;
 489                 } else
 490                         return 0;
 491         }
 492
 493         *offset1 = dev->resource[VMD_MEMBAR1].start -
 494                         (phys1 & PCI_BASE_ADDRESS_MEM_MASK);
 495         *offset2 = dev->resource[VMD_MEMBAR2].start -
 496                         (phys2 & PCI_BASE_ADDRESS_MEM_MASK);
 497
 498         return 0;
 499 }
 500
 501 static int vmd_get_bus_number_start(struct vmd_dev *vmd)
 502 {
 503         struct pci_dev *dev = vmd->dev;
 504         u16 reg;
 505
 506         pci_read_config_word(dev, PCI_REG_VMCAP, &reg);
 507         if (BUS_RESTRICT_CAP(reg)) {
 508                 pci_read_config_word(dev, PCI_REG_VMCONFIG, &reg);
 509
 510                 switch (BUS_RESTRICT_CFG(reg)) {
 511                 case 0:
 512                         vmd->busn_start = 0;
 513                         break;
 514                 case 1:
 515                         vmd->busn_start = 128;
 516                         break;
 517                 case 2:
 518                         vmd->busn_start = 224;
 519                         break;
 520                 default:
 521                         pci_err(dev, "Unknown Bus Offset Setting (%d)\n",
 522                                 BUS_RESTRICT_CFG(reg));
 523                         return -ENODEV;
 524                 }
 525         }
 526
 527         return 0;
 528 }
 529
 530 static irqreturn_t vmd_irq(int irq, void *data)
 531 {
 532         struct vmd_irq_list *irqs = data;
 533         struct vmd_irq *vmdirq;
 534         int idx;
 535
 536         idx = srcu_read_lock(&irqs->srcu);
 537         list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node)
 538                 generic_handle_irq(vmdirq->virq);
 539         srcu_read_unlock(&irqs->srcu, idx);
 540
 541         return IRQ_HANDLED;
 542 }
 543
 544 static int vmd_alloc_irqs(struct vmd_dev *vmd)
 545 {
 546         struct pci_dev *dev = vmd->dev;
 547         int i, err;
 548
 549         vmd->msix_count = pci_msix_vec_count(dev);
 550         if (vmd->msix_count < 0)
 551                 return -ENODEV;
 552
 553         vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count,
 554                                                 PCI_IRQ_MSIX);
 555         if (vmd->msix_count < 0)
 556                 return vmd->msix_count;
 557
 558         vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs),
 559                                  GFP_KERNEL);
 560         if (!vmd->irqs)
 561                 return -ENOMEM;
 562
 563         for (i = 0; i < vmd->msix_count; i++) {
 564                 err = init_srcu_struct(&vmd->irqs[i].srcu);
 565                 if (err)
 566                         return err;
 567
 568                 INIT_LIST_HEAD(&vmd->irqs[i].irq_list);
 569                 err = devm_request_irq(&dev->dev, pci_irq_vector(dev, i),
 570                                        vmd_irq, IRQF_NO_THREAD,
 571                                        "vmd", &vmd->irqs[i]);
 572                 if (err)
 573                         return err;
 574         }
 575
 576         return 0;
 577 }
 578
 579 static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)
 580 {
 581         struct pci_sysdata *sd = &vmd->sysdata;
 582         struct resource *res;
 583         u32 upper_bits;
 584         unsigned long flags;
 585         LIST_HEAD(resources);
 586         resource_size_t offset[2] = {0};
 587         resource_size_t membar2_offset = 0x2000;
 588         struct pci_bus *child;
 589         int ret;
 590
 591         /*
 592          * Shadow registers may exist in certain VMD device ids which allow
 593          * guests to correctly assign host physical addresses to the root ports
 594          * and child devices. These registers will either return the host value
 595          * or 0, depending on an enable bit in the VMD device.
 596          */
 597         if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) {
 598                 membar2_offset = MB2_SHADOW_OFFSET + MB2_SHADOW_SIZE;
 599                 ret = vmd_get_phys_offsets(vmd, true, &offset[0], &offset[1]);
 600                 if (ret)
 601                         return ret;
 602         } else if (features & VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP) {
 603                 ret = vmd_get_phys_offsets(vmd, false, &offset[0], &offset[1]);
 604                 if (ret)
 605                         return ret;
 606         }
 607
 608         /*
 609          * Certain VMD devices may have a root port configuration option which
 610          * limits the bus range to between 0-127, 128-255, or 224-255
 611          */
 612         if (features & VMD_FEAT_HAS_BUS_RESTRICTIONS) {
 613                 ret = vmd_get_bus_number_start(vmd);
 614                 if (ret)
 615                         return ret;
 616         }
 617
 618         res = &vmd->dev->resource[VMD_CFGBAR];
 619         vmd->resources[0] = (struct resource) {
 620                 .name  = "VMD CFGBAR",
 621                 .start = vmd->busn_start,
 622                 .end   = vmd->busn_start + (resource_size(res) >> 20) - 1,
 623                 .flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED,
 624         };
 625
 626         /*
 627          * If the window is below 4GB, clear IORESOURCE_MEM_64 so we can
 628          * put 32-bit resources in the window.
 629          *
 630          * There's no hardware reason why a 64-bit window *couldn't*
 631          * contain a 32-bit resource, but pbus_size_mem() computes the
 632          * bridge window size assuming a 64-bit window will contain no
 633          * 32-bit resources.  __pci_assign_resource() enforces that
 634          * artificial restriction to make sure everything will fit.
 635          *
 636          * The only way we could use a 64-bit non-prefetchable MEMBAR is
 637          * if its address is <4GB so that we can convert it to a 32-bit
 638          * resource.  To be visible to the host OS, all VMD endpoints must
 639          * be initially configured by platform BIOS, which includes setting
 640          * up these resources.  We can assume the device is configured
 641          * according to the platform needs.
 642          */
 643         res = &vmd->dev->resource[VMD_MEMBAR1];
 644         upper_bits = upper_32_bits(res->end);
 645         flags = res->flags & ~IORESOURCE_SIZEALIGN;
 646         if (!upper_bits)
 647                 flags &= ~IORESOURCE_MEM_64;
 648         vmd->resources[1] = (struct resource) {
 649                 .name  = "VMD MEMBAR1",
 650                 .start = res->start,
 651                 .end   = res->end,
 652                 .flags = flags,
 653                 .parent = res,
 654         };
 655
 656         res = &vmd->dev->resource[VMD_MEMBAR2];
 657         upper_bits = upper_32_bits(res->end);
 658         flags = res->flags & ~IORESOURCE_SIZEALIGN;
 659         if (!upper_bits)
 660                 flags &= ~IORESOURCE_MEM_64;
 661         vmd->resources[2] = (struct resource) {
 662                 .name  = "VMD MEMBAR2",
 663                 .start = res->start + membar2_offset,
 664                 .end   = res->end,
 665                 .flags = flags,
 666                 .parent = res,
 667         };
 668
 669         sd->vmd_dev = vmd->dev;
 670         sd->domain = vmd_find_free_domain();
 671         if (sd->domain < 0)
 672                 return sd->domain;
 673
 674         sd->node = pcibus_to_node(vmd->dev->bus);
 675
 676         ret = vmd_create_irq_domain(vmd);
 677         if (ret)
 678                 return ret;
 679
 680         /*
 681          * Override the irq domain bus token so the domain can be distinguished
 682          * from a regular PCI/MSI domain.
 683          */
 684         irq_domain_update_bus_token(vmd->irq_domain, DOMAIN_BUS_VMD_MSI);
 685
 686         pci_add_resource(&resources, &vmd->resources[0]);
 687         pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]);
 688         pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]);
 689
 690         vmd->bus = pci_create_root_bus(&vmd->dev->dev, vmd->busn_start,
 691                                        &vmd_ops, sd, &resources);
 692         if (!vmd->bus) {
 693                 pci_free_resource_list(&resources);
 694                 vmd_remove_irq_domain(vmd);
 695                 return -ENODEV;
 696         }
 697
 698         vmd_attach_resources(vmd);
 699         if (vmd->irq_domain)
 700                 dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain);
 701
 702         pci_scan_child_bus(vmd->bus);
 703         pci_assign_unassigned_bus_resources(vmd->bus);
 704
 705         /*
 706          * VMD root buses are virtual and don't return true on pci_is_pcie()
 707          * and will fail pcie_bus_configure_settings() early. It can instead be
 708          * run on each of the real root ports.
 709          */
 710         list_for_each_entry(child, &vmd->bus->children, node)
 711                 pcie_bus_configure_settings(child);
 712
 713         pci_bus_add_devices(vmd->bus);
 714
 715         WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj,
 716                                "domain"), "Can't create symlink to domain\n");
 717         return 0;
 718 }
 719
 720 static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
 721 {
 722         struct vmd_dev *vmd;
 723         int err;
 724
 725         if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20))
 726                 return -ENOMEM;
 727
 728         vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL);
 729         if (!vmd)
 730                 return -ENOMEM;
 731
 732         vmd->dev = dev;
 733         err = pcim_enable_device(dev);
 734         if (err < 0)
 735                 return err;
 736
 737         vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0);
 738         if (!vmd->cfgbar)
 739                 return -ENOMEM;
 740
 741         pci_set_master(dev);
 742         if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) &&
 743             dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32)))
 744                 return -ENODEV;
 745
 746         err = vmd_alloc_irqs(vmd);
 747         if (err)
 748                 return err;
 749
 750         spin_lock_init(&vmd->cfg_lock);
 751         pci_set_drvdata(dev, vmd);
 752         err = vmd_enable_domain(vmd, (unsigned long) id->driver_data);
 753         if (err)
 754                 return err;
 755
 756         dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n",
 757                  vmd->sysdata.domain);
 758         return 0;
 759 }
 760
 761 static void vmd_cleanup_srcu(struct vmd_dev *vmd)
 762 {
 763         int i;
 764
 765         for (i = 0; i < vmd->msix_count; i++)
 766                 cleanup_srcu_struct(&vmd->irqs[i].srcu);
 767 }
 768
 769 static void vmd_remove(struct pci_dev *dev)
 770 {
 771         struct vmd_dev *vmd = pci_get_drvdata(dev);
 772
 773         sysfs_remove_link(&vmd->dev->dev.kobj, "domain");
 774         pci_stop_root_bus(vmd->bus);
 775         pci_remove_root_bus(vmd->bus);
 776         vmd_cleanup_srcu(vmd);
 777         vmd_detach_resources(vmd);
 778         vmd_remove_irq_domain(vmd);
 779 }
 780
 781 #ifdef CONFIG_PM_SLEEP
 782 static int vmd_suspend(struct device *dev)
 783 {
 784         struct pci_dev *pdev = to_pci_dev(dev);
 785         struct vmd_dev *vmd = pci_get_drvdata(pdev);
 786         int i;
 787
 788         for (i = 0; i < vmd->msix_count; i++)
 789                 devm_free_irq(dev, pci_irq_vector(pdev, i), &vmd->irqs[i]);
 790
 791         return 0;
 792 }
 793
 794 static int vmd_resume(struct device *dev)
 795 {
 796         struct pci_dev *pdev = to_pci_dev(dev);
 797         struct vmd_dev *vmd = pci_get_drvdata(pdev);
 798         int err, i;
 799
 800         for (i = 0; i < vmd->msix_count; i++) {
 801                 err = devm_request_irq(dev, pci_irq_vector(pdev, i),
 802                                        vmd_irq, IRQF_NO_THREAD,
 803                                        "vmd", &vmd->irqs[i]);
 804                 if (err)
 805                         return err;
 806         }
 807
 808         return 0;
 809 }
 810 #endif
 811 static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume);
 812
 813 static const struct pci_device_id vmd_ids[] = {
 814         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_201D),
 815                 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP,},
 816         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_28C0),
 817                 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW |
 818                                 VMD_FEAT_HAS_BUS_RESTRICTIONS,},
 819         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x467f),
 820                 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
 821                                 VMD_FEAT_HAS_BUS_RESTRICTIONS,},
 822         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x4c3d),
 823                 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
 824                                 VMD_FEAT_HAS_BUS_RESTRICTIONS,},
 825         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B),
 826                 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
 827                                 VMD_FEAT_HAS_BUS_RESTRICTIONS,},
 828         {0,}
 829 };
 830 MODULE_DEVICE_TABLE(pci, vmd_ids);
 831
 832 static struct pci_driver vmd_drv = {
 833         .name           = "vmd",
 834         .id_table       = vmd_ids,
 835         .probe          = vmd_probe,
 836         .remove         = vmd_remove,
 837         .driver         = {
 838                 .pm     = &vmd_dev_pm_ops,
 839         },
 840 };
 841 module_pci_driver(vmd_drv);
 842
 843 MODULE_AUTHOR("Intel Corporation");
 844 MODULE_LICENSE("GPL v2");
 845 MODULE_VERSION("0.6");