drivers/iommu/intel/dmar.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2006, Intel Corporation.
   4  *
   5  * Copyright (C) 2006-2008 Intel Corporation
   6  * Author: Ashok Raj <ashok.raj@intel.com>
   7  * Author: Shaohua Li <shaohua.li@intel.com>
   8  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
   9  *
  10  * This file implements early detection/parsing of Remapping Devices
  11  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  12  * tables.
  13  *
  14  * These routines are used by both DMA-remapping and Interrupt-remapping
  15  */
  16
  17 #define pr_fmt(fmt)     "DMAR: " fmt
  18
  19 #include <linux/pci.h>
  20 #include <linux/dmar.h>
  21 #include <linux/iova.h>
  22 #include <linux/intel-iommu.h>
  23 #include <linux/timer.h>
  24 #include <linux/irq.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/tboot.h>
  27 #include <linux/dmi.h>
  28 #include <linux/slab.h>
  29 #include <linux/iommu.h>
  30 #include <linux/numa.h>
  31 #include <linux/limits.h>
  32 #include <asm/irq_remapping.h>
  33 #include <asm/iommu_table.h>
  34
  35 #include "../irq_remapping.h"
  36
  37 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
  38 struct dmar_res_callback {
  39         dmar_res_handler_t      cb[ACPI_DMAR_TYPE_RESERVED];
  40         void                    *arg[ACPI_DMAR_TYPE_RESERVED];
  41         bool                    ignore_unhandled;
  42         bool                    print_entry;
  43 };
  44
  45 /*
  46  * Assumptions:
  47  * 1) The hotplug framework guarentees that DMAR unit will be hot-added
  48  *    before IO devices managed by that unit.
  49  * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
  50  *    after IO devices managed by that unit.
  51  * 3) Hotplug events are rare.
  52  *
  53  * Locking rules for DMA and interrupt remapping related global data structures:
  54  * 1) Use dmar_global_lock in process context
  55  * 2) Use RCU in interrupt context
  56  */
  57 DECLARE_RWSEM(dmar_global_lock);
  58 LIST_HEAD(dmar_drhd_units);
  59
  60 struct acpi_table_header * __initdata dmar_tbl;
  61 static int dmar_dev_scope_status = 1;
  62 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
  63
  64 static int alloc_iommu(struct dmar_drhd_unit *drhd);
  65 static void free_iommu(struct intel_iommu *iommu);
  66
  67 extern const struct iommu_ops intel_iommu_ops;
  68
  69 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  70 {
  71         /*
  72          * add INCLUDE_ALL at the tail, so scan the list will find it at
  73          * the very end.
  74          */
  75         if (drhd->include_all)
  76                 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
  77         else
  78                 list_add_rcu(&drhd->list, &dmar_drhd_units);
  79 }
  80
  81 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
  82 {
  83         struct acpi_dmar_device_scope *scope;
  84
  85         *cnt = 0;
  86         while (start < end) {
  87                 scope = start;
  88                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
  89                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
  90                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
  91                         (*cnt)++;
  92                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
  93                         scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
  94                         pr_warn("Unsupported device scope\n");
  95                 }
  96                 start += scope->length;
  97         }
  98         if (*cnt == 0)
  99                 return NULL;
 100
 101         return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
 102 }
 103
 104 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
 105 {
 106         int i;
 107         struct device *tmp_dev;
 108
 109         if (*devices && *cnt) {
 110                 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
 111                         put_device(tmp_dev);
 112                 kfree(*devices);
 113         }
 114
 115         *devices = NULL;
 116         *cnt = 0;
 117 }
 118
 119 /* Optimize out kzalloc()/kfree() for normal cases */
 120 static char dmar_pci_notify_info_buf[64];
 121
 122 static struct dmar_pci_notify_info *
 123 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
 124 {
 125         int level = 0;
 126         size_t size;
 127         struct pci_dev *tmp;
 128         struct dmar_pci_notify_info *info;
 129
 130         BUG_ON(dev->is_virtfn);
 131
 132         /*
 133          * Ignore devices that have a domain number higher than what can
 134          * be looked up in DMAR, e.g. VMD subdevices with domain 0x10000
 135          */
 136         if (pci_domain_nr(dev->bus) > U16_MAX)
 137                 return NULL;
 138
 139         /* Only generate path[] for device addition event */
 140         if (event == BUS_NOTIFY_ADD_DEVICE)
 141                 for (tmp = dev; tmp; tmp = tmp->bus->self)
 142                         level++;
 143
 144         size = struct_size(info, path, level);
 145         if (size <= sizeof(dmar_pci_notify_info_buf)) {
 146                 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
 147         } else {
 148                 info = kzalloc(size, GFP_KERNEL);
 149                 if (!info) {
 150                         pr_warn("Out of memory when allocating notify_info "
 151                                 "for %s.\n", pci_name(dev));
 152                         if (dmar_dev_scope_status == 0)
 153                                 dmar_dev_scope_status = -ENOMEM;
 154                         return NULL;
 155                 }
 156         }
 157
 158         info->event = event;
 159         info->dev = dev;
 160         info->seg = pci_domain_nr(dev->bus);
 161         info->level = level;
 162         if (event == BUS_NOTIFY_ADD_DEVICE) {
 163                 for (tmp = dev; tmp; tmp = tmp->bus->self) {
 164                         level--;
 165                         info->path[level].bus = tmp->bus->number;
 166                         info->path[level].device = PCI_SLOT(tmp->devfn);
 167                         info->path[level].function = PCI_FUNC(tmp->devfn);
 168                         if (pci_is_root_bus(tmp->bus))
 169                                 info->bus = tmp->bus->number;
 170                 }
 171         }
 172
 173         return info;
 174 }
 175
 176 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
 177 {
 178         if ((void *)info != dmar_pci_notify_info_buf)
 179                 kfree(info);
 180 }
 181
 182 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
 183                                 struct acpi_dmar_pci_path *path, int count)
 184 {
 185         int i;
 186
 187         if (info->bus != bus)
 188                 goto fallback;
 189         if (info->level != count)
 190                 goto fallback;
 191
 192         for (i = 0; i < count; i++) {
 193                 if (path[i].device != info->path[i].device ||
 194                     path[i].function != info->path[i].function)
 195                         goto fallback;
 196         }
 197
 198         return true;
 199
 200 fallback:
 201
 202         if (count != 1)
 203                 return false;
 204
 205         i = info->level - 1;
 206         if (bus              == info->path[i].bus &&
 207             path[0].device   == info->path[i].device &&
 208             path[0].function == info->path[i].function) {
 209                 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
 210                         bus, path[0].device, path[0].function);
 211                 return true;
 212         }
 213
 214         return false;
 215 }
 216
 217 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
 218 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
 219                           void *start, void*end, u16 segment,
 220                           struct dmar_dev_scope *devices,
 221                           int devices_cnt)
 222 {
 223         int i, level;
 224         struct device *tmp, *dev = &info->dev->dev;
 225         struct acpi_dmar_device_scope *scope;
 226         struct acpi_dmar_pci_path *path;
 227
 228         if (segment != info->seg)
 229                 return 0;
 230
 231         for (; start < end; start += scope->length) {
 232                 scope = start;
 233                 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 234                     scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 235                         continue;
 236
 237                 path = (struct acpi_dmar_pci_path *)(scope + 1);
 238                 level = (scope->length - sizeof(*scope)) / sizeof(*path);
 239                 if (!dmar_match_pci_path(info, scope->bus, path, level))
 240                         continue;
 241
 242                 /*
 243                  * We expect devices with endpoint scope to have normal PCI
 244                  * headers, and devices with bridge scope to have bridge PCI
 245                  * headers.  However PCI NTB devices may be listed in the
 246                  * DMAR table with bridge scope, even though they have a
 247                  * normal PCI header.  NTB devices are identified by class
 248                  * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
 249                  * for this special case.
 250                  */
 251                 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 252                      info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
 253                     (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
 254                      (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
 255                       info->dev->class >> 16 != PCI_BASE_CLASS_BRIDGE))) {
 256                         pr_warn("Device scope type does not match for %s\n",
 257                                 pci_name(info->dev));
 258                         return -EINVAL;
 259                 }
 260
 261                 for_each_dev_scope(devices, devices_cnt, i, tmp)
 262                         if (tmp == NULL) {
 263                                 devices[i].bus = info->dev->bus->number;
 264                                 devices[i].devfn = info->dev->devfn;
 265                                 rcu_assign_pointer(devices[i].dev,
 266                                                    get_device(dev));
 267                                 return 1;
 268                         }
 269                 BUG_ON(i >= devices_cnt);
 270         }
 271
 272         return 0;
 273 }
 274
 275 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
 276                           struct dmar_dev_scope *devices, int count)
 277 {
 278         int index;
 279         struct device *tmp;
 280
 281         if (info->seg != segment)
 282                 return 0;
 283
 284         for_each_active_dev_scope(devices, count, index, tmp)
 285                 if (tmp == &info->dev->dev) {
 286                         RCU_INIT_POINTER(devices[index].dev, NULL);
 287                         synchronize_rcu();
 288                         put_device(tmp);
 289                         return 1;
 290                 }
 291
 292         return 0;
 293 }
 294
 295 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
 296 {
 297         int ret = 0;
 298         struct dmar_drhd_unit *dmaru;
 299         struct acpi_dmar_hardware_unit *drhd;
 300
 301         for_each_drhd_unit(dmaru) {
 302                 if (dmaru->include_all)
 303                         continue;
 304
 305                 drhd = container_of(dmaru->hdr,
 306                                     struct acpi_dmar_hardware_unit, header);
 307                 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
 308                                 ((void *)drhd) + drhd->header.length,
 309                                 dmaru->segment,
 310                                 dmaru->devices, dmaru->devices_cnt);
 311                 if (ret)
 312                         break;
 313         }
 314         if (ret >= 0)
 315                 ret = dmar_iommu_notify_scope_dev(info);
 316         if (ret < 0 && dmar_dev_scope_status == 0)
 317                 dmar_dev_scope_status = ret;
 318
 319         if (ret >= 0)
 320                 intel_irq_remap_add_device(info);
 321
 322         return ret;
 323 }
 324
 325 static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
 326 {
 327         struct dmar_drhd_unit *dmaru;
 328
 329         for_each_drhd_unit(dmaru)
 330                 if (dmar_remove_dev_scope(info, dmaru->segment,
 331                         dmaru->devices, dmaru->devices_cnt))
 332                         break;
 333         dmar_iommu_notify_scope_dev(info);
 334 }
 335
 336 static inline void vf_inherit_msi_domain(struct pci_dev *pdev)
 337 {
 338         struct pci_dev *physfn = pci_physfn(pdev);
 339
 340         dev_set_msi_domain(&pdev->dev, dev_get_msi_domain(&physfn->dev));
 341 }
 342
 343 static int dmar_pci_bus_notifier(struct notifier_block *nb,
 344                                  unsigned long action, void *data)
 345 {
 346         struct pci_dev *pdev = to_pci_dev(data);
 347         struct dmar_pci_notify_info *info;
 348
 349         /* Only care about add/remove events for physical functions.
 350          * For VFs we actually do the lookup based on the corresponding
 351          * PF in device_to_iommu() anyway. */
 352         if (pdev->is_virtfn) {
 353                 /*
 354                  * Ensure that the VF device inherits the irq domain of the
 355                  * PF device. Ideally the device would inherit the domain
 356                  * from the bus, but DMAR can have multiple units per bus
 357                  * which makes this impossible. The VF 'bus' could inherit
 358                  * from the PF device, but that's yet another x86'sism to
 359                  * inflict on everybody else.
 360                  */
 361                 if (action == BUS_NOTIFY_ADD_DEVICE)
 362                         vf_inherit_msi_domain(pdev);
 363                 return NOTIFY_DONE;
 364         }
 365
 366         if (action != BUS_NOTIFY_ADD_DEVICE &&
 367             action != BUS_NOTIFY_REMOVED_DEVICE)
 368                 return NOTIFY_DONE;
 369
 370         info = dmar_alloc_pci_notify_info(pdev, action);
 371         if (!info)
 372                 return NOTIFY_DONE;
 373
 374         down_write(&dmar_global_lock);
 375         if (action == BUS_NOTIFY_ADD_DEVICE)
 376                 dmar_pci_bus_add_dev(info);
 377         else if (action == BUS_NOTIFY_REMOVED_DEVICE)
 378                 dmar_pci_bus_del_dev(info);
 379         up_write(&dmar_global_lock);
 380
 381         dmar_free_pci_notify_info(info);
 382
 383         return NOTIFY_OK;
 384 }
 385
 386 static struct notifier_block dmar_pci_bus_nb = {
 387         .notifier_call = dmar_pci_bus_notifier,
 388         .priority = INT_MIN,
 389 };
 390
 391 static struct dmar_drhd_unit *
 392 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
 393 {
 394         struct dmar_drhd_unit *dmaru;
 395
 396         list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list,
 397                                 dmar_rcu_check())
 398                 if (dmaru->segment == drhd->segment &&
 399                     dmaru->reg_base_addr == drhd->address)
 400                         return dmaru;
 401
 402         return NULL;
 403 }
 404
 405 /*
 406  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 407  * structure which uniquely represent one DMA remapping hardware unit
 408  * present in the platform
 409  */
 410 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
 411 {
 412         struct acpi_dmar_hardware_unit *drhd;
 413         struct dmar_drhd_unit *dmaru;
 414         int ret;
 415
 416         drhd = (struct acpi_dmar_hardware_unit *)header;
 417         dmaru = dmar_find_dmaru(drhd);
 418         if (dmaru)
 419                 goto out;
 420
 421         dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
 422         if (!dmaru)
 423                 return -ENOMEM;
 424
 425         /*
 426          * If header is allocated from slab by ACPI _DSM method, we need to
 427          * copy the content because the memory buffer will be freed on return.
 428          */
 429         dmaru->hdr = (void *)(dmaru + 1);
 430         memcpy(dmaru->hdr, header, header->length);
 431         dmaru->reg_base_addr = drhd->address;
 432         dmaru->segment = drhd->segment;
 433         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 434         dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
 435                                               ((void *)drhd) + drhd->header.length,
 436                                               &dmaru->devices_cnt);
 437         if (dmaru->devices_cnt && dmaru->devices == NULL) {
 438                 kfree(dmaru);
 439                 return -ENOMEM;
 440         }
 441
 442         ret = alloc_iommu(dmaru);
 443         if (ret) {
 444                 dmar_free_dev_scope(&dmaru->devices,
 445                                     &dmaru->devices_cnt);
 446                 kfree(dmaru);
 447                 return ret;
 448         }
 449         dmar_register_drhd_unit(dmaru);
 450
 451 out:
 452         if (arg)
 453                 (*(int *)arg)++;
 454
 455         return 0;
 456 }
 457
 458 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
 459 {
 460         if (dmaru->devices && dmaru->devices_cnt)
 461                 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
 462         if (dmaru->iommu)
 463                 free_iommu(dmaru->iommu);
 464         kfree(dmaru);
 465 }
 466
 467 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
 468                                       void *arg)
 469 {
 470         struct acpi_dmar_andd *andd = (void *)header;
 471
 472         /* Check for NUL termination within the designated length */
 473         if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
 474                 pr_warn(FW_BUG
 475                            "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
 476                            "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 477                            dmi_get_system_info(DMI_BIOS_VENDOR),
 478                            dmi_get_system_info(DMI_BIOS_VERSION),
 479                            dmi_get_system_info(DMI_PRODUCT_VERSION));
 480                 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 481                 return -EINVAL;
 482         }
 483         pr_info("ANDD device: %x name: %s\n", andd->device_number,
 484                 andd->device_name);
 485
 486         return 0;
 487 }
 488
 489 #ifdef CONFIG_ACPI_NUMA
 490 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
 491 {
 492         struct acpi_dmar_rhsa *rhsa;
 493         struct dmar_drhd_unit *drhd;
 494
 495         rhsa = (struct acpi_dmar_rhsa *)header;
 496         for_each_drhd_unit(drhd) {
 497                 if (drhd->reg_base_addr == rhsa->base_address) {
 498                         int node = pxm_to_node(rhsa->proximity_domain);
 499
 500                         if (!node_online(node))
 501                                 node = NUMA_NO_NODE;
 502                         drhd->iommu->node = node;
 503                         return 0;
 504                 }
 505         }
 506         pr_warn(FW_BUG
 507                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 508                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 509                 rhsa->base_address,
 510                 dmi_get_system_info(DMI_BIOS_VENDOR),
 511                 dmi_get_system_info(DMI_BIOS_VERSION),
 512                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 513         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 514
 515         return 0;
 516 }
 517 #else
 518 #define dmar_parse_one_rhsa             dmar_res_noop
 519 #endif
 520
 521 static void
 522 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 523 {
 524         struct acpi_dmar_hardware_unit *drhd;
 525         struct acpi_dmar_reserved_memory *rmrr;
 526         struct acpi_dmar_atsr *atsr;
 527         struct acpi_dmar_rhsa *rhsa;
 528
 529         switch (header->type) {
 530         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 531                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 532                                     header);
 533                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 534                         (unsigned long long)drhd->address, drhd->flags);
 535                 break;
 536         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 537                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 538                                     header);
 539                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 540                         (unsigned long long)rmrr->base_address,
 541                         (unsigned long long)rmrr->end_address);
 542                 break;
 543         case ACPI_DMAR_TYPE_ROOT_ATS:
 544                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 545                 pr_info("ATSR flags: %#x\n", atsr->flags);
 546                 break;
 547         case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
 548                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 549                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 550                        (unsigned long long)rhsa->base_address,
 551                        rhsa->proximity_domain);
 552                 break;
 553         case ACPI_DMAR_TYPE_NAMESPACE:
 554                 /* We don't print this here because we need to sanity-check
 555                    it first. So print it in dmar_parse_one_andd() instead. */
 556                 break;
 557         }
 558 }
 559
 560 /**
 561  * dmar_table_detect - checks to see if the platform supports DMAR devices
 562  */
 563 static int __init dmar_table_detect(void)
 564 {
 565         acpi_status status = AE_OK;
 566
 567         /* if we could find DMAR table, then there are DMAR devices */
 568         status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
 569
 570         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 571                 pr_warn("Unable to map DMAR\n");
 572                 status = AE_NOT_FOUND;
 573         }
 574
 575         return ACPI_SUCCESS(status) ? 0 : -ENOENT;
 576 }
 577
 578 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
 579                                        size_t len, struct dmar_res_callback *cb)
 580 {
 581         struct acpi_dmar_header *iter, *next;
 582         struct acpi_dmar_header *end = ((void *)start) + len;
 583
 584         for (iter = start; iter < end; iter = next) {
 585                 next = (void *)iter + iter->length;
 586                 if (iter->length == 0) {
 587                         /* Avoid looping forever on bad ACPI tables */
 588                         pr_debug(FW_BUG "Invalid 0-length structure\n");
 589                         break;
 590                 } else if (next > end) {
 591                         /* Avoid passing table end */
 592                         pr_warn(FW_BUG "Record passes table end\n");
 593                         return -EINVAL;
 594                 }
 595
 596                 if (cb->print_entry)
 597                         dmar_table_print_dmar_entry(iter);
 598
 599                 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
 600                         /* continue for forward compatibility */
 601                         pr_debug("Unknown DMAR structure type %d\n",
 602                                  iter->type);
 603                 } else if (cb->cb[iter->type]) {
 604                         int ret;
 605
 606                         ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
 607                         if (ret)
 608                                 return ret;
 609                 } else if (!cb->ignore_unhandled) {
 610                         pr_warn("No handler for DMAR structure type %d\n",
 611                                 iter->type);
 612                         return -EINVAL;
 613                 }
 614         }
 615
 616         return 0;
 617 }
 618
 619 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
 620                                        struct dmar_res_callback *cb)
 621 {
 622         return dmar_walk_remapping_entries((void *)(dmar + 1),
 623                         dmar->header.length - sizeof(*dmar), cb);
 624 }
 625
 626 /**
 627  * parse_dmar_table - parses the DMA reporting table
 628  */
 629 static int __init
 630 parse_dmar_table(void)
 631 {
 632         struct acpi_table_dmar *dmar;
 633         int drhd_count = 0;
 634         int ret;
 635         struct dmar_res_callback cb = {
 636                 .print_entry = true,
 637                 .ignore_unhandled = true,
 638                 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
 639                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
 640                 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
 641                 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
 642                 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
 643                 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
 644         };
 645
 646         /*
 647          * Do it again, earlier dmar_tbl mapping could be mapped with
 648          * fixed map.
 649          */
 650         dmar_table_detect();
 651
 652         /*
 653          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 654          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 655          */
 656         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 657
 658         dmar = (struct acpi_table_dmar *)dmar_tbl;
 659         if (!dmar)
 660                 return -ENODEV;
 661
 662         if (dmar->width < PAGE_SHIFT - 1) {
 663                 pr_warn("Invalid DMAR haw\n");
 664                 return -EINVAL;
 665         }
 666
 667         pr_info("Host address width %d\n", dmar->width + 1);
 668         ret = dmar_walk_dmar_table(dmar, &cb);
 669         if (ret == 0 && drhd_count == 0)
 670                 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 671
 672         return ret;
 673 }
 674
 675 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
 676                                  int cnt, struct pci_dev *dev)
 677 {
 678         int index;
 679         struct device *tmp;
 680
 681         while (dev) {
 682                 for_each_active_dev_scope(devices, cnt, index, tmp)
 683                         if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
 684                                 return 1;
 685
 686                 /* Check our parent */
 687                 dev = dev->bus->self;
 688         }
 689
 690         return 0;
 691 }
 692
 693 struct dmar_drhd_unit *
 694 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 695 {
 696         struct dmar_drhd_unit *dmaru;
 697         struct acpi_dmar_hardware_unit *drhd;
 698
 699         dev = pci_physfn(dev);
 700
 701         rcu_read_lock();
 702         for_each_drhd_unit(dmaru) {
 703                 drhd = container_of(dmaru->hdr,
 704                                     struct acpi_dmar_hardware_unit,
 705                                     header);
 706
 707                 if (dmaru->include_all &&
 708                     drhd->segment == pci_domain_nr(dev->bus))
 709                         goto out;
 710
 711                 if (dmar_pci_device_match(dmaru->devices,
 712                                           dmaru->devices_cnt, dev))
 713                         goto out;
 714         }
 715         dmaru = NULL;
 716 out:
 717         rcu_read_unlock();
 718
 719         return dmaru;
 720 }
 721
 722 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
 723                                               struct acpi_device *adev)
 724 {
 725         struct dmar_drhd_unit *dmaru;
 726         struct acpi_dmar_hardware_unit *drhd;
 727         struct acpi_dmar_device_scope *scope;
 728         struct device *tmp;
 729         int i;
 730         struct acpi_dmar_pci_path *path;
 731
 732         for_each_drhd_unit(dmaru) {
 733                 drhd = container_of(dmaru->hdr,
 734                                     struct acpi_dmar_hardware_unit,
 735                                     header);
 736
 737                 for (scope = (void *)(drhd + 1);
 738                      (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
 739                      scope = ((void *)scope) + scope->length) {
 740                         if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
 741                                 continue;
 742                         if (scope->enumeration_id != device_number)
 743                                 continue;
 744
 745                         path = (void *)(scope + 1);
 746                         pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
 747                                 dev_name(&adev->dev), dmaru->reg_base_addr,
 748                                 scope->bus, path->device, path->function);
 749                         for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
 750                                 if (tmp == NULL) {
 751                                         dmaru->devices[i].bus = scope->bus;
 752                                         dmaru->devices[i].devfn = PCI_DEVFN(path->device,
 753                                                                             path->function);
 754                                         rcu_assign_pointer(dmaru->devices[i].dev,
 755                                                            get_device(&adev->dev));
 756                                         return;
 757                                 }
 758                         BUG_ON(i >= dmaru->devices_cnt);
 759                 }
 760         }
 761         pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
 762                 device_number, dev_name(&adev->dev));
 763 }
 764
 765 static int __init dmar_acpi_dev_scope_init(void)
 766 {
 767         struct acpi_dmar_andd *andd;
 768
 769         if (dmar_tbl == NULL)
 770                 return -ENODEV;
 771
 772         for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
 773              ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
 774              andd = ((void *)andd) + andd->header.length) {
 775                 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
 776                         acpi_handle h;
 777                         struct acpi_device *adev;
 778
 779                         if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
 780                                                           andd->device_name,
 781                                                           &h))) {
 782                                 pr_err("Failed to find handle for ACPI object %s\n",
 783                                        andd->device_name);
 784                                 continue;
 785                         }
 786                         if (acpi_bus_get_device(h, &adev)) {
 787                                 pr_err("Failed to get device for ACPI object %s\n",
 788                                        andd->device_name);
 789                                 continue;
 790                         }
 791                         dmar_acpi_insert_dev_scope(andd->device_number, adev);
 792                 }
 793         }
 794         return 0;
 795 }
 796
 797 int __init dmar_dev_scope_init(void)
 798 {
 799         struct pci_dev *dev = NULL;
 800         struct dmar_pci_notify_info *info;
 801
 802         if (dmar_dev_scope_status != 1)
 803                 return dmar_dev_scope_status;
 804
 805         if (list_empty(&dmar_drhd_units)) {
 806                 dmar_dev_scope_status = -ENODEV;
 807         } else {
 808                 dmar_dev_scope_status = 0;
 809
 810                 dmar_acpi_dev_scope_init();
 811
 812                 for_each_pci_dev(dev) {
 813                         if (dev->is_virtfn)
 814                                 continue;
 815
 816                         info = dmar_alloc_pci_notify_info(dev,
 817                                         BUS_NOTIFY_ADD_DEVICE);
 818                         if (!info) {
 819                                 return dmar_dev_scope_status;
 820                         } else {
 821                                 dmar_pci_bus_add_dev(info);
 822                                 dmar_free_pci_notify_info(info);
 823                         }
 824                 }
 825         }
 826
 827         return dmar_dev_scope_status;
 828 }
 829
 830 void __init dmar_register_bus_notifier(void)
 831 {
 832         bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
 833 }
 834
 835
 836 int __init dmar_table_init(void)
 837 {
 838         static int dmar_table_initialized;
 839         int ret;
 840
 841         if (dmar_table_initialized == 0) {
 842                 ret = parse_dmar_table();
 843                 if (ret < 0) {
 844                         if (ret != -ENODEV)
 845                                 pr_info("Parse DMAR table failure.\n");
 846                 } else  if (list_empty(&dmar_drhd_units)) {
 847                         pr_info("No DMAR devices found\n");
 848                         ret = -ENODEV;
 849                 }
 850
 851                 if (ret < 0)
 852                         dmar_table_initialized = ret;
 853                 else
 854                         dmar_table_initialized = 1;
 855         }
 856
 857         return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
 858 }
 859
 860 static void warn_invalid_dmar(u64 addr, const char *message)
 861 {
 862         pr_warn_once(FW_BUG
 863                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 864                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 865                 addr, message,
 866                 dmi_get_system_info(DMI_BIOS_VENDOR),
 867                 dmi_get_system_info(DMI_BIOS_VERSION),
 868                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 869         add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 870 }
 871
 872 static int __ref
 873 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
 874 {
 875         struct acpi_dmar_hardware_unit *drhd;
 876         void __iomem *addr;
 877         u64 cap, ecap;
 878
 879         drhd = (void *)entry;
 880         if (!drhd->address) {
 881                 warn_invalid_dmar(0, "");
 882                 return -EINVAL;
 883         }
 884
 885         if (arg)
 886                 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
 887         else
 888                 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 889         if (!addr) {
 890                 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
 891                 return -EINVAL;
 892         }
 893
 894         cap = dmar_readq(addr + DMAR_CAP_REG);
 895         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 896
 897         if (arg)
 898                 iounmap(addr);
 899         else
 900                 early_iounmap(addr, VTD_PAGE_SIZE);
 901
 902         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 903                 warn_invalid_dmar(drhd->address, " returns all ones");
 904                 return -EINVAL;
 905         }
 906
 907         return 0;
 908 }
 909
 910 int __init detect_intel_iommu(void)
 911 {
 912         int ret;
 913         struct dmar_res_callback validate_drhd_cb = {
 914                 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
 915                 .ignore_unhandled = true,
 916         };
 917
 918         down_write(&dmar_global_lock);
 919         ret = dmar_table_detect();
 920         if (!ret)
 921                 ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
 922                                            &validate_drhd_cb);
 923         if (!ret && !no_iommu && !iommu_detected &&
 924             (!dmar_disabled || dmar_platform_optin())) {
 925                 iommu_detected = 1;
 926                 /* Make sure ACS will be enabled */
 927                 pci_request_acs();
 928         }
 929
 930 #ifdef CONFIG_X86
 931         if (!ret) {
 932                 x86_init.iommu.iommu_init = intel_iommu_init;
 933                 x86_platform.iommu_shutdown = intel_iommu_shutdown;
 934         }
 935
 936 #endif
 937
 938         if (dmar_tbl) {
 939                 acpi_put_table(dmar_tbl);
 940                 dmar_tbl = NULL;
 941         }
 942         up_write(&dmar_global_lock);
 943
 944         return ret ? ret : 1;
 945 }
 946
 947 static void unmap_iommu(struct intel_iommu *iommu)
 948 {
 949         iounmap(iommu->reg);
 950         release_mem_region(iommu->reg_phys, iommu->reg_size);
 951 }
 952
 953 /**
 954  * map_iommu: map the iommu's registers
 955  * @iommu: the iommu to map
 956  * @phys_addr: the physical address of the base resgister
 957  *
 958  * Memory map the iommu's registers.  Start w/ a single page, and
 959  * possibly expand if that turns out to be insufficent.
 960  */
 961 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 962 {
 963         int map_size, err=0;
 964
 965         iommu->reg_phys = phys_addr;
 966         iommu->reg_size = VTD_PAGE_SIZE;
 967
 968         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 969                 pr_err("Can't reserve memory\n");
 970                 err = -EBUSY;
 971                 goto out;
 972         }
 973
 974         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 975         if (!iommu->reg) {
 976                 pr_err("Can't map the region\n");
 977                 err = -ENOMEM;
 978                 goto release;
 979         }
 980
 981         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 982         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 983
 984         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 985                 err = -EINVAL;
 986                 warn_invalid_dmar(phys_addr, " returns all ones");
 987                 goto unmap;
 988         }
 989         if (ecap_vcs(iommu->ecap))
 990                 iommu->vccap = dmar_readq(iommu->reg + DMAR_VCCAP_REG);
 991
 992         /* the registers might be more than one page */
 993         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 994                          cap_max_fault_reg_offset(iommu->cap));
 995         map_size = VTD_PAGE_ALIGN(map_size);
 996         if (map_size > iommu->reg_size) {
 997                 iounmap(iommu->reg);
 998                 release_mem_region(iommu->reg_phys, iommu->reg_size);
 999                 iommu->reg_size = map_size;
1000                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
1001                                         iommu->name)) {
1002                         pr_err("Can't reserve memory\n");
1003                         err = -EBUSY;
1004                         goto out;
1005                 }
1006                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
1007                 if (!iommu->reg) {
1008                         pr_err("Can't map the region\n");
1009                         err = -ENOMEM;
1010                         goto release;
1011                 }
1012         }
1013         err = 0;
1014         goto out;
1015
1016 unmap:
1017         iounmap(iommu->reg);
1018 release:
1019         release_mem_region(iommu->reg_phys, iommu->reg_size);
1020 out:
1021         return err;
1022 }
1023
1024 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
1025 {
1026         iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
1027                                             DMAR_UNITS_SUPPORTED);
1028         if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
1029                 iommu->seq_id = -1;
1030         } else {
1031                 set_bit(iommu->seq_id, dmar_seq_ids);
1032                 sprintf(iommu->name, "dmar%d", iommu->seq_id);
1033         }
1034
1035         return iommu->seq_id;
1036 }
1037
1038 static void dmar_free_seq_id(struct intel_iommu *iommu)
1039 {
1040         if (iommu->seq_id >= 0) {
1041                 clear_bit(iommu->seq_id, dmar_seq_ids);
1042                 iommu->seq_id = -1;
1043         }
1044 }
1045
1046 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1047 {
1048         struct intel_iommu *iommu;
1049         u32 ver, sts;
1050         int agaw = -1;
1051         int msagaw = -1;
1052         int err;
1053
1054         if (!drhd->reg_base_addr) {
1055                 warn_invalid_dmar(0, "");
1056                 return -EINVAL;
1057         }
1058
1059         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1060         if (!iommu)
1061                 return -ENOMEM;
1062
1063         if (dmar_alloc_seq_id(iommu) < 0) {
1064                 pr_err("Failed to allocate seq_id\n");
1065                 err = -ENOSPC;
1066                 goto error;
1067         }
1068
1069         err = map_iommu(iommu, drhd->reg_base_addr);
1070         if (err) {
1071                 pr_err("Failed to map %s\n", iommu->name);
1072                 goto error_free_seq_id;
1073         }
1074
1075         err = -EINVAL;
1076         if (cap_sagaw(iommu->cap) == 0) {
1077                 pr_info("%s: No supported address widths. Not attempting DMA translation.\n",
1078                         iommu->name);
1079                 drhd->ignored = 1;
1080         }
1081
1082         if (!drhd->ignored) {
1083                 agaw = iommu_calculate_agaw(iommu);
1084                 if (agaw < 0) {
1085                         pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1086                                iommu->seq_id);
1087                         drhd->ignored = 1;
1088                 }
1089         }
1090         if (!drhd->ignored) {
1091                 msagaw = iommu_calculate_max_sagaw(iommu);
1092                 if (msagaw < 0) {
1093                         pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1094                                iommu->seq_id);
1095                         drhd->ignored = 1;
1096                         agaw = -1;
1097                 }
1098         }
1099         iommu->agaw = agaw;
1100         iommu->msagaw = msagaw;
1101         iommu->segment = drhd->segment;
1102
1103         iommu->node = NUMA_NO_NODE;
1104
1105         ver = readl(iommu->reg + DMAR_VER_REG);
1106         pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1107                 iommu->name,
1108                 (unsigned long long)drhd->reg_base_addr,
1109                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1110                 (unsigned long long)iommu->cap,
1111                 (unsigned long long)iommu->ecap);
1112
1113         /* Reflect status in gcmd */
1114         sts = readl(iommu->reg + DMAR_GSTS_REG);
1115         if (sts & DMA_GSTS_IRES)
1116                 iommu->gcmd |= DMA_GCMD_IRE;
1117         if (sts & DMA_GSTS_TES)
1118                 iommu->gcmd |= DMA_GCMD_TE;
1119         if (sts & DMA_GSTS_QIES)
1120                 iommu->gcmd |= DMA_GCMD_QIE;
1121
1122         raw_spin_lock_init(&iommu->register_lock);
1123
1124         /*
1125          * This is only for hotplug; at boot time intel_iommu_enabled won't
1126          * be set yet. When intel_iommu_init() runs, it registers the units
1127          * present at boot time, then sets intel_iommu_enabled.
1128          */
1129         if (intel_iommu_enabled && !drhd->ignored) {
1130                 err = iommu_device_sysfs_add(&iommu->iommu, NULL,
1131                                              intel_iommu_groups,
1132                                              "%s", iommu->name);
1133                 if (err)
1134                         goto err_unmap;
1135
1136                 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
1137
1138                 err = iommu_device_register(&iommu->iommu);
1139                 if (err)
1140                         goto err_unmap;
1141         }
1142
1143         drhd->iommu = iommu;
1144         iommu->drhd = drhd;
1145
1146         return 0;
1147
1148 err_unmap:
1149         unmap_iommu(iommu);
1150 error_free_seq_id:
1151         dmar_free_seq_id(iommu);
1152 error:
1153         kfree(iommu);
1154         return err;
1155 }
1156
1157 static void free_iommu(struct intel_iommu *iommu)
1158 {
1159         if (intel_iommu_enabled && !iommu->drhd->ignored) {
1160                 iommu_device_unregister(&iommu->iommu);
1161                 iommu_device_sysfs_remove(&iommu->iommu);
1162         }
1163
1164         if (iommu->irq) {
1165                 if (iommu->pr_irq) {
1166                         free_irq(iommu->pr_irq, iommu);
1167                         dmar_free_hwirq(iommu->pr_irq);
1168                         iommu->pr_irq = 0;
1169                 }
1170                 free_irq(iommu->irq, iommu);
1171                 dmar_free_hwirq(iommu->irq);
1172                 iommu->irq = 0;
1173         }
1174
1175         if (iommu->qi) {
1176                 free_page((unsigned long)iommu->qi->desc);
1177                 kfree(iommu->qi->desc_status);
1178                 kfree(iommu->qi);
1179         }
1180
1181         if (iommu->reg)
1182                 unmap_iommu(iommu);
1183
1184         dmar_free_seq_id(iommu);
1185         kfree(iommu);
1186 }
1187
1188 /*
1189  * Reclaim all the submitted descriptors which have completed its work.
1190  */
1191 static inline void reclaim_free_desc(struct q_inval *qi)
1192 {
1193         while (qi->desc_status[qi->free_tail] == QI_DONE ||
1194                qi->desc_status[qi->free_tail] == QI_ABORT) {
1195                 qi->desc_status[qi->free_tail] = QI_FREE;
1196                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1197                 qi->free_cnt++;
1198         }
1199 }
1200
1201 static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
1202 {
1203         u32 fault;
1204         int head, tail;
1205         struct q_inval *qi = iommu->qi;
1206         int shift = qi_shift(iommu);
1207
1208         if (qi->desc_status[wait_index] == QI_ABORT)
1209                 return -EAGAIN;
1210
1211         fault = readl(iommu->reg + DMAR_FSTS_REG);
1212
1213         /*
1214          * If IQE happens, the head points to the descriptor associated
1215          * with the error. No new descriptors are fetched until the IQE
1216          * is cleared.
1217          */
1218         if (fault & DMA_FSTS_IQE) {
1219                 head = readl(iommu->reg + DMAR_IQH_REG);
1220                 if ((head >> shift) == index) {
1221                         struct qi_desc *desc = qi->desc + head;
1222
1223                         /*
1224                          * desc->qw2 and desc->qw3 are either reserved or
1225                          * used by software as private data. We won't print
1226                          * out these two qw's for security consideration.
1227                          */
1228                         pr_err("VT-d detected invalid descriptor: qw0 = %llx, qw1 = %llx\n",
1229                                (unsigned long long)desc->qw0,
1230                                (unsigned long long)desc->qw1);
1231                         memcpy(desc, qi->desc + (wait_index << shift),
1232                                1 << shift);
1233                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1234                         return -EINVAL;
1235                 }
1236         }
1237
1238         /*
1239          * If ITE happens, all pending wait_desc commands are aborted.
1240          * No new descriptors are fetched until the ITE is cleared.
1241          */
1242         if (fault & DMA_FSTS_ITE) {
1243                 head = readl(iommu->reg + DMAR_IQH_REG);
1244                 head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1245                 head |= 1;
1246                 tail = readl(iommu->reg + DMAR_IQT_REG);
1247                 tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1248
1249                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1250
1251                 do {
1252                         if (qi->desc_status[head] == QI_IN_USE)
1253                                 qi->desc_status[head] = QI_ABORT;
1254                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1255                 } while (head != tail);
1256
1257                 if (qi->desc_status[wait_index] == QI_ABORT)
1258                         return -EAGAIN;
1259         }
1260
1261         if (fault & DMA_FSTS_ICE)
1262                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1263
1264         return 0;
1265 }
1266
1267 /*
1268  * Function to submit invalidation descriptors of all types to the queued
1269  * invalidation interface(QI). Multiple descriptors can be submitted at a
1270  * time, a wait descriptor will be appended to each submission to ensure
1271  * hardware has completed the invalidation before return. Wait descriptors
1272  * can be part of the submission but it will not be polled for completion.
1273  */
1274 int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
1275                    unsigned int count, unsigned long options)
1276 {
1277         struct q_inval *qi = iommu->qi;
1278         struct qi_desc wait_desc;
1279         int wait_index, index;
1280         unsigned long flags;
1281         int offset, shift;
1282         int rc, i;
1283
1284         if (!qi)
1285                 return 0;
1286
1287 restart:
1288         rc = 0;
1289
1290         raw_spin_lock_irqsave(&qi->q_lock, flags);
1291         /*
1292          * Check if we have enough empty slots in the queue to submit,
1293          * the calculation is based on:
1294          * # of desc + 1 wait desc + 1 space between head and tail
1295          */
1296         while (qi->free_cnt < count + 2) {
1297                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1298                 cpu_relax();
1299                 raw_spin_lock_irqsave(&qi->q_lock, flags);
1300         }
1301
1302         index = qi->free_head;
1303         wait_index = (index + count) % QI_LENGTH;
1304         shift = qi_shift(iommu);
1305
1306         for (i = 0; i < count; i++) {
1307                 offset = ((index + i) % QI_LENGTH) << shift;
1308                 memcpy(qi->desc + offset, &desc[i], 1 << shift);
1309                 qi->desc_status[(index + i) % QI_LENGTH] = QI_IN_USE;
1310         }
1311         qi->desc_status[wait_index] = QI_IN_USE;
1312
1313         wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
1314                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1315         if (options & QI_OPT_WAIT_DRAIN)
1316                 wait_desc.qw0 |= QI_IWD_PRQ_DRAIN;
1317         wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
1318         wait_desc.qw2 = 0;
1319         wait_desc.qw3 = 0;
1320
1321         offset = wait_index << shift;
1322         memcpy(qi->desc + offset, &wait_desc, 1 << shift);
1323
1324         qi->free_head = (qi->free_head + count + 1) % QI_LENGTH;
1325         qi->free_cnt -= count + 1;
1326
1327         /*
1328          * update the HW tail register indicating the presence of
1329          * new descriptors.
1330          */
1331         writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);
1332
1333         while (qi->desc_status[wait_index] != QI_DONE) {
1334                 /*
1335                  * We will leave the interrupts disabled, to prevent interrupt
1336                  * context to queue another cmd while a cmd is already submitted
1337                  * and waiting for completion on this cpu. This is to avoid
1338                  * a deadlock where the interrupt context can wait indefinitely
1339                  * for free slots in the queue.
1340                  */
1341                 rc = qi_check_fault(iommu, index, wait_index);
1342                 if (rc)
1343                         break;
1344
1345                 raw_spin_unlock(&qi->q_lock);
1346                 cpu_relax();
1347                 raw_spin_lock(&qi->q_lock);
1348         }
1349
1350         for (i = 0; i < count; i++)
1351                 qi->desc_status[(index + i) % QI_LENGTH] = QI_DONE;
1352
1353         reclaim_free_desc(qi);
1354         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1355
1356         if (rc == -EAGAIN)
1357                 goto restart;
1358
1359         return rc;
1360 }
1361
1362 /*
1363  * Flush the global interrupt entry cache.
1364  */
1365 void qi_global_iec(struct intel_iommu *iommu)
1366 {
1367         struct qi_desc desc;
1368
1369         desc.qw0 = QI_IEC_TYPE;
1370         desc.qw1 = 0;
1371         desc.qw2 = 0;
1372         desc.qw3 = 0;
1373
1374         /* should never fail */
1375         qi_submit_sync(iommu, &desc, 1, 0);
1376 }
1377
1378 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1379                       u64 type)
1380 {
1381         struct qi_desc desc;
1382
1383         desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1384                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1385         desc.qw1 = 0;
1386         desc.qw2 = 0;
1387         desc.qw3 = 0;
1388
1389         qi_submit_sync(iommu, &desc, 1, 0);
1390 }
1391
1392 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1393                     unsigned int size_order, u64 type)
1394 {
1395         u8 dw = 0, dr = 0;
1396
1397         struct qi_desc desc;
1398         int ih = 0;
1399
1400         if (cap_write_drain(iommu->cap))
1401                 dw = 1;
1402
1403         if (cap_read_drain(iommu->cap))
1404                 dr = 1;
1405
1406         desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1407                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1408         desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1409                 | QI_IOTLB_AM(size_order);
1410         desc.qw2 = 0;
1411         desc.qw3 = 0;
1412
1413         qi_submit_sync(iommu, &desc, 1, 0);
1414 }
1415
1416 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1417                         u16 qdep, u64 addr, unsigned mask)
1418 {
1419         struct qi_desc desc;
1420
1421         if (mask) {
1422                 addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1423                 desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1424         } else
1425                 desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
1426
1427         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1428                 qdep = 0;
1429
1430         desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1431                    QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1432         desc.qw2 = 0;
1433         desc.qw3 = 0;
1434
1435         qi_submit_sync(iommu, &desc, 1, 0);
1436 }
1437
1438 /* PASID-based IOTLB invalidation */
1439 void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
1440                      unsigned long npages, bool ih)
1441 {
1442         struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
1443
1444         /*
1445          * npages == -1 means a PASID-selective invalidation, otherwise,
1446          * a positive value for Page-selective-within-PASID invalidation.
1447          * 0 is not a valid input.
1448          */
1449         if (WARN_ON(!npages)) {
1450                 pr_err("Invalid input npages = %ld\n", npages);
1451                 return;
1452         }
1453
1454         if (npages == -1) {
1455                 desc.qw0 = QI_EIOTLB_PASID(pasid) |
1456                                 QI_EIOTLB_DID(did) |
1457                                 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
1458                                 QI_EIOTLB_TYPE;
1459                 desc.qw1 = 0;
1460         } else {
1461                 int mask = ilog2(__roundup_pow_of_two(npages));
1462                 unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
1463
1464                 if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
1465                         addr = ALIGN_DOWN(addr, align);
1466
1467                 desc.qw0 = QI_EIOTLB_PASID(pasid) |
1468                                 QI_EIOTLB_DID(did) |
1469                                 QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
1470                                 QI_EIOTLB_TYPE;
1471                 desc.qw1 = QI_EIOTLB_ADDR(addr) |
1472                                 QI_EIOTLB_IH(ih) |
1473                                 QI_EIOTLB_AM(mask);
1474         }
1475
1476         qi_submit_sync(iommu, &desc, 1, 0);
1477 }
1478
1479 /* PASID-based device IOTLB Invalidate */
1480 void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1481                               u32 pasid,  u16 qdep, u64 addr, unsigned int size_order)
1482 {
1483         unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);
1484         struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1485
1486         desc.qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
1487                 QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |
1488                 QI_DEV_IOTLB_PFSID(pfsid);
1489
1490         /*
1491          * If S bit is 0, we only flush a single page. If S bit is set,
1492          * The least significant zero bit indicates the invalidation address
1493          * range. VT-d spec 6.5.2.6.
1494          * e.g. address bit 12[0] indicates 8KB, 13[0] indicates 16KB.
1495          * size order = 0 is PAGE_SIZE 4KB
1496          * Max Invs Pending (MIP) is set to 0 for now until we have DIT in
1497          * ECAP.
1498          */
1499         if (addr & GENMASK_ULL(size_order + VTD_PAGE_SHIFT, 0))
1500                 pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",
1501                                     addr, size_order);
1502
1503         /* Take page address */
1504         desc.qw1 = QI_DEV_EIOTLB_ADDR(addr);
1505
1506         if (size_order) {
1507                 /*
1508                  * Existing 0s in address below size_order may be the least
1509                  * significant bit, we must set them to 1s to avoid having
1510                  * smaller size than desired.
1511                  */
1512                 desc.qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,
1513                                         VTD_PAGE_SHIFT);
1514                 /* Clear size_order bit to indicate size */
1515                 desc.qw1 &= ~mask;
1516                 /* Set the S bit to indicate flushing more than 1 page */
1517                 desc.qw1 |= QI_DEV_EIOTLB_SIZE;
1518         }
1519
1520         qi_submit_sync(iommu, &desc, 1, 0);
1521 }
1522
1523 void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,
1524                           u64 granu, u32 pasid)
1525 {
1526         struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1527
1528         desc.qw0 = QI_PC_PASID(pasid) | QI_PC_DID(did) |
1529                         QI_PC_GRAN(granu) | QI_PC_TYPE;
1530         qi_submit_sync(iommu, &desc, 1, 0);
1531 }
1532
1533 /*
1534  * Disable Queued Invalidation interface.
1535  */
1536 void dmar_disable_qi(struct intel_iommu *iommu)
1537 {
1538         unsigned long flags;
1539         u32 sts;
1540         cycles_t start_time = get_cycles();
1541
1542         if (!ecap_qis(iommu->ecap))
1543                 return;
1544
1545         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1546
1547         sts =  readl(iommu->reg + DMAR_GSTS_REG);
1548         if (!(sts & DMA_GSTS_QIES))
1549                 goto end;
1550
1551         /*
1552          * Give a chance to HW to complete the pending invalidation requests.
1553          */
1554         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1555                 readl(iommu->reg + DMAR_IQH_REG)) &&
1556                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1557                 cpu_relax();
1558
1559         iommu->gcmd &= ~DMA_GCMD_QIE;
1560         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1561
1562         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1563                       !(sts & DMA_GSTS_QIES), sts);
1564 end:
1565         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1566 }
1567
1568 /*
1569  * Enable queued invalidation.
1570  */
1571 static void __dmar_enable_qi(struct intel_iommu *iommu)
1572 {
1573         u32 sts;
1574         unsigned long flags;
1575         struct q_inval *qi = iommu->qi;
1576         u64 val = virt_to_phys(qi->desc);
1577
1578         qi->free_head = qi->free_tail = 0;
1579         qi->free_cnt = QI_LENGTH;
1580
1581         /*
1582          * Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
1583          * is present.
1584          */
1585         if (ecap_smts(iommu->ecap))
1586                 val |= (1 << 11) | 1;
1587
1588         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1589
1590         /* write zero to the tail reg */
1591         writel(0, iommu->reg + DMAR_IQT_REG);
1592
1593         dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
1594
1595         iommu->gcmd |= DMA_GCMD_QIE;
1596         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1597
1598         /* Make sure hardware complete it */
1599         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1600
1601         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1602 }
1603
1604 /*
1605  * Enable Queued Invalidation interface. This is a must to support
1606  * interrupt-remapping. Also used by DMA-remapping, which replaces
1607  * register based IOTLB invalidation.
1608  */
1609 int dmar_enable_qi(struct intel_iommu *iommu)
1610 {
1611         struct q_inval *qi;
1612         struct page *desc_page;
1613
1614         if (!ecap_qis(iommu->ecap))
1615                 return -ENOENT;
1616
1617         /*
1618          * queued invalidation is already setup and enabled.
1619          */
1620         if (iommu->qi)
1621                 return 0;
1622
1623         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1624         if (!iommu->qi)
1625                 return -ENOMEM;
1626
1627         qi = iommu->qi;
1628
1629         /*
1630          * Need two pages to accommodate 256 descriptors of 256 bits each
1631          * if the remapping hardware supports scalable mode translation.
1632          */
1633         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
1634                                      !!ecap_smts(iommu->ecap));
1635         if (!desc_page) {
1636                 kfree(qi);
1637                 iommu->qi = NULL;
1638                 return -ENOMEM;
1639         }
1640
1641         qi->desc = page_address(desc_page);
1642
1643         qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC);
1644         if (!qi->desc_status) {
1645                 free_page((unsigned long) qi->desc);
1646                 kfree(qi);
1647                 iommu->qi = NULL;
1648                 return -ENOMEM;
1649         }
1650
1651         raw_spin_lock_init(&qi->q_lock);
1652
1653         __dmar_enable_qi(iommu);
1654
1655         return 0;
1656 }
1657
1658 /* iommu interrupt handling. Most stuff are MSI-like. */
1659
1660 enum faulttype {
1661         DMA_REMAP,
1662         INTR_REMAP,
1663         UNKNOWN,
1664 };
1665
1666 static const char *dma_remap_fault_reasons[] =
1667 {
1668         "Software",
1669         "Present bit in root entry is clear",
1670         "Present bit in context entry is clear",
1671         "Invalid context entry",
1672         "Access beyond MGAW",
1673         "PTE Write access is not set",
1674         "PTE Read access is not set",
1675         "Next page table ptr is invalid",
1676         "Root table address invalid",
1677         "Context table ptr is invalid",
1678         "non-zero reserved fields in RTP",
1679         "non-zero reserved fields in CTP",
1680         "non-zero reserved fields in PTE",
1681         "PCE for translation request specifies blocking",
1682 };
1683
1684 static const char * const dma_remap_sm_fault_reasons[] = {
1685         "SM: Invalid Root Table Address",
1686         "SM: TTM 0 for request with PASID",
1687         "SM: TTM 0 for page group request",
1688         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x33-0x37 */
1689         "SM: Error attempting to access Root Entry",
1690         "SM: Present bit in Root Entry is clear",
1691         "SM: Non-zero reserved field set in Root Entry",
1692         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x3B-0x3F */
1693         "SM: Error attempting to access Context Entry",
1694         "SM: Present bit in Context Entry is clear",
1695         "SM: Non-zero reserved field set in the Context Entry",
1696         "SM: Invalid Context Entry",
1697         "SM: DTE field in Context Entry is clear",
1698         "SM: PASID Enable field in Context Entry is clear",
1699         "SM: PASID is larger than the max in Context Entry",
1700         "SM: PRE field in Context-Entry is clear",
1701         "SM: RID_PASID field error in Context-Entry",
1702         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x49-0x4F */
1703         "SM: Error attempting to access the PASID Directory Entry",
1704         "SM: Present bit in Directory Entry is clear",
1705         "SM: Non-zero reserved field set in PASID Directory Entry",
1706         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x53-0x57 */
1707         "SM: Error attempting to access PASID Table Entry",
1708         "SM: Present bit in PASID Table Entry is clear",
1709         "SM: Non-zero reserved field set in PASID Table Entry",
1710         "SM: Invalid Scalable-Mode PASID Table Entry",
1711         "SM: ERE field is clear in PASID Table Entry",
1712         "SM: SRE field is clear in PASID Table Entry",
1713         "Unknown", "Unknown",/* 0x5E-0x5F */
1714         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x60-0x67 */
1715         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x68-0x6F */
1716         "SM: Error attempting to access first-level paging entry",
1717         "SM: Present bit in first-level paging entry is clear",
1718         "SM: Non-zero reserved field set in first-level paging entry",
1719         "SM: Error attempting to access FL-PML4 entry",
1720         "SM: First-level entry address beyond MGAW in Nested translation",
1721         "SM: Read permission error in FL-PML4 entry in Nested translation",
1722         "SM: Read permission error in first-level paging entry in Nested translation",
1723         "SM: Write permission error in first-level paging entry in Nested translation",
1724         "SM: Error attempting to access second-level paging entry",
1725         "SM: Read/Write permission error in second-level paging entry",
1726         "SM: Non-zero reserved field set in second-level paging entry",
1727         "SM: Invalid second-level page table pointer",
1728         "SM: A/D bit update needed in second-level entry when set up in no snoop",
1729         "Unknown", "Unknown", "Unknown", /* 0x7D-0x7F */
1730         "SM: Address in first-level translation is not canonical",
1731         "SM: U/S set 0 for first-level translation with user privilege",
1732         "SM: No execute permission for request with PASID and ER=1",
1733         "SM: Address beyond the DMA hardware max",
1734         "SM: Second-level entry address beyond the max",
1735         "SM: No write permission for Write/AtomicOp request",
1736         "SM: No read permission for Read/AtomicOp request",
1737         "SM: Invalid address-interrupt address",
1738         "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x88-0x8F */
1739         "SM: A/D bit update needed in first-level entry when set up in no snoop",
1740 };
1741
1742 static const char *irq_remap_fault_reasons[] =
1743 {
1744         "Detected reserved fields in the decoded interrupt-remapped request",
1745         "Interrupt index exceeded the interrupt-remapping table size",
1746         "Present field in the IRTE entry is clear",
1747         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1748         "Detected reserved fields in the IRTE entry",
1749         "Blocked a compatibility format interrupt request",
1750         "Blocked an interrupt request due to source-id verification failure",
1751 };
1752
1753 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1754 {
1755         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1756                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1757                 *fault_type = INTR_REMAP;
1758                 return irq_remap_fault_reasons[fault_reason - 0x20];
1759         } else if (fault_reason >= 0x30 && (fault_reason - 0x30 <
1760                         ARRAY_SIZE(dma_remap_sm_fault_reasons))) {
1761                 *fault_type = DMA_REMAP;
1762                 return dma_remap_sm_fault_reasons[fault_reason - 0x30];
1763         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1764                 *fault_type = DMA_REMAP;
1765                 return dma_remap_fault_reasons[fault_reason];
1766         } else {
1767                 *fault_type = UNKNOWN;
1768                 return "Unknown";
1769         }
1770 }
1771
1772
1773 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1774 {
1775         if (iommu->irq == irq)
1776                 return DMAR_FECTL_REG;
1777         else if (iommu->pr_irq == irq)
1778                 return DMAR_PECTL_REG;
1779         else
1780                 BUG();
1781 }
1782
1783 void dmar_msi_unmask(struct irq_data *data)
1784 {
1785         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1786         int reg = dmar_msi_reg(iommu, data->irq);
1787         unsigned long flag;
1788
1789         /* unmask it */
1790         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1791         writel(0, iommu->reg + reg);
1792         /* Read a reg to force flush the post write */
1793         readl(iommu->reg + reg);
1794         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1795 }
1796
1797 void dmar_msi_mask(struct irq_data *data)
1798 {
1799         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1800         int reg = dmar_msi_reg(iommu, data->irq);
1801         unsigned long flag;
1802
1803         /* mask it */
1804         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1805         writel(DMA_FECTL_IM, iommu->reg + reg);
1806         /* Read a reg to force flush the post write */
1807         readl(iommu->reg + reg);
1808         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1809 }
1810
1811 void dmar_msi_write(int irq, struct msi_msg *msg)
1812 {
1813         struct intel_iommu *iommu = irq_get_handler_data(irq);
1814         int reg = dmar_msi_reg(iommu, irq);
1815         unsigned long flag;
1816
1817         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1818         writel(msg->data, iommu->reg + reg + 4);
1819         writel(msg->address_lo, iommu->reg + reg + 8);
1820         writel(msg->address_hi, iommu->reg + reg + 12);
1821         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1822 }
1823
1824 void dmar_msi_read(int irq, struct msi_msg *msg)
1825 {
1826         struct intel_iommu *iommu = irq_get_handler_data(irq);
1827         int reg = dmar_msi_reg(iommu, irq);
1828         unsigned long flag;
1829
1830         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1831         msg->data = readl(iommu->reg + reg + 4);
1832         msg->address_lo = readl(iommu->reg + reg + 8);
1833         msg->address_hi = readl(iommu->reg + reg + 12);
1834         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1835 }
1836
1837 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1838                 u8 fault_reason, u32 pasid, u16 source_id,
1839                 unsigned long long addr)
1840 {
1841         const char *reason;
1842         int fault_type;
1843
1844         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1845
1846         if (fault_type == INTR_REMAP)
1847                 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n",
1848                         source_id >> 8, PCI_SLOT(source_id & 0xFF),
1849                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1850                         fault_reason, reason);
1851         else
1852                 pr_err("[%s] Request device [%02x:%02x.%d] PASID %x fault addr %llx [fault reason %02d] %s\n",
1853                        type ? "DMA Read" : "DMA Write",
1854                        source_id >> 8, PCI_SLOT(source_id & 0xFF),
1855                        PCI_FUNC(source_id & 0xFF), pasid, addr,
1856                        fault_reason, reason);
1857         return 0;
1858 }
1859
1860 #define PRIMARY_FAULT_REG_LEN (16)
1861 irqreturn_t dmar_fault(int irq, void *dev_id)
1862 {
1863         struct intel_iommu *iommu = dev_id;
1864         int reg, fault_index;
1865         u32 fault_status;
1866         unsigned long flag;
1867         static DEFINE_RATELIMIT_STATE(rs,
1868                                       DEFAULT_RATELIMIT_INTERVAL,
1869                                       DEFAULT_RATELIMIT_BURST);
1870
1871         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1872         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1873         if (fault_status && __ratelimit(&rs))
1874                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1875
1876         /* TBD: ignore advanced fault log currently */
1877         if (!(fault_status & DMA_FSTS_PPF))
1878                 goto unlock_exit;
1879
1880         fault_index = dma_fsts_fault_record_index(fault_status);
1881         reg = cap_fault_reg_offset(iommu->cap);
1882         while (1) {
1883                 /* Disable printing, simply clear the fault when ratelimited */
1884                 bool ratelimited = !__ratelimit(&rs);
1885                 u8 fault_reason;
1886                 u16 source_id;
1887                 u64 guest_addr;
1888                 u32 pasid;
1889                 int type;
1890                 u32 data;
1891                 bool pasid_present;
1892
1893                 /* highest 32 bits */
1894                 data = readl(iommu->reg + reg +
1895                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1896                 if (!(data & DMA_FRCD_F))
1897                         break;
1898
1899                 if (!ratelimited) {
1900                         fault_reason = dma_frcd_fault_reason(data);
1901                         type = dma_frcd_type(data);
1902
1903                         pasid = dma_frcd_pasid_value(data);
1904                         data = readl(iommu->reg + reg +
1905                                      fault_index * PRIMARY_FAULT_REG_LEN + 8);
1906                         source_id = dma_frcd_source_id(data);
1907
1908                         pasid_present = dma_frcd_pasid_present(data);
1909                         guest_addr = dmar_readq(iommu->reg + reg +
1910                                         fault_index * PRIMARY_FAULT_REG_LEN);
1911                         guest_addr = dma_frcd_page_addr(guest_addr);
1912                 }
1913
1914                 /* clear the fault */
1915                 writel(DMA_FRCD_F, iommu->reg + reg +
1916                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1917
1918                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1919
1920                 if (!ratelimited)
1921                         /* Using pasid -1 if pasid is not present */
1922                         dmar_fault_do_one(iommu, type, fault_reason,
1923                                           pasid_present ? pasid : -1,
1924                                           source_id, guest_addr);
1925
1926                 fault_index++;
1927                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1928                         fault_index = 0;
1929                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1930         }
1931
1932         writel(DMA_FSTS_PFO | DMA_FSTS_PPF | DMA_FSTS_PRO,
1933                iommu->reg + DMAR_FSTS_REG);
1934
1935 unlock_exit:
1936         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1937         return IRQ_HANDLED;
1938 }
1939
1940 int dmar_set_interrupt(struct intel_iommu *iommu)
1941 {
1942         int irq, ret;
1943
1944         /*
1945          * Check if the fault interrupt is already initialized.
1946          */
1947         if (iommu->irq)
1948                 return 0;
1949
1950         irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1951         if (irq > 0) {
1952                 iommu->irq = irq;
1953         } else {
1954                 pr_err("No free IRQ vectors\n");
1955                 return -EINVAL;
1956         }
1957
1958         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1959         if (ret)
1960                 pr_err("Can't request irq\n");
1961         return ret;
1962 }
1963
1964 int __init enable_drhd_fault_handling(void)
1965 {
1966         struct dmar_drhd_unit *drhd;
1967         struct intel_iommu *iommu;
1968
1969         /*
1970          * Enable fault control interrupt.
1971          */
1972         for_each_iommu(iommu, drhd) {
1973                 u32 fault_status;
1974                 int ret = dmar_set_interrupt(iommu);
1975
1976                 if (ret) {
1977                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1978                                (unsigned long long)drhd->reg_base_addr, ret);
1979                         return -1;
1980                 }
1981
1982                 /*
1983                  * Clear any previous faults.
1984                  */
1985                 dmar_fault(iommu->irq, iommu);
1986                 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1987                 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1988         }
1989
1990         return 0;
1991 }
1992
1993 /*
1994  * Re-enable Queued Invalidation interface.
1995  */
1996 int dmar_reenable_qi(struct intel_iommu *iommu)
1997 {
1998         if (!ecap_qis(iommu->ecap))
1999                 return -ENOENT;
2000
2001         if (!iommu->qi)
2002                 return -ENOENT;
2003
2004         /*
2005          * First disable queued invalidation.
2006          */
2007         dmar_disable_qi(iommu);
2008         /*
2009          * Then enable queued invalidation again. Since there is no pending
2010          * invalidation requests now, it's safe to re-enable queued
2011          * invalidation.
2012          */
2013         __dmar_enable_qi(iommu);
2014
2015         return 0;
2016 }
2017
2018 /*
2019  * Check interrupt remapping support in DMAR table description.
2020  */
2021 int __init dmar_ir_support(void)
2022 {
2023         struct acpi_table_dmar *dmar;
2024         dmar = (struct acpi_table_dmar *)dmar_tbl;
2025         if (!dmar)
2026                 return 0;
2027         return dmar->flags & 0x1;
2028 }
2029
2030 /* Check whether DMAR units are in use */
2031 static inline bool dmar_in_use(void)
2032 {
2033         return irq_remapping_enabled || intel_iommu_enabled;
2034 }
2035
2036 static int __init dmar_free_unused_resources(void)
2037 {
2038         struct dmar_drhd_unit *dmaru, *dmaru_n;
2039
2040         if (dmar_in_use())
2041                 return 0;
2042
2043         if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
2044                 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
2045
2046         down_write(&dmar_global_lock);
2047         list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
2048                 list_del(&dmaru->list);
2049                 dmar_free_drhd(dmaru);
2050         }
2051         up_write(&dmar_global_lock);
2052
2053         return 0;
2054 }
2055
2056 late_initcall(dmar_free_unused_resources);
2057 IOMMU_INIT_POST(detect_intel_iommu);
2058
2059 /*
2060  * DMAR Hotplug Support
2061  * For more details, please refer to Intel(R) Virtualization Technology
2062  * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
2063  * "Remapping Hardware Unit Hot Plug".
2064  */
2065 static guid_t dmar_hp_guid =
2066         GUID_INIT(0xD8C1A3A6, 0xBE9B, 0x4C9B,
2067                   0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF);
2068
2069 /*
2070  * Currently there's only one revision and BIOS will not check the revision id,
2071  * so use 0 for safety.
2072  */
2073 #define DMAR_DSM_REV_ID                 0
2074 #define DMAR_DSM_FUNC_DRHD              1
2075 #define DMAR_DSM_FUNC_ATSR              2
2076 #define DMAR_DSM_FUNC_RHSA              3
2077
2078 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
2079 {
2080         return acpi_check_dsm(handle, &dmar_hp_guid, DMAR_DSM_REV_ID, 1 << func);
2081 }
2082
2083 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
2084                                   dmar_res_handler_t handler, void *arg)
2085 {
2086         int ret = -ENODEV;
2087         union acpi_object *obj;
2088         struct acpi_dmar_header *start;
2089         struct dmar_res_callback callback;
2090         static int res_type[] = {
2091                 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
2092                 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
2093                 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
2094         };
2095
2096         if (!dmar_detect_dsm(handle, func))
2097                 return 0;
2098
2099         obj = acpi_evaluate_dsm_typed(handle, &dmar_hp_guid, DMAR_DSM_REV_ID,
2100                                       func, NULL, ACPI_TYPE_BUFFER);
2101         if (!obj)
2102                 return -ENODEV;
2103
2104         memset(&callback, 0, sizeof(callback));
2105         callback.cb[res_type[func]] = handler;
2106         callback.arg[res_type[func]] = arg;
2107         start = (struct acpi_dmar_header *)obj->buffer.pointer;
2108         ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
2109
2110         ACPI_FREE(obj);
2111
2112         return ret;
2113 }
2114
2115 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
2116 {
2117         int ret;
2118         struct dmar_drhd_unit *dmaru;
2119
2120         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2121         if (!dmaru)
2122                 return -ENODEV;
2123
2124         ret = dmar_ir_hotplug(dmaru, true);
2125         if (ret == 0)
2126                 ret = dmar_iommu_hotplug(dmaru, true);
2127
2128         return ret;
2129 }
2130
2131 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
2132 {
2133         int i, ret;
2134         struct device *dev;
2135         struct dmar_drhd_unit *dmaru;
2136
2137         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2138         if (!dmaru)
2139                 return 0;
2140
2141         /*
2142          * All PCI devices managed by this unit should have been destroyed.
2143          */
2144         if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
2145                 for_each_active_dev_scope(dmaru->devices,
2146                                           dmaru->devices_cnt, i, dev)
2147                         return -EBUSY;
2148         }
2149
2150         ret = dmar_ir_hotplug(dmaru, false);
2151         if (ret == 0)
2152                 ret = dmar_iommu_hotplug(dmaru, false);
2153
2154         return ret;
2155 }
2156
2157 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
2158 {
2159         struct dmar_drhd_unit *dmaru;
2160
2161         dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2162         if (dmaru) {
2163                 list_del_rcu(&dmaru->list);
2164                 synchronize_rcu();
2165                 dmar_free_drhd(dmaru);
2166         }
2167
2168         return 0;
2169 }
2170
2171 static int dmar_hotplug_insert(acpi_handle handle)
2172 {
2173         int ret;
2174         int drhd_count = 0;
2175
2176         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2177                                      &dmar_validate_one_drhd, (void *)1);
2178         if (ret)
2179                 goto out;
2180
2181         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2182                                      &dmar_parse_one_drhd, (void *)&drhd_count);
2183         if (ret == 0 && drhd_count == 0) {
2184                 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
2185                 goto out;
2186         } else if (ret) {
2187                 goto release_drhd;
2188         }
2189
2190         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
2191                                      &dmar_parse_one_rhsa, NULL);
2192         if (ret)
2193                 goto release_drhd;
2194
2195         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2196                                      &dmar_parse_one_atsr, NULL);
2197         if (ret)
2198                 goto release_atsr;
2199
2200         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2201                                      &dmar_hp_add_drhd, NULL);
2202         if (!ret)
2203                 return 0;
2204
2205         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2206                                &dmar_hp_remove_drhd, NULL);
2207 release_atsr:
2208         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2209                                &dmar_release_one_atsr, NULL);
2210 release_drhd:
2211         dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2212                                &dmar_hp_release_drhd, NULL);
2213 out:
2214         return ret;
2215 }
2216
2217 static int dmar_hotplug_remove(acpi_handle handle)
2218 {
2219         int ret;
2220
2221         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2222                                      &dmar_check_one_atsr, NULL);
2223         if (ret)
2224                 return ret;
2225
2226         ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2227                                      &dmar_hp_remove_drhd, NULL);
2228         if (ret == 0) {
2229                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2230                                                &dmar_release_one_atsr, NULL));
2231                 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2232                                                &dmar_hp_release_drhd, NULL));
2233         } else {
2234                 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2235                                        &dmar_hp_add_drhd, NULL);
2236         }
2237
2238         return ret;
2239 }
2240
2241 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
2242                                        void *context, void **retval)
2243 {
2244         acpi_handle *phdl = retval;
2245
2246         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2247                 *phdl = handle;
2248                 return AE_CTRL_TERMINATE;
2249         }
2250
2251         return AE_OK;
2252 }
2253
2254 static int dmar_device_hotplug(acpi_handle handle, bool insert)
2255 {
2256         int ret;
2257         acpi_handle tmp = NULL;
2258         acpi_status status;
2259
2260         if (!dmar_in_use())
2261                 return 0;
2262
2263         if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2264                 tmp = handle;
2265         } else {
2266                 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
2267                                              ACPI_UINT32_MAX,
2268                                              dmar_get_dsm_handle,
2269                                              NULL, NULL, &tmp);
2270                 if (ACPI_FAILURE(status)) {
2271                         pr_warn("Failed to locate _DSM method.\n");
2272                         return -ENXIO;
2273                 }
2274         }
2275         if (tmp == NULL)
2276                 return 0;
2277
2278         down_write(&dmar_global_lock);
2279         if (insert)
2280                 ret = dmar_hotplug_insert(tmp);
2281         else
2282                 ret = dmar_hotplug_remove(tmp);
2283         up_write(&dmar_global_lock);
2284
2285         return ret;
2286 }
2287
2288 int dmar_device_add(acpi_handle handle)
2289 {
2290         return dmar_device_hotplug(handle, true);
2291 }
2292
2293 int dmar_device_remove(acpi_handle handle)
2294 {
2295         return dmar_device_hotplug(handle, false);
2296 }
2297
2298 /*
2299  * dmar_platform_optin - Is %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in DMAR table
2300  *
2301  * Returns true if the platform has %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in
2302  * the ACPI DMAR table. This means that the platform boot firmware has made
2303  * sure no device can issue DMA outside of RMRR regions.
2304  */
2305 bool dmar_platform_optin(void)
2306 {
2307         struct acpi_table_dmar *dmar;
2308         acpi_status status;
2309         bool ret;
2310
2311         status = acpi_get_table(ACPI_SIG_DMAR, 0,
2312                                 (struct acpi_table_header **)&dmar);
2313         if (ACPI_FAILURE(status))
2314                 return false;
2315
2316         ret = !!(dmar->flags & DMAR_PLATFORM_OPT_IN);
2317         acpi_put_table((struct acpi_table_header *)dmar);
2318
2319         return ret;
2320 }
2321 EXPORT_SYMBOL_GPL(dmar_platform_optin);