drivers/iommu/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
  30
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/iova.h>
  34 #include <linux/intel-iommu.h>
  35 #include <linux/timer.h>
  36 #include <linux/irq.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/tboot.h>
  39 #include <linux/dmi.h>
  40 #include <linux/slab.h>
  41 #include <asm/irq_remapping.h>
  42 #include <asm/iommu_table.h>
  43
  44 #include "irq_remapping.h"
  45
  46 /*
  47  * Assumptions:
  48  * 1) The hotplug framework guarentees that DMAR unit will be hot-added
  49  *    before IO devices managed by that unit.
  50  * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
  51  *    after IO devices managed by that unit.
  52  * 3) Hotplug events are rare.
  53  *
  54  * Locking rules for DMA and interrupt remapping related global data structures:
  55  * 1) Use dmar_global_lock in process context
  56  * 2) Use RCU in interrupt context
  57  */
  58 DECLARE_RWSEM(dmar_global_lock);
  59 LIST_HEAD(dmar_drhd_units);
  60
  61 struct acpi_table_header * __initdata dmar_tbl;
  62 static acpi_size dmar_tbl_size;
  63 static int dmar_dev_scope_status = 1;
  64
  65 static int alloc_iommu(struct dmar_drhd_unit *drhd);
  66 static void free_iommu(struct intel_iommu *iommu);
  67
  68 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  69 {
  70         /*
  71          * add INCLUDE_ALL at the tail, so scan the list will find it at
  72          * the very end.
  73          */
  74         if (drhd->include_all)
  75                 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
  76         else
  77                 list_add_rcu(&drhd->list, &dmar_drhd_units);
  78 }
  79
  80 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
  81 {
  82         struct acpi_dmar_device_scope *scope;
  83
  84         *cnt = 0;
  85         while (start < end) {
  86                 scope = start;
  87                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ACPI ||
  88                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
  89                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
  90                         (*cnt)++;
  91                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
  92                         scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
  93                         pr_warn("Unsupported device scope\n");
  94                 }
  95                 start += scope->length;
  96         }
  97         if (*cnt == 0)
  98                 return NULL;
  99
 100         return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
 101 }
 102
 103 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
 104 {
 105         int i;
 106         struct device *tmp_dev;
 107
 108         if (*devices && *cnt) {
 109                 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
 110                         put_device(tmp_dev);
 111                 kfree(*devices);
 112         }
 113
 114         *devices = NULL;
 115         *cnt = 0;
 116 }
 117
 118 /* Optimize out kzalloc()/kfree() for normal cases */
 119 static char dmar_pci_notify_info_buf[64];
 120
 121 static struct dmar_pci_notify_info *
 122 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
 123 {
 124         int level = 0;
 125         size_t size;
 126         struct pci_dev *tmp;
 127         struct dmar_pci_notify_info *info;
 128
 129         BUG_ON(dev->is_virtfn);
 130
 131         /* Only generate path[] for device addition event */
 132         if (event == BUS_NOTIFY_ADD_DEVICE)
 133                 for (tmp = dev; tmp; tmp = tmp->bus->self)
 134                         level++;
 135
 136         size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
 137         if (size <= sizeof(dmar_pci_notify_info_buf)) {
 138                 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
 139         } else {
 140                 info = kzalloc(size, GFP_KERNEL);
 141                 if (!info) {
 142                         pr_warn("Out of memory when allocating notify_info "
 143                                 "for %s.\n", pci_name(dev));
 144                         if (dmar_dev_scope_status == 0)
 145                                 dmar_dev_scope_status = -ENOMEM;
 146                         return NULL;
 147                 }
 148         }
 149
 150         info->event = event;
 151         info->dev = dev;
 152         info->seg = pci_domain_nr(dev->bus);
 153         info->level = level;
 154         if (event == BUS_NOTIFY_ADD_DEVICE) {
 155                 for (tmp = dev, level--; tmp; tmp = tmp->bus->self) {
 156                         info->path[level].device = PCI_SLOT(tmp->devfn);
 157                         info->path[level].function = PCI_FUNC(tmp->devfn);
 158                         if (pci_is_root_bus(tmp->bus))
 159                                 info->bus = tmp->bus->number;
 160                 }
 161         }
 162
 163         return info;
 164 }
 165
 166 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
 167 {
 168         if ((void *)info != dmar_pci_notify_info_buf)
 169                 kfree(info);
 170 }
 171
 172 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
 173                                 struct acpi_dmar_pci_path *path, int count)
 174 {
 175         int i;
 176
 177         if (info->bus != bus)
 178                 return false;
 179         if (info->level != count)
 180                 return false;
 181
 182         for (i = 0; i < count; i++) {
 183                 if (path[i].device != info->path[i].device ||
 184                     path[i].function != info->path[i].function)
 185                         return false;
 186         }
 187
 188         return true;
 189 }
 190
 191 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
 192 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
 193                           void *start, void*end, u16 segment,
 194                           struct dmar_dev_scope *devices,
 195                           int devices_cnt)
 196 {
 197         int i, level;
 198         struct device *tmp, *dev = &info->dev->dev;
 199         struct acpi_dmar_device_scope *scope;
 200         struct acpi_dmar_pci_path *path;
 201
 202         if (segment != info->seg)
 203                 return 0;
 204
 205         for (; start < end; start += scope->length) {
 206                 scope = start;
 207                 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 208                     scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 209                         continue;
 210
 211                 path = (struct acpi_dmar_pci_path *)(scope + 1);
 212                 level = (scope->length - sizeof(*scope)) / sizeof(*path);
 213                 if (!dmar_match_pci_path(info, scope->bus, path, level))
 214                         continue;
 215
 216                 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
 217                     (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
 218                         pr_warn("Device scope type does not match for %s\n",
 219                                 pci_name(info->dev));
 220                         return -EINVAL;
 221                 }
 222
 223                 for_each_dev_scope(devices, devices_cnt, i, tmp)
 224                         if (tmp == NULL) {
 225                                 devices[i].bus = info->dev->bus->number;
 226                                 devices[i].devfn = info->dev->devfn;
 227                                 rcu_assign_pointer(devices[i].dev,
 228                                                    get_device(dev));
 229                                 return 1;
 230                         }
 231                 BUG_ON(i >= devices_cnt);
 232         }
 233
 234         return 0;
 235 }
 236
 237 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
 238                           struct dmar_dev_scope *devices, int count)
 239 {
 240         int index;
 241         struct device *tmp;
 242
 243         if (info->seg != segment)
 244                 return 0;
 245
 246         for_each_active_dev_scope(devices, count, index, tmp)
 247                 if (tmp == &info->dev->dev) {
 248                         rcu_assign_pointer(devices[index].dev, NULL);
 249                         synchronize_rcu();
 250                         put_device(tmp);
 251                         return 1;
 252                 }
 253
 254         return 0;
 255 }
 256
 257 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
 258 {
 259         int ret = 0;
 260         struct dmar_drhd_unit *dmaru;
 261         struct acpi_dmar_hardware_unit *drhd;
 262
 263         for_each_drhd_unit(dmaru) {
 264                 if (dmaru->include_all)
 265                         continue;
 266
 267                 drhd = container_of(dmaru->hdr,
 268                                     struct acpi_dmar_hardware_unit, header);
 269                 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
 270                                 ((void *)drhd) + drhd->header.length,
 271                                 dmaru->segment,
 272                                 dmaru->devices, dmaru->devices_cnt);
 273                 if (ret != 0)
 274                         break;
 275         }
 276         if (ret >= 0)
 277                 ret = dmar_iommu_notify_scope_dev(info);
 278         if (ret < 0 && dmar_dev_scope_status == 0)
 279                 dmar_dev_scope_status = ret;
 280
 281         return ret;
 282 }
 283
 284 static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
 285 {
 286         struct dmar_drhd_unit *dmaru;
 287
 288         for_each_drhd_unit(dmaru)
 289                 if (dmar_remove_dev_scope(info, dmaru->segment,
 290                         dmaru->devices, dmaru->devices_cnt))
 291                         break;
 292         dmar_iommu_notify_scope_dev(info);
 293 }
 294
 295 static int dmar_pci_bus_notifier(struct notifier_block *nb,
 296                                  unsigned long action, void *data)
 297 {
 298         struct pci_dev *pdev = to_pci_dev(data);
 299         struct dmar_pci_notify_info *info;
 300
 301         /* Only care about add/remove events for physical functions */
 302         if (pdev->is_virtfn)
 303                 return NOTIFY_DONE;
 304         if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_DEL_DEVICE)
 305                 return NOTIFY_DONE;
 306
 307         info = dmar_alloc_pci_notify_info(pdev, action);
 308         if (!info)
 309                 return NOTIFY_DONE;
 310
 311         down_write(&dmar_global_lock);
 312         if (action == BUS_NOTIFY_ADD_DEVICE)
 313                 dmar_pci_bus_add_dev(info);
 314         else if (action == BUS_NOTIFY_DEL_DEVICE)
 315                 dmar_pci_bus_del_dev(info);
 316         up_write(&dmar_global_lock);
 317
 318         dmar_free_pci_notify_info(info);
 319
 320         return NOTIFY_OK;
 321 }
 322
 323 static struct notifier_block dmar_pci_bus_nb = {
 324         .notifier_call = dmar_pci_bus_notifier,
 325         .priority = INT_MIN,
 326 };
 327
 328 /**
 329  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 330  * structure which uniquely represent one DMA remapping hardware unit
 331  * present in the platform
 332  */
 333 static int __init
 334 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 335 {
 336         struct acpi_dmar_hardware_unit *drhd;
 337         struct dmar_drhd_unit *dmaru;
 338         int ret = 0;
 339
 340         drhd = (struct acpi_dmar_hardware_unit *)header;
 341         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 342         if (!dmaru)
 343                 return -ENOMEM;
 344
 345         dmaru->hdr = header;
 346         dmaru->reg_base_addr = drhd->address;
 347         dmaru->segment = drhd->segment;
 348         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 349         dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
 350                                               ((void *)drhd) + drhd->header.length,
 351                                               &dmaru->devices_cnt);
 352         if (dmaru->devices_cnt && dmaru->devices == NULL) {
 353                 kfree(dmaru);
 354                 return -ENOMEM;
 355         }
 356
 357         ret = alloc_iommu(dmaru);
 358         if (ret) {
 359                 dmar_free_dev_scope(&dmaru->devices,
 360                                     &dmaru->devices_cnt);
 361                 kfree(dmaru);
 362                 return ret;
 363         }
 364         dmar_register_drhd_unit(dmaru);
 365         return 0;
 366 }
 367
 368 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
 369 {
 370         if (dmaru->devices && dmaru->devices_cnt)
 371                 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
 372         if (dmaru->iommu)
 373                 free_iommu(dmaru->iommu);
 374         kfree(dmaru);
 375 }
 376
 377 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header)
 378 {
 379         struct acpi_dmar_andd *andd = (void *)header;
 380
 381         /* Check for NUL termination within the designated length */
 382         if (strnlen(andd->object_name, header->length - 8) == header->length - 8) {
 383                 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
 384                            "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
 385                            "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 386                            dmi_get_system_info(DMI_BIOS_VENDOR),
 387                            dmi_get_system_info(DMI_BIOS_VERSION),
 388                            dmi_get_system_info(DMI_PRODUCT_VERSION));
 389                 return -EINVAL;
 390         }
 391         pr_info("ANDD device: %x name: %s\n", andd->device_number,
 392                 andd->object_name);
 393
 394         return 0;
 395 }
 396
 397 #ifdef CONFIG_ACPI_NUMA
 398 static int __init
 399 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
 400 {
 401         struct acpi_dmar_rhsa *rhsa;
 402         struct dmar_drhd_unit *drhd;
 403
 404         rhsa = (struct acpi_dmar_rhsa *)header;
 405         for_each_drhd_unit(drhd) {
 406                 if (drhd->reg_base_addr == rhsa->base_address) {
 407                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 408
 409                         if (!node_online(node))
 410                                 node = -1;
 411                         drhd->iommu->node = node;
 412                         return 0;
 413                 }
 414         }
 415         WARN_TAINT(
 416                 1, TAINT_FIRMWARE_WORKAROUND,
 417                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 418                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 419                 drhd->reg_base_addr,
 420                 dmi_get_system_info(DMI_BIOS_VENDOR),
 421                 dmi_get_system_info(DMI_BIOS_VERSION),
 422                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 423
 424         return 0;
 425 }
 426 #endif
 427
 428 static void __init
 429 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 430 {
 431         struct acpi_dmar_hardware_unit *drhd;
 432         struct acpi_dmar_reserved_memory *rmrr;
 433         struct acpi_dmar_atsr *atsr;
 434         struct acpi_dmar_rhsa *rhsa;
 435
 436         switch (header->type) {
 437         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 438                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 439                                     header);
 440                 pr_info("DRHD base: %#016Lx flags: %#x\n",
 441                         (unsigned long long)drhd->address, drhd->flags);
 442                 break;
 443         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 444                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 445                                     header);
 446                 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 447                         (unsigned long long)rmrr->base_address,
 448                         (unsigned long long)rmrr->end_address);
 449                 break;
 450         case ACPI_DMAR_TYPE_ATSR:
 451                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 452                 pr_info("ATSR flags: %#x\n", atsr->flags);
 453                 break;
 454         case ACPI_DMAR_HARDWARE_AFFINITY:
 455                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 456                 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 457                        (unsigned long long)rhsa->base_address,
 458                        rhsa->proximity_domain);
 459                 break;
 460         case ACPI_DMAR_TYPE_ANDD:
 461                 /* We don't print this here because we need to sanity-check
 462                    it first. So print it in dmar_parse_one_andd() instead. */
 463                 break;
 464         }
 465 }
 466
 467 /**
 468  * dmar_table_detect - checks to see if the platform supports DMAR devices
 469  */
 470 static int __init dmar_table_detect(void)
 471 {
 472         acpi_status status = AE_OK;
 473
 474         /* if we could find DMAR table, then there are DMAR devices */
 475         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 476                                 (struct acpi_table_header **)&dmar_tbl,
 477                                 &dmar_tbl_size);
 478
 479         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 480                 pr_warn("Unable to map DMAR\n");
 481                 status = AE_NOT_FOUND;
 482         }
 483
 484         return (ACPI_SUCCESS(status) ? 1 : 0);
 485 }
 486
 487 /**
 488  * parse_dmar_table - parses the DMA reporting table
 489  */
 490 static int __init
 491 parse_dmar_table(void)
 492 {
 493         struct acpi_table_dmar *dmar;
 494         struct acpi_dmar_header *entry_header;
 495         int ret = 0;
 496         int drhd_count = 0;
 497
 498         /*
 499          * Do it again, earlier dmar_tbl mapping could be mapped with
 500          * fixed map.
 501          */
 502         dmar_table_detect();
 503
 504         /*
 505          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 506          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 507          */
 508         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 509
 510         dmar = (struct acpi_table_dmar *)dmar_tbl;
 511         if (!dmar)
 512                 return -ENODEV;
 513
 514         if (dmar->width < PAGE_SHIFT - 1) {
 515                 pr_warn("Invalid DMAR haw\n");
 516                 return -EINVAL;
 517         }
 518
 519         pr_info("Host address width %d\n", dmar->width + 1);
 520
 521         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 522         while (((unsigned long)entry_header) <
 523                         (((unsigned long)dmar) + dmar_tbl->length)) {
 524                 /* Avoid looping forever on bad ACPI tables */
 525                 if (entry_header->length == 0) {
 526                         pr_warn("Invalid 0-length structure\n");
 527                         ret = -EINVAL;
 528                         break;
 529                 }
 530
 531                 dmar_table_print_dmar_entry(entry_header);
 532
 533                 switch (entry_header->type) {
 534                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 535                         drhd_count++;
 536                         ret = dmar_parse_one_drhd(entry_header);
 537                         break;
 538                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 539                         ret = dmar_parse_one_rmrr(entry_header);
 540                         break;
 541                 case ACPI_DMAR_TYPE_ATSR:
 542                         ret = dmar_parse_one_atsr(entry_header);
 543                         break;
 544                 case ACPI_DMAR_HARDWARE_AFFINITY:
 545 #ifdef CONFIG_ACPI_NUMA
 546                         ret = dmar_parse_one_rhsa(entry_header);
 547 #endif
 548                         break;
 549                 case ACPI_DMAR_TYPE_ANDD:
 550                         ret = dmar_parse_one_andd(entry_header);
 551                         break;
 552                 default:
 553                         pr_warn("Unknown DMAR structure type %d\n",
 554                                 entry_header->type);
 555                         ret = 0; /* for forward compatibility */
 556                         break;
 557                 }
 558                 if (ret)
 559                         break;
 560
 561                 entry_header = ((void *)entry_header + entry_header->length);
 562         }
 563         if (drhd_count == 0)
 564                 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 565         return ret;
 566 }
 567
 568 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
 569                                  int cnt, struct pci_dev *dev)
 570 {
 571         int index;
 572         struct device *tmp;
 573
 574         while (dev) {
 575                 for_each_active_dev_scope(devices, cnt, index, tmp)
 576                         if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
 577                                 return 1;
 578
 579                 /* Check our parent */
 580                 dev = dev->bus->self;
 581         }
 582
 583         return 0;
 584 }
 585
 586 struct dmar_drhd_unit *
 587 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 588 {
 589         struct dmar_drhd_unit *dmaru;
 590         struct acpi_dmar_hardware_unit *drhd;
 591
 592         dev = pci_physfn(dev);
 593
 594         rcu_read_lock();
 595         for_each_drhd_unit(dmaru) {
 596                 drhd = container_of(dmaru->hdr,
 597                                     struct acpi_dmar_hardware_unit,
 598                                     header);
 599
 600                 if (dmaru->include_all &&
 601                     drhd->segment == pci_domain_nr(dev->bus))
 602                         goto out;
 603
 604                 if (dmar_pci_device_match(dmaru->devices,
 605                                           dmaru->devices_cnt, dev))
 606                         goto out;
 607         }
 608         dmaru = NULL;
 609 out:
 610         rcu_read_unlock();
 611
 612         return dmaru;
 613 }
 614
 615 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
 616                                               struct acpi_device *adev)
 617 {
 618         struct dmar_drhd_unit *dmaru;
 619         struct acpi_dmar_hardware_unit *drhd;
 620         struct acpi_dmar_device_scope *scope;
 621         struct device *tmp;
 622         int i;
 623         struct acpi_dmar_pci_path *path;
 624
 625         for_each_drhd_unit(dmaru) {
 626                 drhd = container_of(dmaru->hdr,
 627                                     struct acpi_dmar_hardware_unit,
 628                                     header);
 629
 630                 for (scope = (void *)(drhd + 1);
 631                      (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
 632                      scope = ((void *)scope) + scope->length) {
 633                         if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ACPI)
 634                                 continue;
 635                         if (scope->enumeration_id != device_number)
 636                                 continue;
 637
 638                         path = (void *)(scope + 1);
 639                         pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
 640                                 dev_name(&adev->dev), dmaru->reg_base_addr,
 641                                 scope->bus, path->device, path->function);
 642                         for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
 643                                 if (tmp == NULL) {
 644                                         dmaru->devices[i].bus = scope->bus;
 645                                         dmaru->devices[i].devfn = PCI_DEVFN(path->device,
 646                                                                             path->function);
 647                                         rcu_assign_pointer(dmaru->devices[i].dev,
 648                                                            get_device(&adev->dev));
 649                                         return;
 650                                 }
 651                         BUG_ON(i >= dmaru->devices_cnt);
 652                 }
 653         }
 654         pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
 655                 device_number, dev_name(&adev->dev));
 656 }
 657
 658 static int __init dmar_acpi_dev_scope_init(void)
 659 {
 660         struct acpi_dmar_andd *andd;
 661
 662         if (dmar_tbl == NULL)
 663                 return -ENODEV;
 664
 665         for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
 666              ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
 667              andd = ((void *)andd) + andd->header.length) {
 668                 if (andd->header.type == ACPI_DMAR_TYPE_ANDD) {
 669                         acpi_handle h;
 670                         struct acpi_device *adev;
 671
 672                         if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
 673                                                           andd->object_name,
 674                                                           &h))) {
 675                                 pr_err("Failed to find handle for ACPI object %s\n",
 676                                        andd->object_name);
 677                                 continue;
 678                         }
 679                         acpi_bus_get_device(h, &adev);
 680                         if (!adev) {
 681                                 pr_err("Failed to get device for ACPI object %s\n",
 682                                        andd->object_name);
 683                                 continue;
 684                         }
 685                         dmar_acpi_insert_dev_scope(andd->device_number, adev);
 686                 }
 687         }
 688         return 0;
 689 }
 690
 691 int __init dmar_dev_scope_init(void)
 692 {
 693         struct pci_dev *dev = NULL;
 694         struct dmar_pci_notify_info *info;
 695
 696         if (dmar_dev_scope_status != 1)
 697                 return dmar_dev_scope_status;
 698
 699         if (list_empty(&dmar_drhd_units)) {
 700                 dmar_dev_scope_status = -ENODEV;
 701         } else {
 702                 dmar_dev_scope_status = 0;
 703
 704                 dmar_acpi_dev_scope_init();
 705
 706                 for_each_pci_dev(dev) {
 707                         if (dev->is_virtfn)
 708                                 continue;
 709
 710                         info = dmar_alloc_pci_notify_info(dev,
 711                                         BUS_NOTIFY_ADD_DEVICE);
 712                         if (!info) {
 713                                 return dmar_dev_scope_status;
 714                         } else {
 715                                 dmar_pci_bus_add_dev(info);
 716                                 dmar_free_pci_notify_info(info);
 717                         }
 718                 }
 719
 720                 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
 721         }
 722
 723         return dmar_dev_scope_status;
 724 }
 725
 726
 727 int __init dmar_table_init(void)
 728 {
 729         static int dmar_table_initialized;
 730         int ret;
 731
 732         if (dmar_table_initialized == 0) {
 733                 ret = parse_dmar_table();
 734                 if (ret < 0) {
 735                         if (ret != -ENODEV)
 736                                 pr_info("parse DMAR table failure.\n");
 737                 } else  if (list_empty(&dmar_drhd_units)) {
 738                         pr_info("No DMAR devices found\n");
 739                         ret = -ENODEV;
 740                 }
 741
 742                 if (ret < 0)
 743                         dmar_table_initialized = ret;
 744                 else
 745                         dmar_table_initialized = 1;
 746         }
 747
 748         return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
 749 }
 750
 751 static void warn_invalid_dmar(u64 addr, const char *message)
 752 {
 753         WARN_TAINT_ONCE(
 754                 1, TAINT_FIRMWARE_WORKAROUND,
 755                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 756                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 757                 addr, message,
 758                 dmi_get_system_info(DMI_BIOS_VENDOR),
 759                 dmi_get_system_info(DMI_BIOS_VERSION),
 760                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 761 }
 762
 763 static int __init check_zero_address(void)
 764 {
 765         struct acpi_table_dmar *dmar;
 766         struct acpi_dmar_header *entry_header;
 767         struct acpi_dmar_hardware_unit *drhd;
 768
 769         dmar = (struct acpi_table_dmar *)dmar_tbl;
 770         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 771
 772         while (((unsigned long)entry_header) <
 773                         (((unsigned long)dmar) + dmar_tbl->length)) {
 774                 /* Avoid looping forever on bad ACPI tables */
 775                 if (entry_header->length == 0) {
 776                         pr_warn("Invalid 0-length structure\n");
 777                         return 0;
 778                 }
 779
 780                 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
 781                         void __iomem *addr;
 782                         u64 cap, ecap;
 783
 784                         drhd = (void *)entry_header;
 785                         if (!drhd->address) {
 786                                 warn_invalid_dmar(0, "");
 787                                 goto failed;
 788                         }
 789
 790                         addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 791                         if (!addr ) {
 792                                 printk("IOMMU: can't validate: %llx\n", drhd->address);
 793                                 goto failed;
 794                         }
 795                         cap = dmar_readq(addr + DMAR_CAP_REG);
 796                         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 797                         early_iounmap(addr, VTD_PAGE_SIZE);
 798                         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 799                                 warn_invalid_dmar(drhd->address,
 800                                                   " returns all ones");
 801                                 goto failed;
 802                         }
 803                 }
 804
 805                 entry_header = ((void *)entry_header + entry_header->length);
 806         }
 807         return 1;
 808
 809 failed:
 810         return 0;
 811 }
 812
 813 int __init detect_intel_iommu(void)
 814 {
 815         int ret;
 816
 817         down_write(&dmar_global_lock);
 818         ret = dmar_table_detect();
 819         if (ret)
 820                 ret = check_zero_address();
 821         {
 822                 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 823                         iommu_detected = 1;
 824                         /* Make sure ACS will be enabled */
 825                         pci_request_acs();
 826                 }
 827
 828 #ifdef CONFIG_X86
 829                 if (ret)
 830                         x86_init.iommu.iommu_init = intel_iommu_init;
 831 #endif
 832         }
 833         early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
 834         dmar_tbl = NULL;
 835         up_write(&dmar_global_lock);
 836
 837         return ret ? 1 : -ENODEV;
 838 }
 839
 840
 841 static void unmap_iommu(struct intel_iommu *iommu)
 842 {
 843         iounmap(iommu->reg);
 844         release_mem_region(iommu->reg_phys, iommu->reg_size);
 845 }
 846
 847 /**
 848  * map_iommu: map the iommu's registers
 849  * @iommu: the iommu to map
 850  * @phys_addr: the physical address of the base resgister
 851  *
 852  * Memory map the iommu's registers.  Start w/ a single page, and
 853  * possibly expand if that turns out to be insufficent.
 854  */
 855 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 856 {
 857         int map_size, err=0;
 858
 859         iommu->reg_phys = phys_addr;
 860         iommu->reg_size = VTD_PAGE_SIZE;
 861
 862         if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 863                 pr_err("IOMMU: can't reserve memory\n");
 864                 err = -EBUSY;
 865                 goto out;
 866         }
 867
 868         iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 869         if (!iommu->reg) {
 870                 pr_err("IOMMU: can't map the region\n");
 871                 err = -ENOMEM;
 872                 goto release;
 873         }
 874
 875         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 876         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 877
 878         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 879                 err = -EINVAL;
 880                 warn_invalid_dmar(phys_addr, " returns all ones");
 881                 goto unmap;
 882         }
 883
 884         /* the registers might be more than one page */
 885         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 886                          cap_max_fault_reg_offset(iommu->cap));
 887         map_size = VTD_PAGE_ALIGN(map_size);
 888         if (map_size > iommu->reg_size) {
 889                 iounmap(iommu->reg);
 890                 release_mem_region(iommu->reg_phys, iommu->reg_size);
 891                 iommu->reg_size = map_size;
 892                 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 893                                         iommu->name)) {
 894                         pr_err("IOMMU: can't reserve memory\n");
 895                         err = -EBUSY;
 896                         goto out;
 897                 }
 898                 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 899                 if (!iommu->reg) {
 900                         pr_err("IOMMU: can't map the region\n");
 901                         err = -ENOMEM;
 902                         goto release;
 903                 }
 904         }
 905         err = 0;
 906         goto out;
 907
 908 unmap:
 909         iounmap(iommu->reg);
 910 release:
 911         release_mem_region(iommu->reg_phys, iommu->reg_size);
 912 out:
 913         return err;
 914 }
 915
 916 static int alloc_iommu(struct dmar_drhd_unit *drhd)
 917 {
 918         struct intel_iommu *iommu;
 919         u32 ver, sts;
 920         static int iommu_allocated = 0;
 921         int agaw = 0;
 922         int msagaw = 0;
 923         int err;
 924
 925         if (!drhd->reg_base_addr) {
 926                 warn_invalid_dmar(0, "");
 927                 return -EINVAL;
 928         }
 929
 930         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 931         if (!iommu)
 932                 return -ENOMEM;
 933
 934         iommu->seq_id = iommu_allocated++;
 935         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 936
 937         err = map_iommu(iommu, drhd->reg_base_addr);
 938         if (err) {
 939                 pr_err("IOMMU: failed to map %s\n", iommu->name);
 940                 goto error;
 941         }
 942
 943         err = -EINVAL;
 944         agaw = iommu_calculate_agaw(iommu);
 945         if (agaw < 0) {
 946                 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
 947                         iommu->seq_id);
 948                 goto err_unmap;
 949         }
 950         msagaw = iommu_calculate_max_sagaw(iommu);
 951         if (msagaw < 0) {
 952                 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 953                         iommu->seq_id);
 954                 goto err_unmap;
 955         }
 956         iommu->agaw = agaw;
 957         iommu->msagaw = msagaw;
 958         iommu->segment = drhd->segment;
 959
 960         iommu->node = -1;
 961
 962         ver = readl(iommu->reg + DMAR_VER_REG);
 963         pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
 964                 iommu->seq_id,
 965                 (unsigned long long)drhd->reg_base_addr,
 966                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 967                 (unsigned long long)iommu->cap,
 968                 (unsigned long long)iommu->ecap);
 969
 970         /* Reflect status in gcmd */
 971         sts = readl(iommu->reg + DMAR_GSTS_REG);
 972         if (sts & DMA_GSTS_IRES)
 973                 iommu->gcmd |= DMA_GCMD_IRE;
 974         if (sts & DMA_GSTS_TES)
 975                 iommu->gcmd |= DMA_GCMD_TE;
 976         if (sts & DMA_GSTS_QIES)
 977                 iommu->gcmd |= DMA_GCMD_QIE;
 978
 979         raw_spin_lock_init(&iommu->register_lock);
 980
 981         drhd->iommu = iommu;
 982         return 0;
 983
 984  err_unmap:
 985         unmap_iommu(iommu);
 986  error:
 987         kfree(iommu);
 988         return err;
 989 }
 990
 991 static void free_iommu(struct intel_iommu *iommu)
 992 {
 993         if (iommu->irq) {
 994                 free_irq(iommu->irq, iommu);
 995                 irq_set_handler_data(iommu->irq, NULL);
 996                 destroy_irq(iommu->irq);
 997         }
 998
 999         if (iommu->qi) {
1000                 free_page((unsigned long)iommu->qi->desc);
1001                 kfree(iommu->qi->desc_status);
1002                 kfree(iommu->qi);
1003         }
1004
1005         if (iommu->reg)
1006                 unmap_iommu(iommu);
1007
1008         kfree(iommu);
1009 }
1010
1011 /*
1012  * Reclaim all the submitted descriptors which have completed its work.
1013  */
1014 static inline void reclaim_free_desc(struct q_inval *qi)
1015 {
1016         while (qi->desc_status[qi->free_tail] == QI_DONE ||
1017                qi->desc_status[qi->free_tail] == QI_ABORT) {
1018                 qi->desc_status[qi->free_tail] = QI_FREE;
1019                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1020                 qi->free_cnt++;
1021         }
1022 }
1023
1024 static int qi_check_fault(struct intel_iommu *iommu, int index)
1025 {
1026         u32 fault;
1027         int head, tail;
1028         struct q_inval *qi = iommu->qi;
1029         int wait_index = (index + 1) % QI_LENGTH;
1030
1031         if (qi->desc_status[wait_index] == QI_ABORT)
1032                 return -EAGAIN;
1033
1034         fault = readl(iommu->reg + DMAR_FSTS_REG);
1035
1036         /*
1037          * If IQE happens, the head points to the descriptor associated
1038          * with the error. No new descriptors are fetched until the IQE
1039          * is cleared.
1040          */
1041         if (fault & DMA_FSTS_IQE) {
1042                 head = readl(iommu->reg + DMAR_IQH_REG);
1043                 if ((head >> DMAR_IQ_SHIFT) == index) {
1044                         pr_err("VT-d detected invalid descriptor: "
1045                                 "low=%llx, high=%llx\n",
1046                                 (unsigned long long)qi->desc[index].low,
1047                                 (unsigned long long)qi->desc[index].high);
1048                         memcpy(&qi->desc[index], &qi->desc[wait_index],
1049                                         sizeof(struct qi_desc));
1050                         __iommu_flush_cache(iommu, &qi->desc[index],
1051                                         sizeof(struct qi_desc));
1052                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1053                         return -EINVAL;
1054                 }
1055         }
1056
1057         /*
1058          * If ITE happens, all pending wait_desc commands are aborted.
1059          * No new descriptors are fetched until the ITE is cleared.
1060          */
1061         if (fault & DMA_FSTS_ITE) {
1062                 head = readl(iommu->reg + DMAR_IQH_REG);
1063                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1064                 head |= 1;
1065                 tail = readl(iommu->reg + DMAR_IQT_REG);
1066                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1067
1068                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1069
1070                 do {
1071                         if (qi->desc_status[head] == QI_IN_USE)
1072                                 qi->desc_status[head] = QI_ABORT;
1073                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1074                 } while (head != tail);
1075
1076                 if (qi->desc_status[wait_index] == QI_ABORT)
1077                         return -EAGAIN;
1078         }
1079
1080         if (fault & DMA_FSTS_ICE)
1081                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1082
1083         return 0;
1084 }
1085
1086 /*
1087  * Submit the queued invalidation descriptor to the remapping
1088  * hardware unit and wait for its completion.
1089  */
1090 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1091 {
1092         int rc;
1093         struct q_inval *qi = iommu->qi;
1094         struct qi_desc *hw, wait_desc;
1095         int wait_index, index;
1096         unsigned long flags;
1097
1098         if (!qi)
1099                 return 0;
1100
1101         hw = qi->desc;
1102
1103 restart:
1104         rc = 0;
1105
1106         raw_spin_lock_irqsave(&qi->q_lock, flags);
1107         while (qi->free_cnt < 3) {
1108                 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1109                 cpu_relax();
1110                 raw_spin_lock_irqsave(&qi->q_lock, flags);
1111         }
1112
1113         index = qi->free_head;
1114         wait_index = (index + 1) % QI_LENGTH;
1115
1116         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1117
1118         hw[index] = *desc;
1119
1120         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1121                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1122         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1123
1124         hw[wait_index] = wait_desc;
1125
1126         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1127         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1128
1129         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1130         qi->free_cnt -= 2;
1131
1132         /*
1133          * update the HW tail register indicating the presence of
1134          * new descriptors.
1135          */
1136         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1137
1138         while (qi->desc_status[wait_index] != QI_DONE) {
1139                 /*
1140                  * We will leave the interrupts disabled, to prevent interrupt
1141                  * context to queue another cmd while a cmd is already submitted
1142                  * and waiting for completion on this cpu. This is to avoid
1143                  * a deadlock where the interrupt context can wait indefinitely
1144                  * for free slots in the queue.
1145                  */
1146                 rc = qi_check_fault(iommu, index);
1147                 if (rc)
1148                         break;
1149
1150                 raw_spin_unlock(&qi->q_lock);
1151                 cpu_relax();
1152                 raw_spin_lock(&qi->q_lock);
1153         }
1154
1155         qi->desc_status[index] = QI_DONE;
1156
1157         reclaim_free_desc(qi);
1158         raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1159
1160         if (rc == -EAGAIN)
1161                 goto restart;
1162
1163         return rc;
1164 }
1165
1166 /*
1167  * Flush the global interrupt entry cache.
1168  */
1169 void qi_global_iec(struct intel_iommu *iommu)
1170 {
1171         struct qi_desc desc;
1172
1173         desc.low = QI_IEC_TYPE;
1174         desc.high = 0;
1175
1176         /* should never fail */
1177         qi_submit_sync(&desc, iommu);
1178 }
1179
1180 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1181                       u64 type)
1182 {
1183         struct qi_desc desc;
1184
1185         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1186                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1187         desc.high = 0;
1188
1189         qi_submit_sync(&desc, iommu);
1190 }
1191
1192 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1193                     unsigned int size_order, u64 type)
1194 {
1195         u8 dw = 0, dr = 0;
1196
1197         struct qi_desc desc;
1198         int ih = 0;
1199
1200         if (cap_write_drain(iommu->cap))
1201                 dw = 1;
1202
1203         if (cap_read_drain(iommu->cap))
1204                 dr = 1;
1205
1206         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1207                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1208         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1209                 | QI_IOTLB_AM(size_order);
1210
1211         qi_submit_sync(&desc, iommu);
1212 }
1213
1214 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1215                         u64 addr, unsigned mask)
1216 {
1217         struct qi_desc desc;
1218
1219         if (mask) {
1220                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1221                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1222                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1223         } else
1224                 desc.high = QI_DEV_IOTLB_ADDR(addr);
1225
1226         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1227                 qdep = 0;
1228
1229         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1230                    QI_DIOTLB_TYPE;
1231
1232         qi_submit_sync(&desc, iommu);
1233 }
1234
1235 /*
1236  * Disable Queued Invalidation interface.
1237  */
1238 void dmar_disable_qi(struct intel_iommu *iommu)
1239 {
1240         unsigned long flags;
1241         u32 sts;
1242         cycles_t start_time = get_cycles();
1243
1244         if (!ecap_qis(iommu->ecap))
1245                 return;
1246
1247         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1248
1249         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
1250         if (!(sts & DMA_GSTS_QIES))
1251                 goto end;
1252
1253         /*
1254          * Give a chance to HW to complete the pending invalidation requests.
1255          */
1256         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1257                 readl(iommu->reg + DMAR_IQH_REG)) &&
1258                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1259                 cpu_relax();
1260
1261         iommu->gcmd &= ~DMA_GCMD_QIE;
1262         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1263
1264         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1265                       !(sts & DMA_GSTS_QIES), sts);
1266 end:
1267         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1268 }
1269
1270 /*
1271  * Enable queued invalidation.
1272  */
1273 static void __dmar_enable_qi(struct intel_iommu *iommu)
1274 {
1275         u32 sts;
1276         unsigned long flags;
1277         struct q_inval *qi = iommu->qi;
1278
1279         qi->free_head = qi->free_tail = 0;
1280         qi->free_cnt = QI_LENGTH;
1281
1282         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1283
1284         /* write zero to the tail reg */
1285         writel(0, iommu->reg + DMAR_IQT_REG);
1286
1287         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1288
1289         iommu->gcmd |= DMA_GCMD_QIE;
1290         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1291
1292         /* Make sure hardware complete it */
1293         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1294
1295         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1296 }
1297
1298 /*
1299  * Enable Queued Invalidation interface. This is a must to support
1300  * interrupt-remapping. Also used by DMA-remapping, which replaces
1301  * register based IOTLB invalidation.
1302  */
1303 int dmar_enable_qi(struct intel_iommu *iommu)
1304 {
1305         struct q_inval *qi;
1306         struct page *desc_page;
1307
1308         if (!ecap_qis(iommu->ecap))
1309                 return -ENOENT;
1310
1311         /*
1312          * queued invalidation is already setup and enabled.
1313          */
1314         if (iommu->qi)
1315                 return 0;
1316
1317         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1318         if (!iommu->qi)
1319                 return -ENOMEM;
1320
1321         qi = iommu->qi;
1322
1323
1324         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1325         if (!desc_page) {
1326                 kfree(qi);
1327                 iommu->qi = NULL;
1328                 return -ENOMEM;
1329         }
1330
1331         qi->desc = page_address(desc_page);
1332
1333         qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1334         if (!qi->desc_status) {
1335                 free_page((unsigned long) qi->desc);
1336                 kfree(qi);
1337                 iommu->qi = NULL;
1338                 return -ENOMEM;
1339         }
1340
1341         qi->free_head = qi->free_tail = 0;
1342         qi->free_cnt = QI_LENGTH;
1343
1344         raw_spin_lock_init(&qi->q_lock);
1345
1346         __dmar_enable_qi(iommu);
1347
1348         return 0;
1349 }
1350
1351 /* iommu interrupt handling. Most stuff are MSI-like. */
1352
1353 enum faulttype {
1354         DMA_REMAP,
1355         INTR_REMAP,
1356         UNKNOWN,
1357 };
1358
1359 static const char *dma_remap_fault_reasons[] =
1360 {
1361         "Software",
1362         "Present bit in root entry is clear",
1363         "Present bit in context entry is clear",
1364         "Invalid context entry",
1365         "Access beyond MGAW",
1366         "PTE Write access is not set",
1367         "PTE Read access is not set",
1368         "Next page table ptr is invalid",
1369         "Root table address invalid",
1370         "Context table ptr is invalid",
1371         "non-zero reserved fields in RTP",
1372         "non-zero reserved fields in CTP",
1373         "non-zero reserved fields in PTE",
1374         "PCE for translation request specifies blocking",
1375 };
1376
1377 static const char *irq_remap_fault_reasons[] =
1378 {
1379         "Detected reserved fields in the decoded interrupt-remapped request",
1380         "Interrupt index exceeded the interrupt-remapping table size",
1381         "Present field in the IRTE entry is clear",
1382         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1383         "Detected reserved fields in the IRTE entry",
1384         "Blocked a compatibility format interrupt request",
1385         "Blocked an interrupt request due to source-id verification failure",
1386 };
1387
1388 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1389 {
1390         if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1391                                         ARRAY_SIZE(irq_remap_fault_reasons))) {
1392                 *fault_type = INTR_REMAP;
1393                 return irq_remap_fault_reasons[fault_reason - 0x20];
1394         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1395                 *fault_type = DMA_REMAP;
1396                 return dma_remap_fault_reasons[fault_reason];
1397         } else {
1398                 *fault_type = UNKNOWN;
1399                 return "Unknown";
1400         }
1401 }
1402
1403 void dmar_msi_unmask(struct irq_data *data)
1404 {
1405         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1406         unsigned long flag;
1407
1408         /* unmask it */
1409         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1410         writel(0, iommu->reg + DMAR_FECTL_REG);
1411         /* Read a reg to force flush the post write */
1412         readl(iommu->reg + DMAR_FECTL_REG);
1413         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1414 }
1415
1416 void dmar_msi_mask(struct irq_data *data)
1417 {
1418         unsigned long flag;
1419         struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1420
1421         /* mask it */
1422         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1423         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1424         /* Read a reg to force flush the post write */
1425         readl(iommu->reg + DMAR_FECTL_REG);
1426         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1427 }
1428
1429 void dmar_msi_write(int irq, struct msi_msg *msg)
1430 {
1431         struct intel_iommu *iommu = irq_get_handler_data(irq);
1432         unsigned long flag;
1433
1434         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1435         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1436         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1437         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1438         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1439 }
1440
1441 void dmar_msi_read(int irq, struct msi_msg *msg)
1442 {
1443         struct intel_iommu *iommu = irq_get_handler_data(irq);
1444         unsigned long flag;
1445
1446         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1447         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1448         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1449         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1450         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1451 }
1452
1453 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1454                 u8 fault_reason, u16 source_id, unsigned long long addr)
1455 {
1456         const char *reason;
1457         int fault_type;
1458
1459         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1460
1461         if (fault_type == INTR_REMAP)
1462                 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1463                        "fault index %llx\n"
1464                         "INTR-REMAP:[fault reason %02d] %s\n",
1465                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1466                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1467                         fault_reason, reason);
1468         else
1469                 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1470                        "fault addr %llx \n"
1471                        "DMAR:[fault reason %02d] %s\n",
1472                        (type ? "DMA Read" : "DMA Write"),
1473                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1474                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1475         return 0;
1476 }
1477
1478 #define PRIMARY_FAULT_REG_LEN (16)
1479 irqreturn_t dmar_fault(int irq, void *dev_id)
1480 {
1481         struct intel_iommu *iommu = dev_id;
1482         int reg, fault_index;
1483         u32 fault_status;
1484         unsigned long flag;
1485
1486         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1487         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1488         if (fault_status)
1489                 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1490
1491         /* TBD: ignore advanced fault log currently */
1492         if (!(fault_status & DMA_FSTS_PPF))
1493                 goto unlock_exit;
1494
1495         fault_index = dma_fsts_fault_record_index(fault_status);
1496         reg = cap_fault_reg_offset(iommu->cap);
1497         while (1) {
1498                 u8 fault_reason;
1499                 u16 source_id;
1500                 u64 guest_addr;
1501                 int type;
1502                 u32 data;
1503
1504                 /* highest 32 bits */
1505                 data = readl(iommu->reg + reg +
1506                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1507                 if (!(data & DMA_FRCD_F))
1508                         break;
1509
1510                 fault_reason = dma_frcd_fault_reason(data);
1511                 type = dma_frcd_type(data);
1512
1513                 data = readl(iommu->reg + reg +
1514                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1515                 source_id = dma_frcd_source_id(data);
1516
1517                 guest_addr = dmar_readq(iommu->reg + reg +
1518                                 fault_index * PRIMARY_FAULT_REG_LEN);
1519                 guest_addr = dma_frcd_page_addr(guest_addr);
1520                 /* clear the fault */
1521                 writel(DMA_FRCD_F, iommu->reg + reg +
1522                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1523
1524                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1525
1526                 dmar_fault_do_one(iommu, type, fault_reason,
1527                                 source_id, guest_addr);
1528
1529                 fault_index++;
1530                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1531                         fault_index = 0;
1532                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1533         }
1534
1535         writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1536
1537 unlock_exit:
1538         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1539         return IRQ_HANDLED;
1540 }
1541
1542 int dmar_set_interrupt(struct intel_iommu *iommu)
1543 {
1544         int irq, ret;
1545
1546         /*
1547          * Check if the fault interrupt is already initialized.
1548          */
1549         if (iommu->irq)
1550                 return 0;
1551
1552         irq = create_irq();
1553         if (!irq) {
1554                 pr_err("IOMMU: no free vectors\n");
1555                 return -EINVAL;
1556         }
1557
1558         irq_set_handler_data(irq, iommu);
1559         iommu->irq = irq;
1560
1561         ret = arch_setup_dmar_msi(irq);
1562         if (ret) {
1563                 irq_set_handler_data(irq, NULL);
1564                 iommu->irq = 0;
1565                 destroy_irq(irq);
1566                 return ret;
1567         }
1568
1569         ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1570         if (ret)
1571                 pr_err("IOMMU: can't request irq\n");
1572         return ret;
1573 }
1574
1575 int __init enable_drhd_fault_handling(void)
1576 {
1577         struct dmar_drhd_unit *drhd;
1578         struct intel_iommu *iommu;
1579
1580         /*
1581          * Enable fault control interrupt.
1582          */
1583         for_each_iommu(iommu, drhd) {
1584                 u32 fault_status;
1585                 int ret = dmar_set_interrupt(iommu);
1586
1587                 if (ret) {
1588                         pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1589                                (unsigned long long)drhd->reg_base_addr, ret);
1590                         return -1;
1591                 }
1592
1593                 /*
1594                  * Clear any previous faults.
1595                  */
1596                 dmar_fault(iommu->irq, iommu);
1597                 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1598                 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1599         }
1600
1601         return 0;
1602 }
1603
1604 /*
1605  * Re-enable Queued Invalidation interface.
1606  */
1607 int dmar_reenable_qi(struct intel_iommu *iommu)
1608 {
1609         if (!ecap_qis(iommu->ecap))
1610                 return -ENOENT;
1611
1612         if (!iommu->qi)
1613                 return -ENOENT;
1614
1615         /*
1616          * First disable queued invalidation.
1617          */
1618         dmar_disable_qi(iommu);
1619         /*
1620          * Then enable queued invalidation again. Since there is no pending
1621          * invalidation requests now, it's safe to re-enable queued
1622          * invalidation.
1623          */
1624         __dmar_enable_qi(iommu);
1625
1626         return 0;
1627 }
1628
1629 /*
1630  * Check interrupt remapping support in DMAR table description.
1631  */
1632 int __init dmar_ir_support(void)
1633 {
1634         struct acpi_table_dmar *dmar;
1635         dmar = (struct acpi_table_dmar *)dmar_tbl;
1636         if (!dmar)
1637                 return 0;
1638         return dmar->flags & 0x1;
1639 }
1640
1641 static int __init dmar_free_unused_resources(void)
1642 {
1643         struct dmar_drhd_unit *dmaru, *dmaru_n;
1644
1645         /* DMAR units are in use */
1646         if (irq_remapping_enabled || intel_iommu_enabled)
1647                 return 0;
1648
1649         if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1650                 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1651
1652         down_write(&dmar_global_lock);
1653         list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1654                 list_del(&dmaru->list);
1655                 dmar_free_drhd(dmaru);
1656         }
1657         up_write(&dmar_global_lock);
1658
1659         return 0;
1660 }
1661
1662 late_initcall(dmar_free_unused_resources);
1663 IOMMU_INIT_POST(detect_intel_iommu);