drivers/iommu/arm/arm-smmu/arm-smmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * IOMMU API for ARM architected SMMU implementations.
   4  *
   5  * Copyright (C) 2013 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  *
   9  * This driver currently supports:
  10  *      - SMMUv1 and v2 implementations
  11  *      - Stream-matching and stream-indexing
  12  *      - v7/v8 long-descriptor format
  13  *      - Non-secure access to the SMMU
  14  *      - Context fault reporting
  15  *      - Extended Stream ID (16 bit)
  16  */
  17
  18 #define pr_fmt(fmt) "arm-smmu: " fmt
  19
  20 #include <linux/acpi.h>
  21 #include <linux/acpi_iort.h>
  22 #include <linux/bitfield.h>
  23 #include <linux/delay.h>
  24 #include <linux/dma-iommu.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/err.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/io.h>
  29 #include <linux/iopoll.h>
  30 #include <linux/module.h>
  31 #include <linux/of.h>
  32 #include <linux/of_address.h>
  33 #include <linux/of_device.h>
  34 #include <linux/pci.h>
  35 #include <linux/platform_device.h>
  36 #include <linux/pm_runtime.h>
  37 #include <linux/ratelimit.h>
  38 #include <linux/slab.h>
  39
  40 #include <linux/amba/bus.h>
  41 #include <linux/fsl/mc.h>
  42
  43 #include "arm-smmu.h"
  44
  45 /*
  46  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
  47  * global register space are still, in fact, using a hypervisor to mediate it
  48  * by trapping and emulating register accesses. Sadly, some deployed versions
  49  * of said trapping code have bugs wherein they go horribly wrong for stores
  50  * using r31 (i.e. XZR/WZR) as the source register.
  51  */
  52 #define QCOM_DUMMY_VAL -1
  53
  54 #define MSI_IOVA_BASE                   0x8000000
  55 #define MSI_IOVA_LENGTH                 0x100000
  56
  57 static int force_stage;
  58 module_param(force_stage, int, S_IRUGO);
  59 MODULE_PARM_DESC(force_stage,
  60         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
  61 static bool disable_bypass =
  62         IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
  63 module_param(disable_bypass, bool, S_IRUGO);
  64 MODULE_PARM_DESC(disable_bypass,
  65         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
  66
  67 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
  68         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
  69 }
  70
  71 static bool using_legacy_binding, using_generic_binding;
  72
  73 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
  74 {
  75         if (pm_runtime_enabled(smmu->dev))
  76                 return pm_runtime_resume_and_get(smmu->dev);
  77
  78         return 0;
  79 }
  80
  81 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
  82 {
  83         if (pm_runtime_enabled(smmu->dev))
  84                 pm_runtime_put_autosuspend(smmu->dev);
  85 }
  86
  87 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
  88 {
  89         return container_of(dom, struct arm_smmu_domain, domain);
  90 }
  91
  92 static struct platform_driver arm_smmu_driver;
  93 static struct iommu_ops arm_smmu_ops;
  94
  95 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
  96 static int arm_smmu_bus_init(struct iommu_ops *ops);
  97
  98 static struct device_node *dev_get_dev_node(struct device *dev)
  99 {
 100         if (dev_is_pci(dev)) {
 101                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 102
 103                 while (!pci_is_root_bus(bus))
 104                         bus = bus->parent;
 105                 return of_node_get(bus->bridge->parent->of_node);
 106         }
 107
 108         return of_node_get(dev->of_node);
 109 }
 110
 111 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 112 {
 113         *((__be32 *)data) = cpu_to_be32(alias);
 114         return 0; /* Continue walking */
 115 }
 116
 117 static int __find_legacy_master_phandle(struct device *dev, void *data)
 118 {
 119         struct of_phandle_iterator *it = *(void **)data;
 120         struct device_node *np = it->node;
 121         int err;
 122
 123         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 124                             "#stream-id-cells", -1)
 125                 if (it->node == np) {
 126                         *(void **)data = dev;
 127                         return 1;
 128                 }
 129         it->node = np;
 130         return err == -ENOENT ? 0 : err;
 131 }
 132
 133 static int arm_smmu_register_legacy_master(struct device *dev,
 134                                            struct arm_smmu_device **smmu)
 135 {
 136         struct device *smmu_dev;
 137         struct device_node *np;
 138         struct of_phandle_iterator it;
 139         void *data = &it;
 140         u32 *sids;
 141         __be32 pci_sid;
 142         int err;
 143
 144         np = dev_get_dev_node(dev);
 145         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 146                 of_node_put(np);
 147                 return -ENODEV;
 148         }
 149
 150         it.node = np;
 151         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 152                                      __find_legacy_master_phandle);
 153         smmu_dev = data;
 154         of_node_put(np);
 155         if (err == 0)
 156                 return -ENODEV;
 157         if (err < 0)
 158                 return err;
 159
 160         if (dev_is_pci(dev)) {
 161                 /* "mmu-masters" assumes Stream ID == Requester ID */
 162                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 163                                        &pci_sid);
 164                 it.cur = &pci_sid;
 165                 it.cur_count = 1;
 166         }
 167
 168         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 169                                 &arm_smmu_ops);
 170         if (err)
 171                 return err;
 172
 173         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 174         if (!sids)
 175                 return -ENOMEM;
 176
 177         *smmu = dev_get_drvdata(smmu_dev);
 178         of_phandle_iterator_args(&it, sids, it.cur_count);
 179         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 180         kfree(sids);
 181         return err;
 182 }
 183
 184 /*
 185  * With the legacy DT binding in play, we have no guarantees about
 186  * probe order, but then we're also not doing default domains, so we can
 187  * delay setting bus ops until we're sure every possible SMMU is ready,
 188  * and that way ensure that no probe_device() calls get missed.
 189  */
 190 static int arm_smmu_legacy_bus_init(void)
 191 {
 192         if (using_legacy_binding)
 193                 return arm_smmu_bus_init(&arm_smmu_ops);
 194         return 0;
 195 }
 196 device_initcall_sync(arm_smmu_legacy_bus_init);
 197 #else
 198 static int arm_smmu_register_legacy_master(struct device *dev,
 199                                            struct arm_smmu_device **smmu)
 200 {
 201         return -ENODEV;
 202 }
 203 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
 204
 205 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 206 {
 207         clear_bit(idx, map);
 208 }
 209
 210 /* Wait for any pending TLB invalidations to complete */
 211 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
 212                                 int sync, int status)
 213 {
 214         unsigned int spin_cnt, delay;
 215         u32 reg;
 216
 217         if (smmu->impl && unlikely(smmu->impl->tlb_sync))
 218                 return smmu->impl->tlb_sync(smmu, page, sync, status);
 219
 220         arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
 221         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 222                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 223                         reg = arm_smmu_readl(smmu, page, status);
 224                         if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
 225                                 return;
 226                         cpu_relax();
 227                 }
 228                 udelay(delay);
 229         }
 230         dev_err_ratelimited(smmu->dev,
 231                             "TLB sync timed out -- SMMU may be deadlocked\n");
 232 }
 233
 234 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 235 {
 236         unsigned long flags;
 237
 238         spin_lock_irqsave(&smmu->global_sync_lock, flags);
 239         __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
 240                             ARM_SMMU_GR0_sTLBGSTATUS);
 241         spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 242 }
 243
 244 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
 245 {
 246         struct arm_smmu_device *smmu = smmu_domain->smmu;
 247         unsigned long flags;
 248
 249         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 250         __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
 251                             ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
 252         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 253 }
 254
 255 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 256 {
 257         struct arm_smmu_domain *smmu_domain = cookie;
 258         /*
 259          * The TLBI write may be relaxed, so ensure that PTEs cleared by the
 260          * current CPU are visible beforehand.
 261          */
 262         wmb();
 263         arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
 264                           ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
 265         arm_smmu_tlb_sync_context(smmu_domain);
 266 }
 267
 268 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 269 {
 270         struct arm_smmu_domain *smmu_domain = cookie;
 271         struct arm_smmu_device *smmu = smmu_domain->smmu;
 272
 273         /* See above */
 274         wmb();
 275         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 276         arm_smmu_tlb_sync_global(smmu);
 277 }
 278
 279 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
 280                                       size_t granule, void *cookie, int reg)
 281 {
 282         struct arm_smmu_domain *smmu_domain = cookie;
 283         struct arm_smmu_device *smmu = smmu_domain->smmu;
 284         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 285         int idx = cfg->cbndx;
 286
 287         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 288                 wmb();
 289
 290         if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 291                 iova = (iova >> 12) << 12;
 292                 iova |= cfg->asid;
 293                 do {
 294                         arm_smmu_cb_write(smmu, idx, reg, iova);
 295                         iova += granule;
 296                 } while (size -= granule);
 297         } else {
 298                 iova >>= 12;
 299                 iova |= (u64)cfg->asid << 48;
 300                 do {
 301                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 302                         iova += granule >> 12;
 303                 } while (size -= granule);
 304         }
 305 }
 306
 307 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
 308                                       size_t granule, void *cookie, int reg)
 309 {
 310         struct arm_smmu_domain *smmu_domain = cookie;
 311         struct arm_smmu_device *smmu = smmu_domain->smmu;
 312         int idx = smmu_domain->cfg.cbndx;
 313
 314         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 315                 wmb();
 316
 317         iova >>= 12;
 318         do {
 319                 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
 320                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 321                 else
 322                         arm_smmu_cb_write(smmu, idx, reg, iova);
 323                 iova += granule >> 12;
 324         } while (size -= granule);
 325 }
 326
 327 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
 328                                      size_t granule, void *cookie)
 329 {
 330         arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 331                                   ARM_SMMU_CB_S1_TLBIVA);
 332         arm_smmu_tlb_sync_context(cookie);
 333 }
 334
 335 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
 336                                      unsigned long iova, size_t granule,
 337                                      void *cookie)
 338 {
 339         arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
 340                                   ARM_SMMU_CB_S1_TLBIVAL);
 341 }
 342
 343 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
 344                                      size_t granule, void *cookie)
 345 {
 346         arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 347                                   ARM_SMMU_CB_S2_TLBIIPAS2);
 348         arm_smmu_tlb_sync_context(cookie);
 349 }
 350
 351 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
 352                                      unsigned long iova, size_t granule,
 353                                      void *cookie)
 354 {
 355         arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
 356                                   ARM_SMMU_CB_S2_TLBIIPAS2L);
 357 }
 358
 359 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
 360                                         size_t granule, void *cookie)
 361 {
 362         arm_smmu_tlb_inv_context_s2(cookie);
 363 }
 364 /*
 365  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 366  * almost negligible, but the benefit of getting the first one in as far ahead
 367  * of the sync as possible is significant, hence we don't just make this a
 368  * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
 369  * think.
 370  */
 371 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
 372                                         unsigned long iova, size_t granule,
 373                                         void *cookie)
 374 {
 375         struct arm_smmu_domain *smmu_domain = cookie;
 376         struct arm_smmu_device *smmu = smmu_domain->smmu;
 377
 378         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 379                 wmb();
 380
 381         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 382 }
 383
 384 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
 385         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 386         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
 387         .tlb_add_page   = arm_smmu_tlb_add_page_s1,
 388 };
 389
 390 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
 391         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 392         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
 393         .tlb_add_page   = arm_smmu_tlb_add_page_s2,
 394 };
 395
 396 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
 397         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 398         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
 399         .tlb_add_page   = arm_smmu_tlb_add_page_s2_v1,
 400 };
 401
 402 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 403 {
 404         u32 fsr, fsynr, cbfrsynra;
 405         unsigned long iova;
 406         struct iommu_domain *domain = dev;
 407         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 408         struct arm_smmu_device *smmu = smmu_domain->smmu;
 409         int idx = smmu_domain->cfg.cbndx;
 410         int ret;
 411
 412         fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
 413         if (!(fsr & ARM_SMMU_FSR_FAULT))
 414                 return IRQ_NONE;
 415
 416         fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
 417         iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
 418         cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
 419
 420         ret = report_iommu_fault(domain, NULL, iova,
 421                 fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
 422
 423         if (ret == -ENOSYS)
 424                 dev_err_ratelimited(smmu->dev,
 425                 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 426                             fsr, iova, fsynr, cbfrsynra, idx);
 427
 428         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
 429         return IRQ_HANDLED;
 430 }
 431
 432 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 433 {
 434         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 435         struct arm_smmu_device *smmu = dev;
 436         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 437                                       DEFAULT_RATELIMIT_BURST);
 438
 439         gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
 440         gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
 441         gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
 442         gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
 443
 444         if (!gfsr)
 445                 return IRQ_NONE;
 446
 447         if (__ratelimit(&rs)) {
 448                 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
 449                     (gfsr & ARM_SMMU_sGFSR_USF))
 450                         dev_err(smmu->dev,
 451                                 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
 452                                 (u16)gfsynr1);
 453                 else
 454                         dev_err(smmu->dev,
 455                                 "Unexpected global fault, this could be serious\n");
 456                 dev_err(smmu->dev,
 457                         "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 458                         gfsr, gfsynr0, gfsynr1, gfsynr2);
 459         }
 460
 461         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
 462         return IRQ_HANDLED;
 463 }
 464
 465 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 466                                        struct io_pgtable_cfg *pgtbl_cfg)
 467 {
 468         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 469         struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 470         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 471
 472         cb->cfg = cfg;
 473
 474         /* TCR */
 475         if (stage1) {
 476                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 477                         cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 478                 } else {
 479                         cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
 480                         cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
 481                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 482                                 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
 483                         else
 484                                 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
 485                 }
 486         } else {
 487                 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
 488         }
 489
 490         /* TTBRs */
 491         if (stage1) {
 492                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 493                         cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
 494                         cb->ttbr[1] = 0;
 495                 } else {
 496                         cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
 497                                                  cfg->asid);
 498                         cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
 499                                                  cfg->asid);
 500
 501                         if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
 502                                 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 503                         else
 504                                 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 505                 }
 506         } else {
 507                 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 508         }
 509
 510         /* MAIRs (stage-1 only) */
 511         if (stage1) {
 512                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 513                         cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 514                         cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 515                 } else {
 516                         cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
 517                         cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
 518                 }
 519         }
 520 }
 521
 522 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 523 {
 524         u32 reg;
 525         bool stage1;
 526         struct arm_smmu_cb *cb = &smmu->cbs[idx];
 527         struct arm_smmu_cfg *cfg = cb->cfg;
 528
 529         /* Unassigned context banks only need disabling */
 530         if (!cfg) {
 531                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
 532                 return;
 533         }
 534
 535         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 536
 537         /* CBA2R */
 538         if (smmu->version > ARM_SMMU_V1) {
 539                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 540                         reg = ARM_SMMU_CBA2R_VA64;
 541                 else
 542                         reg = 0;
 543                 /* 16-bit VMIDs live in CBA2R */
 544                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 545                         reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
 546
 547                 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
 548         }
 549
 550         /* CBAR */
 551         reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
 552         if (smmu->version < ARM_SMMU_V2)
 553                 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
 554
 555         /*
 556          * Use the weakest shareability/memory types, so they are
 557          * overridden by the ttbcr/pte.
 558          */
 559         if (stage1) {
 560                 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
 561                                   ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
 562                        FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
 563                                   ARM_SMMU_CBAR_S1_MEMATTR_WB);
 564         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 565                 /* 8-bit VMIDs live in CBAR */
 566                 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
 567         }
 568         arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
 569
 570         /*
 571          * TCR
 572          * We must write this before the TTBRs, since it determines the
 573          * access behaviour of some fields (in particular, ASID[15:8]).
 574          */
 575         if (stage1 && smmu->version > ARM_SMMU_V1)
 576                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
 577         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
 578
 579         /* TTBRs */
 580         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 581                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
 582                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 583                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
 584         } else {
 585                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 586                 if (stage1)
 587                         arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
 588                                            cb->ttbr[1]);
 589         }
 590
 591         /* MAIRs (stage-1 only) */
 592         if (stage1) {
 593                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
 594                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
 595         }
 596
 597         /* SCTLR */
 598         reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
 599               ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
 600         if (stage1)
 601                 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
 602         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 603                 reg |= ARM_SMMU_SCTLR_E;
 604
 605         if (smmu->impl && smmu->impl->write_sctlr)
 606                 smmu->impl->write_sctlr(smmu, idx, reg);
 607         else
 608                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
 609 }
 610
 611 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
 612                                        struct arm_smmu_device *smmu,
 613                                        struct device *dev, unsigned int start)
 614 {
 615         if (smmu->impl && smmu->impl->alloc_context_bank)
 616                 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
 617
 618         return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
 619 }
 620
 621 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 622                                         struct arm_smmu_device *smmu,
 623                                         struct device *dev)
 624 {
 625         int irq, start, ret = 0;
 626         unsigned long ias, oas;
 627         struct io_pgtable_ops *pgtbl_ops;
 628         struct io_pgtable_cfg pgtbl_cfg;
 629         enum io_pgtable_fmt fmt;
 630         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 631         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 632         irqreturn_t (*context_fault)(int irq, void *dev);
 633
 634         mutex_lock(&smmu_domain->init_mutex);
 635         if (smmu_domain->smmu)
 636                 goto out_unlock;
 637
 638         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 639                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 640                 smmu_domain->smmu = smmu;
 641                 goto out_unlock;
 642         }
 643
 644         /*
 645          * Mapping the requested stage onto what we support is surprisingly
 646          * complicated, mainly because the spec allows S1+S2 SMMUs without
 647          * support for nested translation. That means we end up with the
 648          * following table:
 649          *
 650          * Requested        Supported        Actual
 651          *     S1               N              S1
 652          *     S1             S1+S2            S1
 653          *     S1               S2             S2
 654          *     S1               S1             S1
 655          *     N                N              N
 656          *     N              S1+S2            S2
 657          *     N                S2             S2
 658          *     N                S1             S1
 659          *
 660          * Note that you can't actually request stage-2 mappings.
 661          */
 662         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 663                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 664         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 665                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 666
 667         /*
 668          * Choosing a suitable context format is even more fiddly. Until we
 669          * grow some way for the caller to express a preference, and/or move
 670          * the decision into the io-pgtable code where it arguably belongs,
 671          * just aim for the closest thing to the rest of the system, and hope
 672          * that the hardware isn't esoteric enough that we can't assume AArch64
 673          * support to be a superset of AArch32 support...
 674          */
 675         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 676                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 677         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 678             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 679             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 680             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 681                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 682         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 683             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 684                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 685                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 686                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 687
 688         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 689                 ret = -EINVAL;
 690                 goto out_unlock;
 691         }
 692
 693         switch (smmu_domain->stage) {
 694         case ARM_SMMU_DOMAIN_S1:
 695                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 696                 start = smmu->num_s2_context_banks;
 697                 ias = smmu->va_size;
 698                 oas = smmu->ipa_size;
 699                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 700                         fmt = ARM_64_LPAE_S1;
 701                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 702                         fmt = ARM_32_LPAE_S1;
 703                         ias = min(ias, 32UL);
 704                         oas = min(oas, 40UL);
 705                 } else {
 706                         fmt = ARM_V7S;
 707                         ias = min(ias, 32UL);
 708                         oas = min(oas, 32UL);
 709                 }
 710                 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
 711                 break;
 712         case ARM_SMMU_DOMAIN_NESTED:
 713                 /*
 714                  * We will likely want to change this if/when KVM gets
 715                  * involved.
 716                  */
 717         case ARM_SMMU_DOMAIN_S2:
 718                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 719                 start = 0;
 720                 ias = smmu->ipa_size;
 721                 oas = smmu->pa_size;
 722                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 723                         fmt = ARM_64_LPAE_S2;
 724                 } else {
 725                         fmt = ARM_32_LPAE_S2;
 726                         ias = min(ias, 40UL);
 727                         oas = min(oas, 40UL);
 728                 }
 729                 if (smmu->version == ARM_SMMU_V2)
 730                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
 731                 else
 732                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
 733                 break;
 734         default:
 735                 ret = -EINVAL;
 736                 goto out_unlock;
 737         }
 738
 739         ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
 740         if (ret < 0) {
 741                 goto out_unlock;
 742         }
 743
 744         smmu_domain->smmu = smmu;
 745
 746         cfg->cbndx = ret;
 747         if (smmu->version < ARM_SMMU_V2) {
 748                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 749                 cfg->irptndx %= smmu->num_context_irqs;
 750         } else {
 751                 cfg->irptndx = cfg->cbndx;
 752         }
 753
 754         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 755                 cfg->vmid = cfg->cbndx + 1;
 756         else
 757                 cfg->asid = cfg->cbndx;
 758
 759         pgtbl_cfg = (struct io_pgtable_cfg) {
 760                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 761                 .ias            = ias,
 762                 .oas            = oas,
 763                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
 764                 .tlb            = smmu_domain->flush_ops,
 765                 .iommu_dev      = smmu->dev,
 766         };
 767
 768         if (smmu->impl && smmu->impl->init_context) {
 769                 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
 770                 if (ret)
 771                         goto out_clear_smmu;
 772         }
 773
 774         if (smmu_domain->pgtbl_quirks)
 775                 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
 776
 777         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 778         if (!pgtbl_ops) {
 779                 ret = -ENOMEM;
 780                 goto out_clear_smmu;
 781         }
 782
 783         /* Update the domain's page sizes to reflect the page table format */
 784         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 785
 786         if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
 787                 domain->geometry.aperture_start = ~0UL << ias;
 788                 domain->geometry.aperture_end = ~0UL;
 789         } else {
 790                 domain->geometry.aperture_end = (1UL << ias) - 1;
 791         }
 792
 793         domain->geometry.force_aperture = true;
 794
 795         /* Initialise the context bank with our page table cfg */
 796         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 797         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 798
 799         /*
 800          * Request context fault interrupt. Do this last to avoid the
 801          * handler seeing a half-initialised domain state.
 802          */
 803         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 804
 805         if (smmu->impl && smmu->impl->context_fault)
 806                 context_fault = smmu->impl->context_fault;
 807         else
 808                 context_fault = arm_smmu_context_fault;
 809
 810         ret = devm_request_irq(smmu->dev, irq, context_fault,
 811                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 812         if (ret < 0) {
 813                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 814                         cfg->irptndx, irq);
 815                 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
 816         }
 817
 818         mutex_unlock(&smmu_domain->init_mutex);
 819
 820         /* Publish page table ops for map/unmap */
 821         smmu_domain->pgtbl_ops = pgtbl_ops;
 822         return 0;
 823
 824 out_clear_smmu:
 825         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 826         smmu_domain->smmu = NULL;
 827 out_unlock:
 828         mutex_unlock(&smmu_domain->init_mutex);
 829         return ret;
 830 }
 831
 832 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 833 {
 834         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 835         struct arm_smmu_device *smmu = smmu_domain->smmu;
 836         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 837         int ret, irq;
 838
 839         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
 840                 return;
 841
 842         ret = arm_smmu_rpm_get(smmu);
 843         if (ret < 0)
 844                 return;
 845
 846         /*
 847          * Disable the context bank and free the page tables before freeing
 848          * it.
 849          */
 850         smmu->cbs[cfg->cbndx].cfg = NULL;
 851         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 852
 853         if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
 854                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 855                 devm_free_irq(smmu->dev, irq, domain);
 856         }
 857
 858         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 859         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 860
 861         arm_smmu_rpm_put(smmu);
 862 }
 863
 864 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 865 {
 866         struct arm_smmu_domain *smmu_domain;
 867
 868         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_IDENTITY) {
 869                 if (using_legacy_binding || type != IOMMU_DOMAIN_DMA)
 870                         return NULL;
 871         }
 872         /*
 873          * Allocate the domain and initialise some of its data structures.
 874          * We can't really do anything meaningful until we've added a
 875          * master.
 876          */
 877         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
 878         if (!smmu_domain)
 879                 return NULL;
 880
 881         mutex_init(&smmu_domain->init_mutex);
 882         spin_lock_init(&smmu_domain->cb_lock);
 883
 884         return &smmu_domain->domain;
 885 }
 886
 887 static void arm_smmu_domain_free(struct iommu_domain *domain)
 888 {
 889         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 890
 891         /*
 892          * Free the domain resources. We assume that all devices have
 893          * already been detached.
 894          */
 895         arm_smmu_destroy_domain_context(domain);
 896         kfree(smmu_domain);
 897 }
 898
 899 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
 900 {
 901         struct arm_smmu_smr *smr = smmu->smrs + idx;
 902         u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
 903                   FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
 904
 905         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
 906                 reg |= ARM_SMMU_SMR_VALID;
 907         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
 908 }
 909
 910 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
 911 {
 912         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
 913         u32 reg;
 914
 915         if (smmu->impl && smmu->impl->write_s2cr) {
 916                 smmu->impl->write_s2cr(smmu, idx);
 917                 return;
 918         }
 919
 920         reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
 921               FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
 922               FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
 923
 924         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
 925             smmu->smrs[idx].valid)
 926                 reg |= ARM_SMMU_S2CR_EXIDVALID;
 927         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
 928 }
 929
 930 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
 931 {
 932         arm_smmu_write_s2cr(smmu, idx);
 933         if (smmu->smrs)
 934                 arm_smmu_write_smr(smmu, idx);
 935 }
 936
 937 /*
 938  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
 939  * should be called after sCR0 is written.
 940  */
 941 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
 942 {
 943         u32 smr;
 944         int i;
 945
 946         if (!smmu->smrs)
 947                 return;
 948         /*
 949          * If we've had to accommodate firmware memory regions, we may
 950          * have live SMRs by now; tread carefully...
 951          *
 952          * Somewhat perversely, not having a free SMR for this test implies we
 953          * can get away without it anyway, as we'll only be able to 'allocate'
 954          * these SMRs for the ID/mask values we're already trusting to be OK.
 955          */
 956         for (i = 0; i < smmu->num_mapping_groups; i++)
 957                 if (!smmu->smrs[i].valid)
 958                         goto smr_ok;
 959         return;
 960 smr_ok:
 961         /*
 962          * SMR.ID bits may not be preserved if the corresponding MASK
 963          * bits are set, so check each one separately. We can reject
 964          * masters later if they try to claim IDs outside these masks.
 965          */
 966         smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
 967         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
 968         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
 969         smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
 970
 971         smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
 972         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
 973         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
 974         smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
 975 }
 976
 977 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
 978 {
 979         struct arm_smmu_smr *smrs = smmu->smrs;
 980         int i, free_idx = -ENOSPC;
 981
 982         /* Stream indexing is blissfully easy */
 983         if (!smrs)
 984                 return id;
 985
 986         /* Validating SMRs is... less so */
 987         for (i = 0; i < smmu->num_mapping_groups; ++i) {
 988                 if (!smrs[i].valid) {
 989                         /*
 990                          * Note the first free entry we come across, which
 991                          * we'll claim in the end if nothing else matches.
 992                          */
 993                         if (free_idx < 0)
 994                                 free_idx = i;
 995                         continue;
 996                 }
 997                 /*
 998                  * If the new entry is _entirely_ matched by an existing entry,
 999                  * then reuse that, with the guarantee that there also cannot
1000                  * be any subsequent conflicting entries. In normal use we'd
1001                  * expect simply identical entries for this case, but there's
1002                  * no harm in accommodating the generalisation.
1003                  */
1004                 if ((mask & smrs[i].mask) == mask &&
1005                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1006                         return i;
1007                 /*
1008                  * If the new entry has any other overlap with an existing one,
1009                  * though, then there always exists at least one stream ID
1010                  * which would cause a conflict, and we can't allow that risk.
1011                  */
1012                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1013                         return -EINVAL;
1014         }
1015
1016         return free_idx;
1017 }
1018
1019 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1020 {
1021         if (--smmu->s2crs[idx].count)
1022                 return false;
1023
1024         smmu->s2crs[idx] = s2cr_init_val;
1025         if (smmu->smrs)
1026                 smmu->smrs[idx].valid = false;
1027
1028         return true;
1029 }
1030
1031 static int arm_smmu_master_alloc_smes(struct device *dev)
1032 {
1033         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1034         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1035         struct arm_smmu_device *smmu = cfg->smmu;
1036         struct arm_smmu_smr *smrs = smmu->smrs;
1037         int i, idx, ret;
1038
1039         mutex_lock(&smmu->stream_map_mutex);
1040         /* Figure out a viable stream map entry allocation */
1041         for_each_cfg_sme(cfg, fwspec, i, idx) {
1042                 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1043                 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1044
1045                 if (idx != INVALID_SMENDX) {
1046                         ret = -EEXIST;
1047                         goto out_err;
1048                 }
1049
1050                 ret = arm_smmu_find_sme(smmu, sid, mask);
1051                 if (ret < 0)
1052                         goto out_err;
1053
1054                 idx = ret;
1055                 if (smrs && smmu->s2crs[idx].count == 0) {
1056                         smrs[idx].id = sid;
1057                         smrs[idx].mask = mask;
1058                         smrs[idx].valid = true;
1059                 }
1060                 smmu->s2crs[idx].count++;
1061                 cfg->smendx[i] = (s16)idx;
1062         }
1063
1064         /* It worked! Now, poke the actual hardware */
1065         for_each_cfg_sme(cfg, fwspec, i, idx)
1066                 arm_smmu_write_sme(smmu, idx);
1067
1068         mutex_unlock(&smmu->stream_map_mutex);
1069         return 0;
1070
1071 out_err:
1072         while (i--) {
1073                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1074                 cfg->smendx[i] = INVALID_SMENDX;
1075         }
1076         mutex_unlock(&smmu->stream_map_mutex);
1077         return ret;
1078 }
1079
1080 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1081                                       struct iommu_fwspec *fwspec)
1082 {
1083         struct arm_smmu_device *smmu = cfg->smmu;
1084         int i, idx;
1085
1086         mutex_lock(&smmu->stream_map_mutex);
1087         for_each_cfg_sme(cfg, fwspec, i, idx) {
1088                 if (arm_smmu_free_sme(smmu, idx))
1089                         arm_smmu_write_sme(smmu, idx);
1090                 cfg->smendx[i] = INVALID_SMENDX;
1091         }
1092         mutex_unlock(&smmu->stream_map_mutex);
1093 }
1094
1095 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1096                                       struct arm_smmu_master_cfg *cfg,
1097                                       struct iommu_fwspec *fwspec)
1098 {
1099         struct arm_smmu_device *smmu = smmu_domain->smmu;
1100         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1101         u8 cbndx = smmu_domain->cfg.cbndx;
1102         enum arm_smmu_s2cr_type type;
1103         int i, idx;
1104
1105         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1106                 type = S2CR_TYPE_BYPASS;
1107         else
1108                 type = S2CR_TYPE_TRANS;
1109
1110         for_each_cfg_sme(cfg, fwspec, i, idx) {
1111                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1112                         continue;
1113
1114                 s2cr[idx].type = type;
1115                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1116                 s2cr[idx].cbndx = cbndx;
1117                 arm_smmu_write_s2cr(smmu, idx);
1118         }
1119         return 0;
1120 }
1121
1122 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1123 {
1124         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1125         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1126         struct arm_smmu_master_cfg *cfg;
1127         struct arm_smmu_device *smmu;
1128         int ret;
1129
1130         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1131                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1132                 return -ENXIO;
1133         }
1134
1135         /*
1136          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1137          * domains between of_xlate() and probe_device() - we have no way to cope
1138          * with that, so until ARM gets converted to rely on groups and default
1139          * domains, just say no (but more politely than by dereferencing NULL).
1140          * This should be at least a WARN_ON once that's sorted.
1141          */
1142         cfg = dev_iommu_priv_get(dev);
1143         if (!cfg)
1144                 return -ENODEV;
1145
1146         smmu = cfg->smmu;
1147
1148         ret = arm_smmu_rpm_get(smmu);
1149         if (ret < 0)
1150                 return ret;
1151
1152         /* Ensure that the domain is finalised */
1153         ret = arm_smmu_init_domain_context(domain, smmu, dev);
1154         if (ret < 0)
1155                 goto rpm_put;
1156
1157         /*
1158          * Sanity check the domain. We don't support domains across
1159          * different SMMUs.
1160          */
1161         if (smmu_domain->smmu != smmu) {
1162                 dev_err(dev,
1163                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1164                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1165                 ret = -EINVAL;
1166                 goto rpm_put;
1167         }
1168
1169         /* Looks ok, so add the device to the domain */
1170         ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1171
1172         /*
1173          * Setup an autosuspend delay to avoid bouncing runpm state.
1174          * Otherwise, if a driver for a suspended consumer device
1175          * unmaps buffers, it will runpm resume/suspend for each one.
1176          *
1177          * For example, when used by a GPU device, when an application
1178          * or game exits, it can trigger unmapping 100s or 1000s of
1179          * buffers.  With a runpm cycle for each buffer, that adds up
1180          * to 5-10sec worth of reprogramming the context bank, while
1181          * the system appears to be locked up to the user.
1182          */
1183         pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1184         pm_runtime_use_autosuspend(smmu->dev);
1185
1186 rpm_put:
1187         arm_smmu_rpm_put(smmu);
1188         return ret;
1189 }
1190
1191 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1192                               phys_addr_t paddr, size_t pgsize, size_t pgcount,
1193                               int prot, gfp_t gfp, size_t *mapped)
1194 {
1195         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1196         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1197         int ret;
1198
1199         if (!ops)
1200                 return -ENODEV;
1201
1202         arm_smmu_rpm_get(smmu);
1203         ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1204         arm_smmu_rpm_put(smmu);
1205
1206         return ret;
1207 }
1208
1209 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1210                                    size_t pgsize, size_t pgcount,
1211                                    struct iommu_iotlb_gather *iotlb_gather)
1212 {
1213         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1214         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1215         size_t ret;
1216
1217         if (!ops)
1218                 return 0;
1219
1220         arm_smmu_rpm_get(smmu);
1221         ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1222         arm_smmu_rpm_put(smmu);
1223
1224         return ret;
1225 }
1226
1227 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1228 {
1229         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1230         struct arm_smmu_device *smmu = smmu_domain->smmu;
1231
1232         if (smmu_domain->flush_ops) {
1233                 arm_smmu_rpm_get(smmu);
1234                 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1235                 arm_smmu_rpm_put(smmu);
1236         }
1237 }
1238
1239 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1240                                 struct iommu_iotlb_gather *gather)
1241 {
1242         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1243         struct arm_smmu_device *smmu = smmu_domain->smmu;
1244
1245         if (!smmu)
1246                 return;
1247
1248         arm_smmu_rpm_get(smmu);
1249         if (smmu->version == ARM_SMMU_V2 ||
1250             smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1251                 arm_smmu_tlb_sync_context(smmu_domain);
1252         else
1253                 arm_smmu_tlb_sync_global(smmu);
1254         arm_smmu_rpm_put(smmu);
1255 }
1256
1257 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1258                                               dma_addr_t iova)
1259 {
1260         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1261         struct arm_smmu_device *smmu = smmu_domain->smmu;
1262         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1263         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1264         struct device *dev = smmu->dev;
1265         void __iomem *reg;
1266         u32 tmp;
1267         u64 phys;
1268         unsigned long va, flags;
1269         int ret, idx = cfg->cbndx;
1270         phys_addr_t addr = 0;
1271
1272         ret = arm_smmu_rpm_get(smmu);
1273         if (ret < 0)
1274                 return 0;
1275
1276         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1277         va = iova & ~0xfffUL;
1278         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1279                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1280         else
1281                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1282
1283         reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1284         if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1285                                       5, 50)) {
1286                 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1287                 dev_err(dev,
1288                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1289                         &iova);
1290                 arm_smmu_rpm_put(smmu);
1291                 return ops->iova_to_phys(ops, iova);
1292         }
1293
1294         phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1295         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1296         if (phys & ARM_SMMU_CB_PAR_F) {
1297                 dev_err(dev, "translation fault!\n");
1298                 dev_err(dev, "PAR = 0x%llx\n", phys);
1299                 goto out;
1300         }
1301
1302         addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1303 out:
1304         arm_smmu_rpm_put(smmu);
1305
1306         return addr;
1307 }
1308
1309 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1310                                         dma_addr_t iova)
1311 {
1312         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1313         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1314
1315         if (!ops)
1316                 return 0;
1317
1318         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1319                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1320                 return arm_smmu_iova_to_phys_hard(domain, iova);
1321
1322         return ops->iova_to_phys(ops, iova);
1323 }
1324
1325 static bool arm_smmu_capable(enum iommu_cap cap)
1326 {
1327         switch (cap) {
1328         case IOMMU_CAP_CACHE_COHERENCY:
1329                 /*
1330                  * Return true here as the SMMU can always send out coherent
1331                  * requests.
1332                  */
1333                 return true;
1334         case IOMMU_CAP_NOEXEC:
1335                 return true;
1336         default:
1337                 return false;
1338         }
1339 }
1340
1341 static
1342 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1343 {
1344         struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1345                                                           fwnode);
1346         put_device(dev);
1347         return dev ? dev_get_drvdata(dev) : NULL;
1348 }
1349
1350 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1351 {
1352         struct arm_smmu_device *smmu = NULL;
1353         struct arm_smmu_master_cfg *cfg;
1354         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1355         int i, ret;
1356
1357         if (using_legacy_binding) {
1358                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1359
1360                 /*
1361                  * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1362                  * will allocate/initialise a new one. Thus we need to update fwspec for
1363                  * later use.
1364                  */
1365                 fwspec = dev_iommu_fwspec_get(dev);
1366                 if (ret)
1367                         goto out_free;
1368         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1369                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1370         } else {
1371                 return ERR_PTR(-ENODEV);
1372         }
1373
1374         ret = -EINVAL;
1375         for (i = 0; i < fwspec->num_ids; i++) {
1376                 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1377                 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1378
1379                 if (sid & ~smmu->streamid_mask) {
1380                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1381                                 sid, smmu->streamid_mask);
1382                         goto out_free;
1383                 }
1384                 if (mask & ~smmu->smr_mask_mask) {
1385                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1386                                 mask, smmu->smr_mask_mask);
1387                         goto out_free;
1388                 }
1389         }
1390
1391         ret = -ENOMEM;
1392         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1393                       GFP_KERNEL);
1394         if (!cfg)
1395                 goto out_free;
1396
1397         cfg->smmu = smmu;
1398         dev_iommu_priv_set(dev, cfg);
1399         while (i--)
1400                 cfg->smendx[i] = INVALID_SMENDX;
1401
1402         ret = arm_smmu_rpm_get(smmu);
1403         if (ret < 0)
1404                 goto out_cfg_free;
1405
1406         ret = arm_smmu_master_alloc_smes(dev);
1407         arm_smmu_rpm_put(smmu);
1408
1409         if (ret)
1410                 goto out_cfg_free;
1411
1412         device_link_add(dev, smmu->dev,
1413                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1414
1415         return &smmu->iommu;
1416
1417 out_cfg_free:
1418         kfree(cfg);
1419 out_free:
1420         iommu_fwspec_free(dev);
1421         return ERR_PTR(ret);
1422 }
1423
1424 static void arm_smmu_release_device(struct device *dev)
1425 {
1426         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1427         struct arm_smmu_master_cfg *cfg;
1428         struct arm_smmu_device *smmu;
1429         int ret;
1430
1431         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1432                 return;
1433
1434         cfg  = dev_iommu_priv_get(dev);
1435         smmu = cfg->smmu;
1436
1437         ret = arm_smmu_rpm_get(smmu);
1438         if (ret < 0)
1439                 return;
1440
1441         arm_smmu_master_free_smes(cfg, fwspec);
1442
1443         arm_smmu_rpm_put(smmu);
1444
1445         dev_iommu_priv_set(dev, NULL);
1446         kfree(cfg);
1447         iommu_fwspec_free(dev);
1448 }
1449
1450 static void arm_smmu_probe_finalize(struct device *dev)
1451 {
1452         struct arm_smmu_master_cfg *cfg;
1453         struct arm_smmu_device *smmu;
1454
1455         cfg = dev_iommu_priv_get(dev);
1456         smmu = cfg->smmu;
1457
1458         if (smmu->impl && smmu->impl->probe_finalize)
1459                 smmu->impl->probe_finalize(smmu, dev);
1460 }
1461
1462 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1463 {
1464         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1465         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1466         struct arm_smmu_device *smmu = cfg->smmu;
1467         struct iommu_group *group = NULL;
1468         int i, idx;
1469
1470         for_each_cfg_sme(cfg, fwspec, i, idx) {
1471                 if (group && smmu->s2crs[idx].group &&
1472                     group != smmu->s2crs[idx].group)
1473                         return ERR_PTR(-EINVAL);
1474
1475                 group = smmu->s2crs[idx].group;
1476         }
1477
1478         if (group)
1479                 return iommu_group_ref_get(group);
1480
1481         if (dev_is_pci(dev))
1482                 group = pci_device_group(dev);
1483         else if (dev_is_fsl_mc(dev))
1484                 group = fsl_mc_device_group(dev);
1485         else
1486                 group = generic_device_group(dev);
1487
1488         /* Remember group for faster lookups */
1489         if (!IS_ERR(group))
1490                 for_each_cfg_sme(cfg, fwspec, i, idx)
1491                         smmu->s2crs[idx].group = group;
1492
1493         return group;
1494 }
1495
1496 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
1497 {
1498         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1499         int ret = 0;
1500
1501         mutex_lock(&smmu_domain->init_mutex);
1502         if (smmu_domain->smmu)
1503                 ret = -EPERM;
1504         else
1505                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1506         mutex_unlock(&smmu_domain->init_mutex);
1507
1508         return ret;
1509 }
1510
1511 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1512                 unsigned long quirks)
1513 {
1514         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1515         int ret = 0;
1516
1517         mutex_lock(&smmu_domain->init_mutex);
1518         if (smmu_domain->smmu)
1519                 ret = -EPERM;
1520         else
1521                 smmu_domain->pgtbl_quirks = quirks;
1522         mutex_unlock(&smmu_domain->init_mutex);
1523
1524         return ret;
1525 }
1526
1527 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1528 {
1529         u32 mask, fwid = 0;
1530
1531         if (args->args_count > 0)
1532                 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1533
1534         if (args->args_count > 1)
1535                 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1536         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1537                 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1538
1539         return iommu_fwspec_add_ids(dev, &fwid, 1);
1540 }
1541
1542 static void arm_smmu_get_resv_regions(struct device *dev,
1543                                       struct list_head *head)
1544 {
1545         struct iommu_resv_region *region;
1546         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1547
1548         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1549                                          prot, IOMMU_RESV_SW_MSI);
1550         if (!region)
1551                 return;
1552
1553         list_add_tail(&region->list, head);
1554
1555         iommu_dma_get_resv_regions(dev, head);
1556 }
1557
1558 static int arm_smmu_def_domain_type(struct device *dev)
1559 {
1560         struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1561         const struct arm_smmu_impl *impl = cfg->smmu->impl;
1562
1563         if (impl && impl->def_domain_type)
1564                 return impl->def_domain_type(dev);
1565
1566         return 0;
1567 }
1568
1569 static struct iommu_ops arm_smmu_ops = {
1570         .capable                = arm_smmu_capable,
1571         .domain_alloc           = arm_smmu_domain_alloc,
1572         .domain_free            = arm_smmu_domain_free,
1573         .attach_dev             = arm_smmu_attach_dev,
1574         .map_pages              = arm_smmu_map_pages,
1575         .unmap_pages            = arm_smmu_unmap_pages,
1576         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
1577         .iotlb_sync             = arm_smmu_iotlb_sync,
1578         .iova_to_phys           = arm_smmu_iova_to_phys,
1579         .probe_device           = arm_smmu_probe_device,
1580         .release_device         = arm_smmu_release_device,
1581         .probe_finalize         = arm_smmu_probe_finalize,
1582         .device_group           = arm_smmu_device_group,
1583         .enable_nesting         = arm_smmu_enable_nesting,
1584         .set_pgtable_quirks     = arm_smmu_set_pgtable_quirks,
1585         .of_xlate               = arm_smmu_of_xlate,
1586         .get_resv_regions       = arm_smmu_get_resv_regions,
1587         .put_resv_regions       = generic_iommu_put_resv_regions,
1588         .def_domain_type        = arm_smmu_def_domain_type,
1589         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1590         .owner                  = THIS_MODULE,
1591 };
1592
1593 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1594 {
1595         int i;
1596         u32 reg;
1597
1598         /* clear global FSR */
1599         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1600         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1601
1602         /*
1603          * Reset stream mapping groups: Initial values mark all SMRn as
1604          * invalid and all S2CRn as bypass unless overridden.
1605          */
1606         for (i = 0; i < smmu->num_mapping_groups; ++i)
1607                 arm_smmu_write_sme(smmu, i);
1608
1609         /* Make sure all context banks are disabled and clear CB_FSR  */
1610         for (i = 0; i < smmu->num_context_banks; ++i) {
1611                 arm_smmu_write_context_bank(smmu, i);
1612                 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1613         }
1614
1615         /* Invalidate the TLB, just in case */
1616         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1617         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1618
1619         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1620
1621         /* Enable fault reporting */
1622         reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1623                 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1624
1625         /* Disable TLB broadcasting. */
1626         reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1627
1628         /* Enable client access, handling unmatched streams as appropriate */
1629         reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1630         if (disable_bypass)
1631                 reg |= ARM_SMMU_sCR0_USFCFG;
1632         else
1633                 reg &= ~ARM_SMMU_sCR0_USFCFG;
1634
1635         /* Disable forced broadcasting */
1636         reg &= ~ARM_SMMU_sCR0_FB;
1637
1638         /* Don't upgrade barriers */
1639         reg &= ~(ARM_SMMU_sCR0_BSU);
1640
1641         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1642                 reg |= ARM_SMMU_sCR0_VMID16EN;
1643
1644         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1645                 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1646
1647         if (smmu->impl && smmu->impl->reset)
1648                 smmu->impl->reset(smmu);
1649
1650         /* Push the button */
1651         arm_smmu_tlb_sync_global(smmu);
1652         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1653 }
1654
1655 static int arm_smmu_id_size_to_bits(int size)
1656 {
1657         switch (size) {
1658         case 0:
1659                 return 32;
1660         case 1:
1661                 return 36;
1662         case 2:
1663                 return 40;
1664         case 3:
1665                 return 42;
1666         case 4:
1667                 return 44;
1668         case 5:
1669         default:
1670                 return 48;
1671         }
1672 }
1673
1674 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1675 {
1676         unsigned int size;
1677         u32 id;
1678         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1679         int i, ret;
1680
1681         dev_notice(smmu->dev, "probing hardware configuration...\n");
1682         dev_notice(smmu->dev, "SMMUv%d with:\n",
1683                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1684
1685         /* ID0 */
1686         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1687
1688         /* Restrict available stages based on module parameter */
1689         if (force_stage == 1)
1690                 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1691         else if (force_stage == 2)
1692                 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1693
1694         if (id & ARM_SMMU_ID0_S1TS) {
1695                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1696                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1697         }
1698
1699         if (id & ARM_SMMU_ID0_S2TS) {
1700                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1701                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1702         }
1703
1704         if (id & ARM_SMMU_ID0_NTS) {
1705                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1706                 dev_notice(smmu->dev, "\tnested translation\n");
1707         }
1708
1709         if (!(smmu->features &
1710                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1711                 dev_err(smmu->dev, "\tno translation support!\n");
1712                 return -ENODEV;
1713         }
1714
1715         if ((id & ARM_SMMU_ID0_S1TS) &&
1716             ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1717                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1718                 dev_notice(smmu->dev, "\taddress translation ops\n");
1719         }
1720
1721         /*
1722          * In order for DMA API calls to work properly, we must defer to what
1723          * the FW says about coherency, regardless of what the hardware claims.
1724          * Fortunately, this also opens up a workaround for systems where the
1725          * ID register value has ended up configured incorrectly.
1726          */
1727         cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1728         if (cttw_fw || cttw_reg)
1729                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1730                            cttw_fw ? "" : "non-");
1731         if (cttw_fw != cttw_reg)
1732                 dev_notice(smmu->dev,
1733                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1734
1735         /* Max. number of entries we have for stream matching/indexing */
1736         if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1737                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1738                 size = 1 << 16;
1739         } else {
1740                 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1741         }
1742         smmu->streamid_mask = size - 1;
1743         if (id & ARM_SMMU_ID0_SMS) {
1744                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1745                 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1746                 if (size == 0) {
1747                         dev_err(smmu->dev,
1748                                 "stream-matching supported, but no SMRs present!\n");
1749                         return -ENODEV;
1750                 }
1751
1752                 /* Zero-initialised to mark as invalid */
1753                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1754                                           GFP_KERNEL);
1755                 if (!smmu->smrs)
1756                         return -ENOMEM;
1757
1758                 dev_notice(smmu->dev,
1759                            "\tstream matching with %u register groups", size);
1760         }
1761         /* s2cr->type == 0 means translation, so initialise explicitly */
1762         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1763                                          GFP_KERNEL);
1764         if (!smmu->s2crs)
1765                 return -ENOMEM;
1766         for (i = 0; i < size; i++)
1767                 smmu->s2crs[i] = s2cr_init_val;
1768
1769         smmu->num_mapping_groups = size;
1770         mutex_init(&smmu->stream_map_mutex);
1771         spin_lock_init(&smmu->global_sync_lock);
1772
1773         if (smmu->version < ARM_SMMU_V2 ||
1774             !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1775                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1776                 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1777                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1778         }
1779
1780         /* ID1 */
1781         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1782         smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1783
1784         /* Check for size mismatch of SMMU address space from mapped region */
1785         size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1786         if (smmu->numpage != 2 * size << smmu->pgshift)
1787                 dev_warn(smmu->dev,
1788                         "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1789                         2 * size << smmu->pgshift, smmu->numpage);
1790         /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1791         smmu->numpage = size;
1792
1793         smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1794         smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1795         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1796                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1797                 return -ENODEV;
1798         }
1799         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1800                    smmu->num_context_banks, smmu->num_s2_context_banks);
1801         smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1802                                  sizeof(*smmu->cbs), GFP_KERNEL);
1803         if (!smmu->cbs)
1804                 return -ENOMEM;
1805
1806         /* ID2 */
1807         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1808         size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1809         smmu->ipa_size = size;
1810
1811         /* The output mask is also applied for bypass */
1812         size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1813         smmu->pa_size = size;
1814
1815         if (id & ARM_SMMU_ID2_VMID16)
1816                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1817
1818         /*
1819          * What the page table walker can address actually depends on which
1820          * descriptor format is in use, but since a) we don't know that yet,
1821          * and b) it can vary per context bank, this will have to do...
1822          */
1823         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1824                 dev_warn(smmu->dev,
1825                          "failed to set DMA mask for table walker\n");
1826
1827         if (smmu->version < ARM_SMMU_V2) {
1828                 smmu->va_size = smmu->ipa_size;
1829                 if (smmu->version == ARM_SMMU_V1_64K)
1830                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1831         } else {
1832                 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1833                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1834                 if (id & ARM_SMMU_ID2_PTFS_4K)
1835                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1836                 if (id & ARM_SMMU_ID2_PTFS_16K)
1837                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1838                 if (id & ARM_SMMU_ID2_PTFS_64K)
1839                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1840         }
1841
1842         if (smmu->impl && smmu->impl->cfg_probe) {
1843                 ret = smmu->impl->cfg_probe(smmu);
1844                 if (ret)
1845                         return ret;
1846         }
1847
1848         /* Now we've corralled the various formats, what'll it do? */
1849         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1850                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1851         if (smmu->features &
1852             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1853                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1854         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1855                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1856         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1857                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1858
1859         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1860                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1861         else
1862                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1863         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1864                    smmu->pgsize_bitmap);
1865
1866
1867         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1868                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1869                            smmu->va_size, smmu->ipa_size);
1870
1871         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1872                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1873                            smmu->ipa_size, smmu->pa_size);
1874
1875         return 0;
1876 }
1877
1878 struct arm_smmu_match_data {
1879         enum arm_smmu_arch_version version;
1880         enum arm_smmu_implementation model;
1881 };
1882
1883 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1884 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1885
1886 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1887 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1888 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1889 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1890 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1891 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1892
1893 static const struct of_device_id arm_smmu_of_match[] = {
1894         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1895         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1896         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1897         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1898         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1899         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1900         { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1901         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1902         { },
1903 };
1904 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1905
1906 #ifdef CONFIG_ACPI
1907 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1908 {
1909         int ret = 0;
1910
1911         switch (model) {
1912         case ACPI_IORT_SMMU_V1:
1913         case ACPI_IORT_SMMU_CORELINK_MMU400:
1914                 smmu->version = ARM_SMMU_V1;
1915                 smmu->model = GENERIC_SMMU;
1916                 break;
1917         case ACPI_IORT_SMMU_CORELINK_MMU401:
1918                 smmu->version = ARM_SMMU_V1_64K;
1919                 smmu->model = GENERIC_SMMU;
1920                 break;
1921         case ACPI_IORT_SMMU_V2:
1922                 smmu->version = ARM_SMMU_V2;
1923                 smmu->model = GENERIC_SMMU;
1924                 break;
1925         case ACPI_IORT_SMMU_CORELINK_MMU500:
1926                 smmu->version = ARM_SMMU_V2;
1927                 smmu->model = ARM_MMU500;
1928                 break;
1929         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1930                 smmu->version = ARM_SMMU_V2;
1931                 smmu->model = CAVIUM_SMMUV2;
1932                 break;
1933         default:
1934                 ret = -ENODEV;
1935         }
1936
1937         return ret;
1938 }
1939
1940 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1941                                       struct arm_smmu_device *smmu)
1942 {
1943         struct device *dev = smmu->dev;
1944         struct acpi_iort_node *node =
1945                 *(struct acpi_iort_node **)dev_get_platdata(dev);
1946         struct acpi_iort_smmu *iort_smmu;
1947         int ret;
1948
1949         /* Retrieve SMMU1/2 specific data */
1950         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1951
1952         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1953         if (ret < 0)
1954                 return ret;
1955
1956         /* Ignore the configuration access interrupt */
1957         smmu->num_global_irqs = 1;
1958
1959         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1960                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1961
1962         return 0;
1963 }
1964 #else
1965 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1966                                              struct arm_smmu_device *smmu)
1967 {
1968         return -ENODEV;
1969 }
1970 #endif
1971
1972 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
1973                                     struct arm_smmu_device *smmu)
1974 {
1975         const struct arm_smmu_match_data *data;
1976         struct device *dev = &pdev->dev;
1977         bool legacy_binding;
1978
1979         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1980                                  &smmu->num_global_irqs)) {
1981                 dev_err(dev, "missing #global-interrupts property\n");
1982                 return -ENODEV;
1983         }
1984
1985         data = of_device_get_match_data(dev);
1986         smmu->version = data->version;
1987         smmu->model = data->model;
1988
1989         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1990         if (legacy_binding && !using_generic_binding) {
1991                 if (!using_legacy_binding) {
1992                         pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
1993                                   IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
1994                 }
1995                 using_legacy_binding = true;
1996         } else if (!legacy_binding && !using_legacy_binding) {
1997                 using_generic_binding = true;
1998         } else {
1999                 dev_err(dev, "not probing due to mismatched DT properties\n");
2000                 return -ENODEV;
2001         }
2002
2003         if (of_dma_is_coherent(dev->of_node))
2004                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2005
2006         return 0;
2007 }
2008
2009 static int arm_smmu_bus_init(struct iommu_ops *ops)
2010 {
2011         int err;
2012
2013         /* Oh, for a proper bus abstraction */
2014         if (!iommu_present(&platform_bus_type)) {
2015                 err = bus_set_iommu(&platform_bus_type, ops);
2016                 if (err)
2017                         return err;
2018         }
2019 #ifdef CONFIG_ARM_AMBA
2020         if (!iommu_present(&amba_bustype)) {
2021                 err = bus_set_iommu(&amba_bustype, ops);
2022                 if (err)
2023                         goto err_reset_platform_ops;
2024         }
2025 #endif
2026 #ifdef CONFIG_PCI
2027         if (!iommu_present(&pci_bus_type)) {
2028                 err = bus_set_iommu(&pci_bus_type, ops);
2029                 if (err)
2030                         goto err_reset_amba_ops;
2031         }
2032 #endif
2033 #ifdef CONFIG_FSL_MC_BUS
2034         if (!iommu_present(&fsl_mc_bus_type)) {
2035                 err = bus_set_iommu(&fsl_mc_bus_type, ops);
2036                 if (err)
2037                         goto err_reset_pci_ops;
2038         }
2039 #endif
2040         return 0;
2041
2042 err_reset_pci_ops: __maybe_unused;
2043 #ifdef CONFIG_PCI
2044         bus_set_iommu(&pci_bus_type, NULL);
2045 #endif
2046 err_reset_amba_ops: __maybe_unused;
2047 #ifdef CONFIG_ARM_AMBA
2048         bus_set_iommu(&amba_bustype, NULL);
2049 #endif
2050 err_reset_platform_ops: __maybe_unused;
2051         bus_set_iommu(&platform_bus_type, NULL);
2052         return err;
2053 }
2054
2055 static int arm_smmu_device_probe(struct platform_device *pdev)
2056 {
2057         struct resource *res;
2058         resource_size_t ioaddr;
2059         struct arm_smmu_device *smmu;
2060         struct device *dev = &pdev->dev;
2061         int num_irqs, i, err;
2062         irqreturn_t (*global_fault)(int irq, void *dev);
2063
2064         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2065         if (!smmu) {
2066                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2067                 return -ENOMEM;
2068         }
2069         smmu->dev = dev;
2070
2071         if (dev->of_node)
2072                 err = arm_smmu_device_dt_probe(pdev, smmu);
2073         else
2074                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2075
2076         if (err)
2077                 return err;
2078
2079         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2080         ioaddr = res->start;
2081         smmu->base = devm_ioremap_resource(dev, res);
2082         if (IS_ERR(smmu->base))
2083                 return PTR_ERR(smmu->base);
2084         /*
2085          * The resource size should effectively match the value of SMMU_TOP;
2086          * stash that temporarily until we know PAGESIZE to validate it with.
2087          */
2088         smmu->numpage = resource_size(res);
2089
2090         smmu = arm_smmu_impl_init(smmu);
2091         if (IS_ERR(smmu))
2092                 return PTR_ERR(smmu);
2093
2094         num_irqs = 0;
2095         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2096                 num_irqs++;
2097                 if (num_irqs > smmu->num_global_irqs)
2098                         smmu->num_context_irqs++;
2099         }
2100
2101         if (!smmu->num_context_irqs) {
2102                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2103                         num_irqs, smmu->num_global_irqs + 1);
2104                 return -ENODEV;
2105         }
2106
2107         smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2108                                   GFP_KERNEL);
2109         if (!smmu->irqs) {
2110                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2111                 return -ENOMEM;
2112         }
2113
2114         for (i = 0; i < num_irqs; ++i) {
2115                 int irq = platform_get_irq(pdev, i);
2116
2117                 if (irq < 0)
2118                         return -ENODEV;
2119                 smmu->irqs[i] = irq;
2120         }
2121
2122         err = devm_clk_bulk_get_all(dev, &smmu->clks);
2123         if (err < 0) {
2124                 dev_err(dev, "failed to get clocks %d\n", err);
2125                 return err;
2126         }
2127         smmu->num_clks = err;
2128
2129         err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2130         if (err)
2131                 return err;
2132
2133         err = arm_smmu_device_cfg_probe(smmu);
2134         if (err)
2135                 return err;
2136
2137         if (smmu->version == ARM_SMMU_V2) {
2138                 if (smmu->num_context_banks > smmu->num_context_irqs) {
2139                         dev_err(dev,
2140                               "found only %d context irq(s) but %d required\n",
2141                               smmu->num_context_irqs, smmu->num_context_banks);
2142                         return -ENODEV;
2143                 }
2144
2145                 /* Ignore superfluous interrupts */
2146                 smmu->num_context_irqs = smmu->num_context_banks;
2147         }
2148
2149         if (smmu->impl && smmu->impl->global_fault)
2150                 global_fault = smmu->impl->global_fault;
2151         else
2152                 global_fault = arm_smmu_global_fault;
2153
2154         for (i = 0; i < smmu->num_global_irqs; ++i) {
2155                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2156                                        global_fault,
2157                                        IRQF_SHARED,
2158                                        "arm-smmu global fault",
2159                                        smmu);
2160                 if (err) {
2161                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2162                                 i, smmu->irqs[i]);
2163                         return err;
2164                 }
2165         }
2166
2167         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2168                                      "smmu.%pa", &ioaddr);
2169         if (err) {
2170                 dev_err(dev, "Failed to register iommu in sysfs\n");
2171                 return err;
2172         }
2173
2174         err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
2175         if (err) {
2176                 dev_err(dev, "Failed to register iommu\n");
2177                 goto err_sysfs_remove;
2178         }
2179
2180         platform_set_drvdata(pdev, smmu);
2181         arm_smmu_device_reset(smmu);
2182         arm_smmu_test_smr_masks(smmu);
2183
2184         /*
2185          * We want to avoid touching dev->power.lock in fastpaths unless
2186          * it's really going to do something useful - pm_runtime_enabled()
2187          * can serve as an ideal proxy for that decision. So, conditionally
2188          * enable pm_runtime.
2189          */
2190         if (dev->pm_domain) {
2191                 pm_runtime_set_active(dev);
2192                 pm_runtime_enable(dev);
2193         }
2194
2195         /*
2196          * For ACPI and generic DT bindings, an SMMU will be probed before
2197          * any device which might need it, so we want the bus ops in place
2198          * ready to handle default domain setup as soon as any SMMU exists.
2199          */
2200         if (!using_legacy_binding) {
2201                 err = arm_smmu_bus_init(&arm_smmu_ops);
2202                 if (err)
2203                         goto err_unregister_device;
2204         }
2205
2206         return 0;
2207
2208 err_unregister_device:
2209         iommu_device_unregister(&smmu->iommu);
2210 err_sysfs_remove:
2211         iommu_device_sysfs_remove(&smmu->iommu);
2212         return err;
2213 }
2214
2215 static int arm_smmu_device_remove(struct platform_device *pdev)
2216 {
2217         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2218
2219         if (!smmu)
2220                 return -ENODEV;
2221
2222         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2223                 dev_notice(&pdev->dev, "disabling translation\n");
2224
2225         arm_smmu_bus_init(NULL);
2226         iommu_device_unregister(&smmu->iommu);
2227         iommu_device_sysfs_remove(&smmu->iommu);
2228
2229         arm_smmu_rpm_get(smmu);
2230         /* Turn the thing off */
2231         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2232         arm_smmu_rpm_put(smmu);
2233
2234         if (pm_runtime_enabled(smmu->dev))
2235                 pm_runtime_force_suspend(smmu->dev);
2236         else
2237                 clk_bulk_disable(smmu->num_clks, smmu->clks);
2238
2239         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2240         return 0;
2241 }
2242
2243 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2244 {
2245         arm_smmu_device_remove(pdev);
2246 }
2247
2248 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2249 {
2250         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2251         int ret;
2252
2253         ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2254         if (ret)
2255                 return ret;
2256
2257         arm_smmu_device_reset(smmu);
2258
2259         return 0;
2260 }
2261
2262 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2263 {
2264         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2265
2266         clk_bulk_disable(smmu->num_clks, smmu->clks);
2267
2268         return 0;
2269 }
2270
2271 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2272 {
2273         if (pm_runtime_suspended(dev))
2274                 return 0;
2275
2276         return arm_smmu_runtime_resume(dev);
2277 }
2278
2279 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2280 {
2281         if (pm_runtime_suspended(dev))
2282                 return 0;
2283
2284         return arm_smmu_runtime_suspend(dev);
2285 }
2286
2287 static const struct dev_pm_ops arm_smmu_pm_ops = {
2288         SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2289         SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2290                            arm_smmu_runtime_resume, NULL)
2291 };
2292
2293 static struct platform_driver arm_smmu_driver = {
2294         .driver = {
2295                 .name                   = "arm-smmu",
2296                 .of_match_table         = arm_smmu_of_match,
2297                 .pm                     = &arm_smmu_pm_ops,
2298                 .suppress_bind_attrs    = true,
2299         },
2300         .probe  = arm_smmu_device_probe,
2301         .remove = arm_smmu_device_remove,
2302         .shutdown = arm_smmu_device_shutdown,
2303 };
2304 module_platform_driver(arm_smmu_driver);
2305
2306 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2307 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2308 MODULE_ALIAS("platform:arm-smmu");
2309 MODULE_LICENSE("GPL v2");