drivers/iommu/arm-smmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * IOMMU API for ARM architected SMMU implementations.
   4  *
   5  * Copyright (C) 2013 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  *
   9  * This driver currently supports:
  10  *      - SMMUv1 and v2 implementations
  11  *      - Stream-matching and stream-indexing
  12  *      - v7/v8 long-descriptor format
  13  *      - Non-secure access to the SMMU
  14  *      - Context fault reporting
  15  *      - Extended Stream ID (16 bit)
  16  */
  17
  18 #define pr_fmt(fmt) "arm-smmu: " fmt
  19
  20 #include <linux/acpi.h>
  21 #include <linux/acpi_iort.h>
  22 #include <linux/bitfield.h>
  23 #include <linux/delay.h>
  24 #include <linux/dma-iommu.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/err.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/io.h>
  29 #include <linux/iopoll.h>
  30 #include <linux/init.h>
  31 #include <linux/moduleparam.h>
  32 #include <linux/of.h>
  33 #include <linux/of_address.h>
  34 #include <linux/of_device.h>
  35 #include <linux/of_iommu.h>
  36 #include <linux/pci.h>
  37 #include <linux/platform_device.h>
  38 #include <linux/pm_runtime.h>
  39 #include <linux/slab.h>
  40
  41 #include <linux/amba/bus.h>
  42 #include <linux/fsl/mc.h>
  43
  44 #include "arm-smmu.h"
  45
  46 /*
  47  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
  48  * global register space are still, in fact, using a hypervisor to mediate it
  49  * by trapping and emulating register accesses. Sadly, some deployed versions
  50  * of said trapping code have bugs wherein they go horribly wrong for stores
  51  * using r31 (i.e. XZR/WZR) as the source register.
  52  */
  53 #define QCOM_DUMMY_VAL -1
  54
  55 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
  56 #define TLB_SPIN_COUNT                  10
  57
  58 #define MSI_IOVA_BASE                   0x8000000
  59 #define MSI_IOVA_LENGTH                 0x100000
  60
  61 static int force_stage;
  62 /*
  63  * not really modular, but the easiest way to keep compat with existing
  64  * bootargs behaviour is to continue using module_param() here.
  65  */
  66 module_param(force_stage, int, S_IRUGO);
  67 MODULE_PARM_DESC(force_stage,
  68         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
  69 static bool disable_bypass =
  70         IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
  71 module_param(disable_bypass, bool, S_IRUGO);
  72 MODULE_PARM_DESC(disable_bypass,
  73         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
  74
  75 struct arm_smmu_s2cr {
  76         struct iommu_group              *group;
  77         int                             count;
  78         enum arm_smmu_s2cr_type         type;
  79         enum arm_smmu_s2cr_privcfg      privcfg;
  80         u8                              cbndx;
  81 };
  82
  83 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
  84         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
  85 }
  86
  87 struct arm_smmu_smr {
  88         u16                             mask;
  89         u16                             id;
  90         bool                            valid;
  91 };
  92
  93 struct arm_smmu_cb {
  94         u64                             ttbr[2];
  95         u32                             tcr[2];
  96         u32                             mair[2];
  97         struct arm_smmu_cfg             *cfg;
  98 };
  99
 100 struct arm_smmu_master_cfg {
 101         struct arm_smmu_device          *smmu;
 102         s16                             smendx[];
 103 };
 104 #define INVALID_SMENDX                  -1
 105 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 106 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 107 #define fwspec_smendx(fw, i) \
 108         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 109 #define for_each_cfg_sme(fw, i, idx) \
 110         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 111
 112 static bool using_legacy_binding, using_generic_binding;
 113
 114 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
 115 {
 116         if (pm_runtime_enabled(smmu->dev))
 117                 return pm_runtime_get_sync(smmu->dev);
 118
 119         return 0;
 120 }
 121
 122 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
 123 {
 124         if (pm_runtime_enabled(smmu->dev))
 125                 pm_runtime_put(smmu->dev);
 126 }
 127
 128 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 129 {
 130         return container_of(dom, struct arm_smmu_domain, domain);
 131 }
 132
 133 static struct device_node *dev_get_dev_node(struct device *dev)
 134 {
 135         if (dev_is_pci(dev)) {
 136                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 137
 138                 while (!pci_is_root_bus(bus))
 139                         bus = bus->parent;
 140                 return of_node_get(bus->bridge->parent->of_node);
 141         }
 142
 143         return of_node_get(dev->of_node);
 144 }
 145
 146 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 147 {
 148         *((__be32 *)data) = cpu_to_be32(alias);
 149         return 0; /* Continue walking */
 150 }
 151
 152 static int __find_legacy_master_phandle(struct device *dev, void *data)
 153 {
 154         struct of_phandle_iterator *it = *(void **)data;
 155         struct device_node *np = it->node;
 156         int err;
 157
 158         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 159                             "#stream-id-cells", -1)
 160                 if (it->node == np) {
 161                         *(void **)data = dev;
 162                         return 1;
 163                 }
 164         it->node = np;
 165         return err == -ENOENT ? 0 : err;
 166 }
 167
 168 static struct platform_driver arm_smmu_driver;
 169 static struct iommu_ops arm_smmu_ops;
 170
 171 static int arm_smmu_register_legacy_master(struct device *dev,
 172                                            struct arm_smmu_device **smmu)
 173 {
 174         struct device *smmu_dev;
 175         struct device_node *np;
 176         struct of_phandle_iterator it;
 177         void *data = &it;
 178         u32 *sids;
 179         __be32 pci_sid;
 180         int err;
 181
 182         np = dev_get_dev_node(dev);
 183         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 184                 of_node_put(np);
 185                 return -ENODEV;
 186         }
 187
 188         it.node = np;
 189         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 190                                      __find_legacy_master_phandle);
 191         smmu_dev = data;
 192         of_node_put(np);
 193         if (err == 0)
 194                 return -ENODEV;
 195         if (err < 0)
 196                 return err;
 197
 198         if (dev_is_pci(dev)) {
 199                 /* "mmu-masters" assumes Stream ID == Requester ID */
 200                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 201                                        &pci_sid);
 202                 it.cur = &pci_sid;
 203                 it.cur_count = 1;
 204         }
 205
 206         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 207                                 &arm_smmu_ops);
 208         if (err)
 209                 return err;
 210
 211         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 212         if (!sids)
 213                 return -ENOMEM;
 214
 215         *smmu = dev_get_drvdata(smmu_dev);
 216         of_phandle_iterator_args(&it, sids, it.cur_count);
 217         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 218         kfree(sids);
 219         return err;
 220 }
 221
 222 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 223 {
 224         int idx;
 225
 226         do {
 227                 idx = find_next_zero_bit(map, end, start);
 228                 if (idx == end)
 229                         return -ENOSPC;
 230         } while (test_and_set_bit(idx, map));
 231
 232         return idx;
 233 }
 234
 235 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 236 {
 237         clear_bit(idx, map);
 238 }
 239
 240 /* Wait for any pending TLB invalidations to complete */
 241 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
 242                                 int sync, int status)
 243 {
 244         unsigned int spin_cnt, delay;
 245         u32 reg;
 246
 247         if (smmu->impl && unlikely(smmu->impl->tlb_sync))
 248                 return smmu->impl->tlb_sync(smmu, page, sync, status);
 249
 250         arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
 251         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 252                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 253                         reg = arm_smmu_readl(smmu, page, status);
 254                         if (!(reg & sTLBGSTATUS_GSACTIVE))
 255                                 return;
 256                         cpu_relax();
 257                 }
 258                 udelay(delay);
 259         }
 260         dev_err_ratelimited(smmu->dev,
 261                             "TLB sync timed out -- SMMU may be deadlocked\n");
 262 }
 263
 264 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 265 {
 266         unsigned long flags;
 267
 268         spin_lock_irqsave(&smmu->global_sync_lock, flags);
 269         __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
 270                             ARM_SMMU_GR0_sTLBGSTATUS);
 271         spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 272 }
 273
 274 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
 275 {
 276         struct arm_smmu_device *smmu = smmu_domain->smmu;
 277         unsigned long flags;
 278
 279         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 280         __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
 281                             ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
 282         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 283 }
 284
 285 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 286 {
 287         struct arm_smmu_domain *smmu_domain = cookie;
 288         /*
 289          * The TLBI write may be relaxed, so ensure that PTEs cleared by the
 290          * current CPU are visible beforehand.
 291          */
 292         wmb();
 293         arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
 294                           ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
 295         arm_smmu_tlb_sync_context(smmu_domain);
 296 }
 297
 298 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 299 {
 300         struct arm_smmu_domain *smmu_domain = cookie;
 301         struct arm_smmu_device *smmu = smmu_domain->smmu;
 302
 303         /* See above */
 304         wmb();
 305         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 306         arm_smmu_tlb_sync_global(smmu);
 307 }
 308
 309 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
 310                                       size_t granule, void *cookie, int reg)
 311 {
 312         struct arm_smmu_domain *smmu_domain = cookie;
 313         struct arm_smmu_device *smmu = smmu_domain->smmu;
 314         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 315         int idx = cfg->cbndx;
 316
 317         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 318                 wmb();
 319
 320         if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 321                 iova = (iova >> 12) << 12;
 322                 iova |= cfg->asid;
 323                 do {
 324                         arm_smmu_cb_write(smmu, idx, reg, iova);
 325                         iova += granule;
 326                 } while (size -= granule);
 327         } else {
 328                 iova >>= 12;
 329                 iova |= (u64)cfg->asid << 48;
 330                 do {
 331                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 332                         iova += granule >> 12;
 333                 } while (size -= granule);
 334         }
 335 }
 336
 337 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
 338                                       size_t granule, void *cookie, int reg)
 339 {
 340         struct arm_smmu_domain *smmu_domain = cookie;
 341         struct arm_smmu_device *smmu = smmu_domain->smmu;
 342         int idx = smmu_domain->cfg.cbndx;
 343
 344         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 345                 wmb();
 346
 347         iova >>= 12;
 348         do {
 349                 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
 350                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 351                 else
 352                         arm_smmu_cb_write(smmu, idx, reg, iova);
 353                 iova += granule >> 12;
 354         } while (size -= granule);
 355 }
 356
 357 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
 358                                      size_t granule, void *cookie)
 359 {
 360         arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 361                                   ARM_SMMU_CB_S1_TLBIVA);
 362         arm_smmu_tlb_sync_context(cookie);
 363 }
 364
 365 static void arm_smmu_tlb_inv_leaf_s1(unsigned long iova, size_t size,
 366                                      size_t granule, void *cookie)
 367 {
 368         arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 369                                   ARM_SMMU_CB_S1_TLBIVAL);
 370         arm_smmu_tlb_sync_context(cookie);
 371 }
 372
 373 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
 374                                      unsigned long iova, size_t granule,
 375                                      void *cookie)
 376 {
 377         arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
 378                                   ARM_SMMU_CB_S1_TLBIVAL);
 379 }
 380
 381 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
 382                                      size_t granule, void *cookie)
 383 {
 384         arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 385                                   ARM_SMMU_CB_S2_TLBIIPAS2);
 386         arm_smmu_tlb_sync_context(cookie);
 387 }
 388
 389 static void arm_smmu_tlb_inv_leaf_s2(unsigned long iova, size_t size,
 390                                      size_t granule, void *cookie)
 391 {
 392         arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 393                                   ARM_SMMU_CB_S2_TLBIIPAS2L);
 394         arm_smmu_tlb_sync_context(cookie);
 395 }
 396
 397 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
 398                                      unsigned long iova, size_t granule,
 399                                      void *cookie)
 400 {
 401         arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
 402                                   ARM_SMMU_CB_S2_TLBIIPAS2L);
 403 }
 404
 405 static void arm_smmu_tlb_inv_any_s2_v1(unsigned long iova, size_t size,
 406                                        size_t granule, void *cookie)
 407 {
 408         arm_smmu_tlb_inv_context_s2(cookie);
 409 }
 410 /*
 411  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 412  * almost negligible, but the benefit of getting the first one in as far ahead
 413  * of the sync as possible is significant, hence we don't just make this a
 414  * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
 415  * think.
 416  */
 417 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
 418                                         unsigned long iova, size_t granule,
 419                                         void *cookie)
 420 {
 421         struct arm_smmu_domain *smmu_domain = cookie;
 422         struct arm_smmu_device *smmu = smmu_domain->smmu;
 423
 424         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 425                 wmb();
 426
 427         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 428 }
 429
 430 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
 431         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 432         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
 433         .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s1,
 434         .tlb_add_page   = arm_smmu_tlb_add_page_s1,
 435 };
 436
 437 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
 438         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 439         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
 440         .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s2,
 441         .tlb_add_page   = arm_smmu_tlb_add_page_s2,
 442 };
 443
 444 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
 445         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 446         .tlb_flush_walk = arm_smmu_tlb_inv_any_s2_v1,
 447         .tlb_flush_leaf = arm_smmu_tlb_inv_any_s2_v1,
 448         .tlb_add_page   = arm_smmu_tlb_add_page_s2_v1,
 449 };
 450
 451 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 452 {
 453         u32 fsr, fsynr, cbfrsynra;
 454         unsigned long iova;
 455         struct iommu_domain *domain = dev;
 456         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 457         struct arm_smmu_device *smmu = smmu_domain->smmu;
 458         int idx = smmu_domain->cfg.cbndx;
 459
 460         fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
 461         if (!(fsr & FSR_FAULT))
 462                 return IRQ_NONE;
 463
 464         fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
 465         iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
 466         cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
 467
 468         dev_err_ratelimited(smmu->dev,
 469         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 470                             fsr, iova, fsynr, cbfrsynra, idx);
 471
 472         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
 473         return IRQ_HANDLED;
 474 }
 475
 476 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 477 {
 478         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 479         struct arm_smmu_device *smmu = dev;
 480
 481         gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
 482         gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
 483         gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
 484         gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
 485
 486         if (!gfsr)
 487                 return IRQ_NONE;
 488
 489         dev_err_ratelimited(smmu->dev,
 490                 "Unexpected global fault, this could be serious\n");
 491         dev_err_ratelimited(smmu->dev,
 492                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 493                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 494
 495         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
 496         return IRQ_HANDLED;
 497 }
 498
 499 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 500                                        struct io_pgtable_cfg *pgtbl_cfg)
 501 {
 502         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 503         struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 504         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 505
 506         cb->cfg = cfg;
 507
 508         /* TCR */
 509         if (stage1) {
 510                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 511                         cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 512                 } else {
 513                         cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 514                         cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 515                         cb->tcr[1] |= FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM);
 516                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 517                                 cb->tcr[1] |= TCR2_AS;
 518                 }
 519         } else {
 520                 cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 521         }
 522
 523         /* TTBRs */
 524         if (stage1) {
 525                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 526                         cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 527                         cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 528                 } else {
 529                         cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 530                         cb->ttbr[0] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
 531                         cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 532                         cb->ttbr[1] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
 533                 }
 534         } else {
 535                 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 536         }
 537
 538         /* MAIRs (stage-1 only) */
 539         if (stage1) {
 540                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 541                         cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 542                         cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 543                 } else {
 544                         cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 545                         cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 546                 }
 547         }
 548 }
 549
 550 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 551 {
 552         u32 reg;
 553         bool stage1;
 554         struct arm_smmu_cb *cb = &smmu->cbs[idx];
 555         struct arm_smmu_cfg *cfg = cb->cfg;
 556
 557         /* Unassigned context banks only need disabling */
 558         if (!cfg) {
 559                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
 560                 return;
 561         }
 562
 563         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 564
 565         /* CBA2R */
 566         if (smmu->version > ARM_SMMU_V1) {
 567                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 568                         reg = CBA2R_VA64;
 569                 else
 570                         reg = 0;
 571                 /* 16-bit VMIDs live in CBA2R */
 572                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 573                         reg |= FIELD_PREP(CBA2R_VMID16, cfg->vmid);
 574
 575                 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
 576         }
 577
 578         /* CBAR */
 579         reg = FIELD_PREP(CBAR_TYPE, cfg->cbar);
 580         if (smmu->version < ARM_SMMU_V2)
 581                 reg |= FIELD_PREP(CBAR_IRPTNDX, cfg->irptndx);
 582
 583         /*
 584          * Use the weakest shareability/memory types, so they are
 585          * overridden by the ttbcr/pte.
 586          */
 587         if (stage1) {
 588                 reg |= FIELD_PREP(CBAR_S1_BPSHCFG, CBAR_S1_BPSHCFG_NSH) |
 589                         FIELD_PREP(CBAR_S1_MEMATTR, CBAR_S1_MEMATTR_WB);
 590         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 591                 /* 8-bit VMIDs live in CBAR */
 592                 reg |= FIELD_PREP(CBAR_VMID, cfg->vmid);
 593         }
 594         arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
 595
 596         /*
 597          * TCR
 598          * We must write this before the TTBRs, since it determines the
 599          * access behaviour of some fields (in particular, ASID[15:8]).
 600          */
 601         if (stage1 && smmu->version > ARM_SMMU_V1)
 602                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
 603         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
 604
 605         /* TTBRs */
 606         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 607                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
 608                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 609                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
 610         } else {
 611                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 612                 if (stage1)
 613                         arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
 614                                            cb->ttbr[1]);
 615         }
 616
 617         /* MAIRs (stage-1 only) */
 618         if (stage1) {
 619                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
 620                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
 621         }
 622
 623         /* SCTLR */
 624         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 625         if (stage1)
 626                 reg |= SCTLR_S1_ASIDPNE;
 627         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 628                 reg |= SCTLR_E;
 629
 630         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
 631 }
 632
 633 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 634                                         struct arm_smmu_device *smmu)
 635 {
 636         int irq, start, ret = 0;
 637         unsigned long ias, oas;
 638         struct io_pgtable_ops *pgtbl_ops;
 639         struct io_pgtable_cfg pgtbl_cfg;
 640         enum io_pgtable_fmt fmt;
 641         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 642         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 643
 644         mutex_lock(&smmu_domain->init_mutex);
 645         if (smmu_domain->smmu)
 646                 goto out_unlock;
 647
 648         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 649                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 650                 smmu_domain->smmu = smmu;
 651                 goto out_unlock;
 652         }
 653
 654         /*
 655          * Mapping the requested stage onto what we support is surprisingly
 656          * complicated, mainly because the spec allows S1+S2 SMMUs without
 657          * support for nested translation. That means we end up with the
 658          * following table:
 659          *
 660          * Requested        Supported        Actual
 661          *     S1               N              S1
 662          *     S1             S1+S2            S1
 663          *     S1               S2             S2
 664          *     S1               S1             S1
 665          *     N                N              N
 666          *     N              S1+S2            S2
 667          *     N                S2             S2
 668          *     N                S1             S1
 669          *
 670          * Note that you can't actually request stage-2 mappings.
 671          */
 672         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 673                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 674         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 675                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 676
 677         /*
 678          * Choosing a suitable context format is even more fiddly. Until we
 679          * grow some way for the caller to express a preference, and/or move
 680          * the decision into the io-pgtable code where it arguably belongs,
 681          * just aim for the closest thing to the rest of the system, and hope
 682          * that the hardware isn't esoteric enough that we can't assume AArch64
 683          * support to be a superset of AArch32 support...
 684          */
 685         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 686                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 687         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 688             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 689             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 690             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 691                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 692         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 693             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 694                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 695                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 696                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 697
 698         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 699                 ret = -EINVAL;
 700                 goto out_unlock;
 701         }
 702
 703         switch (smmu_domain->stage) {
 704         case ARM_SMMU_DOMAIN_S1:
 705                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 706                 start = smmu->num_s2_context_banks;
 707                 ias = smmu->va_size;
 708                 oas = smmu->ipa_size;
 709                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 710                         fmt = ARM_64_LPAE_S1;
 711                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 712                         fmt = ARM_32_LPAE_S1;
 713                         ias = min(ias, 32UL);
 714                         oas = min(oas, 40UL);
 715                 } else {
 716                         fmt = ARM_V7S;
 717                         ias = min(ias, 32UL);
 718                         oas = min(oas, 32UL);
 719                 }
 720                 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
 721                 break;
 722         case ARM_SMMU_DOMAIN_NESTED:
 723                 /*
 724                  * We will likely want to change this if/when KVM gets
 725                  * involved.
 726                  */
 727         case ARM_SMMU_DOMAIN_S2:
 728                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 729                 start = 0;
 730                 ias = smmu->ipa_size;
 731                 oas = smmu->pa_size;
 732                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 733                         fmt = ARM_64_LPAE_S2;
 734                 } else {
 735                         fmt = ARM_32_LPAE_S2;
 736                         ias = min(ias, 40UL);
 737                         oas = min(oas, 40UL);
 738                 }
 739                 if (smmu->version == ARM_SMMU_V2)
 740                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
 741                 else
 742                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
 743                 break;
 744         default:
 745                 ret = -EINVAL;
 746                 goto out_unlock;
 747         }
 748         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 749                                       smmu->num_context_banks);
 750         if (ret < 0)
 751                 goto out_unlock;
 752
 753         cfg->cbndx = ret;
 754         if (smmu->version < ARM_SMMU_V2) {
 755                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 756                 cfg->irptndx %= smmu->num_context_irqs;
 757         } else {
 758                 cfg->irptndx = cfg->cbndx;
 759         }
 760
 761         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 762                 cfg->vmid = cfg->cbndx + 1;
 763         else
 764                 cfg->asid = cfg->cbndx;
 765
 766         smmu_domain->smmu = smmu;
 767         if (smmu->impl && smmu->impl->init_context) {
 768                 ret = smmu->impl->init_context(smmu_domain);
 769                 if (ret)
 770                         goto out_unlock;
 771         }
 772
 773         pgtbl_cfg = (struct io_pgtable_cfg) {
 774                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 775                 .ias            = ias,
 776                 .oas            = oas,
 777                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
 778                 .tlb            = smmu_domain->flush_ops,
 779                 .iommu_dev      = smmu->dev,
 780         };
 781
 782         if (smmu_domain->non_strict)
 783                 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
 784
 785         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 786         if (!pgtbl_ops) {
 787                 ret = -ENOMEM;
 788                 goto out_clear_smmu;
 789         }
 790
 791         /* Update the domain's page sizes to reflect the page table format */
 792         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 793         domain->geometry.aperture_end = (1UL << ias) - 1;
 794         domain->geometry.force_aperture = true;
 795
 796         /* Initialise the context bank with our page table cfg */
 797         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 798         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 799
 800         /*
 801          * Request context fault interrupt. Do this last to avoid the
 802          * handler seeing a half-initialised domain state.
 803          */
 804         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 805         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 806                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 807         if (ret < 0) {
 808                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 809                         cfg->irptndx, irq);
 810                 cfg->irptndx = INVALID_IRPTNDX;
 811         }
 812
 813         mutex_unlock(&smmu_domain->init_mutex);
 814
 815         /* Publish page table ops for map/unmap */
 816         smmu_domain->pgtbl_ops = pgtbl_ops;
 817         return 0;
 818
 819 out_clear_smmu:
 820         smmu_domain->smmu = NULL;
 821 out_unlock:
 822         mutex_unlock(&smmu_domain->init_mutex);
 823         return ret;
 824 }
 825
 826 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 827 {
 828         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 829         struct arm_smmu_device *smmu = smmu_domain->smmu;
 830         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 831         int ret, irq;
 832
 833         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
 834                 return;
 835
 836         ret = arm_smmu_rpm_get(smmu);
 837         if (ret < 0)
 838                 return;
 839
 840         /*
 841          * Disable the context bank and free the page tables before freeing
 842          * it.
 843          */
 844         smmu->cbs[cfg->cbndx].cfg = NULL;
 845         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 846
 847         if (cfg->irptndx != INVALID_IRPTNDX) {
 848                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 849                 devm_free_irq(smmu->dev, irq, domain);
 850         }
 851
 852         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 853         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 854
 855         arm_smmu_rpm_put(smmu);
 856 }
 857
 858 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 859 {
 860         struct arm_smmu_domain *smmu_domain;
 861
 862         if (type != IOMMU_DOMAIN_UNMANAGED &&
 863             type != IOMMU_DOMAIN_DMA &&
 864             type != IOMMU_DOMAIN_IDENTITY)
 865                 return NULL;
 866         /*
 867          * Allocate the domain and initialise some of its data structures.
 868          * We can't really do anything meaningful until we've added a
 869          * master.
 870          */
 871         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
 872         if (!smmu_domain)
 873                 return NULL;
 874
 875         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
 876             iommu_get_dma_cookie(&smmu_domain->domain))) {
 877                 kfree(smmu_domain);
 878                 return NULL;
 879         }
 880
 881         mutex_init(&smmu_domain->init_mutex);
 882         spin_lock_init(&smmu_domain->cb_lock);
 883
 884         return &smmu_domain->domain;
 885 }
 886
 887 static void arm_smmu_domain_free(struct iommu_domain *domain)
 888 {
 889         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 890
 891         /*
 892          * Free the domain resources. We assume that all devices have
 893          * already been detached.
 894          */
 895         iommu_put_dma_cookie(domain);
 896         arm_smmu_destroy_domain_context(domain);
 897         kfree(smmu_domain);
 898 }
 899
 900 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
 901 {
 902         struct arm_smmu_smr *smr = smmu->smrs + idx;
 903         u32 reg = FIELD_PREP(SMR_ID, smr->id) | FIELD_PREP(SMR_MASK, smr->mask);
 904
 905         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
 906                 reg |= SMR_VALID;
 907         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
 908 }
 909
 910 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
 911 {
 912         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
 913         u32 reg = FIELD_PREP(S2CR_TYPE, s2cr->type) |
 914                   FIELD_PREP(S2CR_CBNDX, s2cr->cbndx) |
 915                   FIELD_PREP(S2CR_PRIVCFG, s2cr->privcfg);
 916
 917         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
 918             smmu->smrs[idx].valid)
 919                 reg |= S2CR_EXIDVALID;
 920         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
 921 }
 922
 923 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
 924 {
 925         arm_smmu_write_s2cr(smmu, idx);
 926         if (smmu->smrs)
 927                 arm_smmu_write_smr(smmu, idx);
 928 }
 929
 930 /*
 931  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
 932  * should be called after sCR0 is written.
 933  */
 934 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
 935 {
 936         u32 smr;
 937
 938         if (!smmu->smrs)
 939                 return;
 940
 941         /*
 942          * SMR.ID bits may not be preserved if the corresponding MASK
 943          * bits are set, so check each one separately. We can reject
 944          * masters later if they try to claim IDs outside these masks.
 945          */
 946         smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);
 947         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
 948         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
 949         smmu->streamid_mask = FIELD_GET(SMR_ID, smr);
 950
 951         smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);
 952         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
 953         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
 954         smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);
 955 }
 956
 957 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
 958 {
 959         struct arm_smmu_smr *smrs = smmu->smrs;
 960         int i, free_idx = -ENOSPC;
 961
 962         /* Stream indexing is blissfully easy */
 963         if (!smrs)
 964                 return id;
 965
 966         /* Validating SMRs is... less so */
 967         for (i = 0; i < smmu->num_mapping_groups; ++i) {
 968                 if (!smrs[i].valid) {
 969                         /*
 970                          * Note the first free entry we come across, which
 971                          * we'll claim in the end if nothing else matches.
 972                          */
 973                         if (free_idx < 0)
 974                                 free_idx = i;
 975                         continue;
 976                 }
 977                 /*
 978                  * If the new entry is _entirely_ matched by an existing entry,
 979                  * then reuse that, with the guarantee that there also cannot
 980                  * be any subsequent conflicting entries. In normal use we'd
 981                  * expect simply identical entries for this case, but there's
 982                  * no harm in accommodating the generalisation.
 983                  */
 984                 if ((mask & smrs[i].mask) == mask &&
 985                     !((id ^ smrs[i].id) & ~smrs[i].mask))
 986                         return i;
 987                 /*
 988                  * If the new entry has any other overlap with an existing one,
 989                  * though, then there always exists at least one stream ID
 990                  * which would cause a conflict, and we can't allow that risk.
 991                  */
 992                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
 993                         return -EINVAL;
 994         }
 995
 996         return free_idx;
 997 }
 998
 999 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1000 {
1001         if (--smmu->s2crs[idx].count)
1002                 return false;
1003
1004         smmu->s2crs[idx] = s2cr_init_val;
1005         if (smmu->smrs)
1006                 smmu->smrs[idx].valid = false;
1007
1008         return true;
1009 }
1010
1011 static int arm_smmu_master_alloc_smes(struct device *dev)
1012 {
1013         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1014         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1015         struct arm_smmu_device *smmu = cfg->smmu;
1016         struct arm_smmu_smr *smrs = smmu->smrs;
1017         struct iommu_group *group;
1018         int i, idx, ret;
1019
1020         mutex_lock(&smmu->stream_map_mutex);
1021         /* Figure out a viable stream map entry allocation */
1022         for_each_cfg_sme(fwspec, i, idx) {
1023                 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1024                 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1025
1026                 if (idx != INVALID_SMENDX) {
1027                         ret = -EEXIST;
1028                         goto out_err;
1029                 }
1030
1031                 ret = arm_smmu_find_sme(smmu, sid, mask);
1032                 if (ret < 0)
1033                         goto out_err;
1034
1035                 idx = ret;
1036                 if (smrs && smmu->s2crs[idx].count == 0) {
1037                         smrs[idx].id = sid;
1038                         smrs[idx].mask = mask;
1039                         smrs[idx].valid = true;
1040                 }
1041                 smmu->s2crs[idx].count++;
1042                 cfg->smendx[i] = (s16)idx;
1043         }
1044
1045         group = iommu_group_get_for_dev(dev);
1046         if (!group)
1047                 group = ERR_PTR(-ENOMEM);
1048         if (IS_ERR(group)) {
1049                 ret = PTR_ERR(group);
1050                 goto out_err;
1051         }
1052         iommu_group_put(group);
1053
1054         /* It worked! Now, poke the actual hardware */
1055         for_each_cfg_sme(fwspec, i, idx) {
1056                 arm_smmu_write_sme(smmu, idx);
1057                 smmu->s2crs[idx].group = group;
1058         }
1059
1060         mutex_unlock(&smmu->stream_map_mutex);
1061         return 0;
1062
1063 out_err:
1064         while (i--) {
1065                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1066                 cfg->smendx[i] = INVALID_SMENDX;
1067         }
1068         mutex_unlock(&smmu->stream_map_mutex);
1069         return ret;
1070 }
1071
1072 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1073 {
1074         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1075         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1076         int i, idx;
1077
1078         mutex_lock(&smmu->stream_map_mutex);
1079         for_each_cfg_sme(fwspec, i, idx) {
1080                 if (arm_smmu_free_sme(smmu, idx))
1081                         arm_smmu_write_sme(smmu, idx);
1082                 cfg->smendx[i] = INVALID_SMENDX;
1083         }
1084         mutex_unlock(&smmu->stream_map_mutex);
1085 }
1086
1087 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1088                                       struct iommu_fwspec *fwspec)
1089 {
1090         struct arm_smmu_device *smmu = smmu_domain->smmu;
1091         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1092         u8 cbndx = smmu_domain->cfg.cbndx;
1093         enum arm_smmu_s2cr_type type;
1094         int i, idx;
1095
1096         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1097                 type = S2CR_TYPE_BYPASS;
1098         else
1099                 type = S2CR_TYPE_TRANS;
1100
1101         for_each_cfg_sme(fwspec, i, idx) {
1102                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1103                         continue;
1104
1105                 s2cr[idx].type = type;
1106                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1107                 s2cr[idx].cbndx = cbndx;
1108                 arm_smmu_write_s2cr(smmu, idx);
1109         }
1110         return 0;
1111 }
1112
1113 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1114 {
1115         int ret;
1116         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1117         struct arm_smmu_device *smmu;
1118         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1119
1120         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1121                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1122                 return -ENXIO;
1123         }
1124
1125         /*
1126          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1127          * domains between of_xlate() and add_device() - we have no way to cope
1128          * with that, so until ARM gets converted to rely on groups and default
1129          * domains, just say no (but more politely than by dereferencing NULL).
1130          * This should be at least a WARN_ON once that's sorted.
1131          */
1132         if (!fwspec->iommu_priv)
1133                 return -ENODEV;
1134
1135         smmu = fwspec_smmu(fwspec);
1136
1137         ret = arm_smmu_rpm_get(smmu);
1138         if (ret < 0)
1139                 return ret;
1140
1141         /* Ensure that the domain is finalised */
1142         ret = arm_smmu_init_domain_context(domain, smmu);
1143         if (ret < 0)
1144                 goto rpm_put;
1145
1146         /*
1147          * Sanity check the domain. We don't support domains across
1148          * different SMMUs.
1149          */
1150         if (smmu_domain->smmu != smmu) {
1151                 dev_err(dev,
1152                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1153                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1154                 ret = -EINVAL;
1155                 goto rpm_put;
1156         }
1157
1158         /* Looks ok, so add the device to the domain */
1159         ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
1160
1161 rpm_put:
1162         arm_smmu_rpm_put(smmu);
1163         return ret;
1164 }
1165
1166 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1167                         phys_addr_t paddr, size_t size, int prot)
1168 {
1169         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1170         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1171         int ret;
1172
1173         if (!ops)
1174                 return -ENODEV;
1175
1176         arm_smmu_rpm_get(smmu);
1177         ret = ops->map(ops, iova, paddr, size, prot);
1178         arm_smmu_rpm_put(smmu);
1179
1180         return ret;
1181 }
1182
1183 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1184                              size_t size, struct iommu_iotlb_gather *gather)
1185 {
1186         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1187         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1188         size_t ret;
1189
1190         if (!ops)
1191                 return 0;
1192
1193         arm_smmu_rpm_get(smmu);
1194         ret = ops->unmap(ops, iova, size, gather);
1195         arm_smmu_rpm_put(smmu);
1196
1197         return ret;
1198 }
1199
1200 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1201 {
1202         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1203         struct arm_smmu_device *smmu = smmu_domain->smmu;
1204
1205         if (smmu_domain->flush_ops) {
1206                 arm_smmu_rpm_get(smmu);
1207                 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1208                 arm_smmu_rpm_put(smmu);
1209         }
1210 }
1211
1212 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1213                                 struct iommu_iotlb_gather *gather)
1214 {
1215         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1216         struct arm_smmu_device *smmu = smmu_domain->smmu;
1217
1218         if (!smmu)
1219                 return;
1220
1221         arm_smmu_rpm_get(smmu);
1222         if (smmu->version == ARM_SMMU_V2 ||
1223             smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1224                 arm_smmu_tlb_sync_context(smmu_domain);
1225         else
1226                 arm_smmu_tlb_sync_global(smmu);
1227         arm_smmu_rpm_put(smmu);
1228 }
1229
1230 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1231                                               dma_addr_t iova)
1232 {
1233         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1234         struct arm_smmu_device *smmu = smmu_domain->smmu;
1235         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1236         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1237         struct device *dev = smmu->dev;
1238         void __iomem *reg;
1239         u32 tmp;
1240         u64 phys;
1241         unsigned long va, flags;
1242         int ret, idx = cfg->cbndx;
1243
1244         ret = arm_smmu_rpm_get(smmu);
1245         if (ret < 0)
1246                 return 0;
1247
1248         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1249         va = iova & ~0xfffUL;
1250         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1251                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1252         else
1253                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1254
1255         reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1256         if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) {
1257                 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1258                 dev_err(dev,
1259                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1260                         &iova);
1261                 return ops->iova_to_phys(ops, iova);
1262         }
1263
1264         phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1265         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1266         if (phys & CB_PAR_F) {
1267                 dev_err(dev, "translation fault!\n");
1268                 dev_err(dev, "PAR = 0x%llx\n", phys);
1269                 return 0;
1270         }
1271
1272         arm_smmu_rpm_put(smmu);
1273
1274         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1275 }
1276
1277 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1278                                         dma_addr_t iova)
1279 {
1280         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1281         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1282
1283         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1284                 return iova;
1285
1286         if (!ops)
1287                 return 0;
1288
1289         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1290                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1291                 return arm_smmu_iova_to_phys_hard(domain, iova);
1292
1293         return ops->iova_to_phys(ops, iova);
1294 }
1295
1296 static bool arm_smmu_capable(enum iommu_cap cap)
1297 {
1298         switch (cap) {
1299         case IOMMU_CAP_CACHE_COHERENCY:
1300                 /*
1301                  * Return true here as the SMMU can always send out coherent
1302                  * requests.
1303                  */
1304                 return true;
1305         case IOMMU_CAP_NOEXEC:
1306                 return true;
1307         default:
1308                 return false;
1309         }
1310 }
1311
1312 static
1313 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1314 {
1315         struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1316                                                           fwnode);
1317         put_device(dev);
1318         return dev ? dev_get_drvdata(dev) : NULL;
1319 }
1320
1321 static int arm_smmu_add_device(struct device *dev)
1322 {
1323         struct arm_smmu_device *smmu;
1324         struct arm_smmu_master_cfg *cfg;
1325         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1326         int i, ret;
1327
1328         if (using_legacy_binding) {
1329                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1330
1331                 /*
1332                  * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1333                  * will allocate/initialise a new one. Thus we need to update fwspec for
1334                  * later use.
1335                  */
1336                 fwspec = dev_iommu_fwspec_get(dev);
1337                 if (ret)
1338                         goto out_free;
1339         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1340                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1341         } else {
1342                 return -ENODEV;
1343         }
1344
1345         ret = -EINVAL;
1346         for (i = 0; i < fwspec->num_ids; i++) {
1347                 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1348                 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1349
1350                 if (sid & ~smmu->streamid_mask) {
1351                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1352                                 sid, smmu->streamid_mask);
1353                         goto out_free;
1354                 }
1355                 if (mask & ~smmu->smr_mask_mask) {
1356                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1357                                 mask, smmu->smr_mask_mask);
1358                         goto out_free;
1359                 }
1360         }
1361
1362         ret = -ENOMEM;
1363         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1364                       GFP_KERNEL);
1365         if (!cfg)
1366                 goto out_free;
1367
1368         cfg->smmu = smmu;
1369         fwspec->iommu_priv = cfg;
1370         while (i--)
1371                 cfg->smendx[i] = INVALID_SMENDX;
1372
1373         ret = arm_smmu_rpm_get(smmu);
1374         if (ret < 0)
1375                 goto out_cfg_free;
1376
1377         ret = arm_smmu_master_alloc_smes(dev);
1378         arm_smmu_rpm_put(smmu);
1379
1380         if (ret)
1381                 goto out_cfg_free;
1382
1383         iommu_device_link(&smmu->iommu, dev);
1384
1385         device_link_add(dev, smmu->dev,
1386                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1387
1388         return 0;
1389
1390 out_cfg_free:
1391         kfree(cfg);
1392 out_free:
1393         iommu_fwspec_free(dev);
1394         return ret;
1395 }
1396
1397 static void arm_smmu_remove_device(struct device *dev)
1398 {
1399         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1400         struct arm_smmu_master_cfg *cfg;
1401         struct arm_smmu_device *smmu;
1402         int ret;
1403
1404         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1405                 return;
1406
1407         cfg  = fwspec->iommu_priv;
1408         smmu = cfg->smmu;
1409
1410         ret = arm_smmu_rpm_get(smmu);
1411         if (ret < 0)
1412                 return;
1413
1414         iommu_device_unlink(&smmu->iommu, dev);
1415         arm_smmu_master_free_smes(fwspec);
1416
1417         arm_smmu_rpm_put(smmu);
1418
1419         iommu_group_remove_device(dev);
1420         kfree(fwspec->iommu_priv);
1421         iommu_fwspec_free(dev);
1422 }
1423
1424 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1425 {
1426         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1427         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1428         struct iommu_group *group = NULL;
1429         int i, idx;
1430
1431         for_each_cfg_sme(fwspec, i, idx) {
1432                 if (group && smmu->s2crs[idx].group &&
1433                     group != smmu->s2crs[idx].group)
1434                         return ERR_PTR(-EINVAL);
1435
1436                 group = smmu->s2crs[idx].group;
1437         }
1438
1439         if (group)
1440                 return iommu_group_ref_get(group);
1441
1442         if (dev_is_pci(dev))
1443                 group = pci_device_group(dev);
1444         else if (dev_is_fsl_mc(dev))
1445                 group = fsl_mc_device_group(dev);
1446         else
1447                 group = generic_device_group(dev);
1448
1449         return group;
1450 }
1451
1452 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1453                                     enum iommu_attr attr, void *data)
1454 {
1455         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1456
1457         switch(domain->type) {
1458         case IOMMU_DOMAIN_UNMANAGED:
1459                 switch (attr) {
1460                 case DOMAIN_ATTR_NESTING:
1461                         *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1462                         return 0;
1463                 default:
1464                         return -ENODEV;
1465                 }
1466                 break;
1467         case IOMMU_DOMAIN_DMA:
1468                 switch (attr) {
1469                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1470                         *(int *)data = smmu_domain->non_strict;
1471                         return 0;
1472                 default:
1473                         return -ENODEV;
1474                 }
1475                 break;
1476         default:
1477                 return -EINVAL;
1478         }
1479 }
1480
1481 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1482                                     enum iommu_attr attr, void *data)
1483 {
1484         int ret = 0;
1485         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1486
1487         mutex_lock(&smmu_domain->init_mutex);
1488
1489         switch(domain->type) {
1490         case IOMMU_DOMAIN_UNMANAGED:
1491                 switch (attr) {
1492                 case DOMAIN_ATTR_NESTING:
1493                         if (smmu_domain->smmu) {
1494                                 ret = -EPERM;
1495                                 goto out_unlock;
1496                         }
1497
1498                         if (*(int *)data)
1499                                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1500                         else
1501                                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1502                         break;
1503                 default:
1504                         ret = -ENODEV;
1505                 }
1506                 break;
1507         case IOMMU_DOMAIN_DMA:
1508                 switch (attr) {
1509                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1510                         smmu_domain->non_strict = *(int *)data;
1511                         break;
1512                 default:
1513                         ret = -ENODEV;
1514                 }
1515                 break;
1516         default:
1517                 ret = -EINVAL;
1518         }
1519 out_unlock:
1520         mutex_unlock(&smmu_domain->init_mutex);
1521         return ret;
1522 }
1523
1524 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1525 {
1526         u32 mask, fwid = 0;
1527
1528         if (args->args_count > 0)
1529                 fwid |= FIELD_PREP(SMR_ID, args->args[0]);
1530
1531         if (args->args_count > 1)
1532                 fwid |= FIELD_PREP(SMR_MASK, args->args[1]);
1533         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1534                 fwid |= FIELD_PREP(SMR_MASK, mask);
1535
1536         return iommu_fwspec_add_ids(dev, &fwid, 1);
1537 }
1538
1539 static void arm_smmu_get_resv_regions(struct device *dev,
1540                                       struct list_head *head)
1541 {
1542         struct iommu_resv_region *region;
1543         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1544
1545         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1546                                          prot, IOMMU_RESV_SW_MSI);
1547         if (!region)
1548                 return;
1549
1550         list_add_tail(&region->list, head);
1551
1552         iommu_dma_get_resv_regions(dev, head);
1553 }
1554
1555 static void arm_smmu_put_resv_regions(struct device *dev,
1556                                       struct list_head *head)
1557 {
1558         struct iommu_resv_region *entry, *next;
1559
1560         list_for_each_entry_safe(entry, next, head, list)
1561                 kfree(entry);
1562 }
1563
1564 static struct iommu_ops arm_smmu_ops = {
1565         .capable                = arm_smmu_capable,
1566         .domain_alloc           = arm_smmu_domain_alloc,
1567         .domain_free            = arm_smmu_domain_free,
1568         .attach_dev             = arm_smmu_attach_dev,
1569         .map                    = arm_smmu_map,
1570         .unmap                  = arm_smmu_unmap,
1571         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
1572         .iotlb_sync             = arm_smmu_iotlb_sync,
1573         .iova_to_phys           = arm_smmu_iova_to_phys,
1574         .add_device             = arm_smmu_add_device,
1575         .remove_device          = arm_smmu_remove_device,
1576         .device_group           = arm_smmu_device_group,
1577         .domain_get_attr        = arm_smmu_domain_get_attr,
1578         .domain_set_attr        = arm_smmu_domain_set_attr,
1579         .of_xlate               = arm_smmu_of_xlate,
1580         .get_resv_regions       = arm_smmu_get_resv_regions,
1581         .put_resv_regions       = arm_smmu_put_resv_regions,
1582         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1583 };
1584
1585 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1586 {
1587         int i;
1588         u32 reg;
1589
1590         /* clear global FSR */
1591         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1592         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1593
1594         /*
1595          * Reset stream mapping groups: Initial values mark all SMRn as
1596          * invalid and all S2CRn as bypass unless overridden.
1597          */
1598         for (i = 0; i < smmu->num_mapping_groups; ++i)
1599                 arm_smmu_write_sme(smmu, i);
1600
1601         /* Make sure all context banks are disabled and clear CB_FSR  */
1602         for (i = 0; i < smmu->num_context_banks; ++i) {
1603                 arm_smmu_write_context_bank(smmu, i);
1604                 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, FSR_FAULT);
1605         }
1606
1607         /* Invalidate the TLB, just in case */
1608         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1609         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1610
1611         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1612
1613         /* Enable fault reporting */
1614         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1615
1616         /* Disable TLB broadcasting. */
1617         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1618
1619         /* Enable client access, handling unmatched streams as appropriate */
1620         reg &= ~sCR0_CLIENTPD;
1621         if (disable_bypass)
1622                 reg |= sCR0_USFCFG;
1623         else
1624                 reg &= ~sCR0_USFCFG;
1625
1626         /* Disable forced broadcasting */
1627         reg &= ~sCR0_FB;
1628
1629         /* Don't upgrade barriers */
1630         reg &= ~(sCR0_BSU);
1631
1632         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1633                 reg |= sCR0_VMID16EN;
1634
1635         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1636                 reg |= sCR0_EXIDENABLE;
1637
1638         if (smmu->impl && smmu->impl->reset)
1639                 smmu->impl->reset(smmu);
1640
1641         /* Push the button */
1642         arm_smmu_tlb_sync_global(smmu);
1643         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1644 }
1645
1646 static int arm_smmu_id_size_to_bits(int size)
1647 {
1648         switch (size) {
1649         case 0:
1650                 return 32;
1651         case 1:
1652                 return 36;
1653         case 2:
1654                 return 40;
1655         case 3:
1656                 return 42;
1657         case 4:
1658                 return 44;
1659         case 5:
1660         default:
1661                 return 48;
1662         }
1663 }
1664
1665 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1666 {
1667         unsigned int size;
1668         u32 id;
1669         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1670         int i;
1671
1672         dev_notice(smmu->dev, "probing hardware configuration...\n");
1673         dev_notice(smmu->dev, "SMMUv%d with:\n",
1674                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1675
1676         /* ID0 */
1677         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1678
1679         /* Restrict available stages based on module parameter */
1680         if (force_stage == 1)
1681                 id &= ~(ID0_S2TS | ID0_NTS);
1682         else if (force_stage == 2)
1683                 id &= ~(ID0_S1TS | ID0_NTS);
1684
1685         if (id & ID0_S1TS) {
1686                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1687                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1688         }
1689
1690         if (id & ID0_S2TS) {
1691                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1692                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1693         }
1694
1695         if (id & ID0_NTS) {
1696                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1697                 dev_notice(smmu->dev, "\tnested translation\n");
1698         }
1699
1700         if (!(smmu->features &
1701                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1702                 dev_err(smmu->dev, "\tno translation support!\n");
1703                 return -ENODEV;
1704         }
1705
1706         if ((id & ID0_S1TS) &&
1707                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1708                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1709                 dev_notice(smmu->dev, "\taddress translation ops\n");
1710         }
1711
1712         /*
1713          * In order for DMA API calls to work properly, we must defer to what
1714          * the FW says about coherency, regardless of what the hardware claims.
1715          * Fortunately, this also opens up a workaround for systems where the
1716          * ID register value has ended up configured incorrectly.
1717          */
1718         cttw_reg = !!(id & ID0_CTTW);
1719         if (cttw_fw || cttw_reg)
1720                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1721                            cttw_fw ? "" : "non-");
1722         if (cttw_fw != cttw_reg)
1723                 dev_notice(smmu->dev,
1724                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1725
1726         /* Max. number of entries we have for stream matching/indexing */
1727         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1728                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1729                 size = 1 << 16;
1730         } else {
1731                 size = 1 << FIELD_GET(ID0_NUMSIDB, id);
1732         }
1733         smmu->streamid_mask = size - 1;
1734         if (id & ID0_SMS) {
1735                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1736                 size = FIELD_GET(ID0_NUMSMRG, id);
1737                 if (size == 0) {
1738                         dev_err(smmu->dev,
1739                                 "stream-matching supported, but no SMRs present!\n");
1740                         return -ENODEV;
1741                 }
1742
1743                 /* Zero-initialised to mark as invalid */
1744                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1745                                           GFP_KERNEL);
1746                 if (!smmu->smrs)
1747                         return -ENOMEM;
1748
1749                 dev_notice(smmu->dev,
1750                            "\tstream matching with %u register groups", size);
1751         }
1752         /* s2cr->type == 0 means translation, so initialise explicitly */
1753         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1754                                          GFP_KERNEL);
1755         if (!smmu->s2crs)
1756                 return -ENOMEM;
1757         for (i = 0; i < size; i++)
1758                 smmu->s2crs[i] = s2cr_init_val;
1759
1760         smmu->num_mapping_groups = size;
1761         mutex_init(&smmu->stream_map_mutex);
1762         spin_lock_init(&smmu->global_sync_lock);
1763
1764         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1765                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1766                 if (!(id & ID0_PTFS_NO_AARCH32S))
1767                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1768         }
1769
1770         /* ID1 */
1771         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1772         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1773
1774         /* Check for size mismatch of SMMU address space from mapped region */
1775         size = 1 << (FIELD_GET(ID1_NUMPAGENDXB, id) + 1);
1776         if (smmu->numpage != 2 * size << smmu->pgshift)
1777                 dev_warn(smmu->dev,
1778                         "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1779                         2 * size << smmu->pgshift, smmu->numpage);
1780         /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1781         smmu->numpage = size;
1782
1783         smmu->num_s2_context_banks = FIELD_GET(ID1_NUMS2CB, id);
1784         smmu->num_context_banks = FIELD_GET(ID1_NUMCB, id);
1785         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1786                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1787                 return -ENODEV;
1788         }
1789         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1790                    smmu->num_context_banks, smmu->num_s2_context_banks);
1791         smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1792                                  sizeof(*smmu->cbs), GFP_KERNEL);
1793         if (!smmu->cbs)
1794                 return -ENOMEM;
1795
1796         /* ID2 */
1797         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1798         size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));
1799         smmu->ipa_size = size;
1800
1801         /* The output mask is also applied for bypass */
1802         size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_OAS, id));
1803         smmu->pa_size = size;
1804
1805         if (id & ID2_VMID16)
1806                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1807
1808         /*
1809          * What the page table walker can address actually depends on which
1810          * descriptor format is in use, but since a) we don't know that yet,
1811          * and b) it can vary per context bank, this will have to do...
1812          */
1813         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1814                 dev_warn(smmu->dev,
1815                          "failed to set DMA mask for table walker\n");
1816
1817         if (smmu->version < ARM_SMMU_V2) {
1818                 smmu->va_size = smmu->ipa_size;
1819                 if (smmu->version == ARM_SMMU_V1_64K)
1820                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1821         } else {
1822                 size = FIELD_GET(ID2_UBS, id);
1823                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1824                 if (id & ID2_PTFS_4K)
1825                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1826                 if (id & ID2_PTFS_16K)
1827                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1828                 if (id & ID2_PTFS_64K)
1829                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1830         }
1831
1832         /* Now we've corralled the various formats, what'll it do? */
1833         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1834                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1835         if (smmu->features &
1836             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1837                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1838         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1839                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1840         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1841                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1842
1843         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1844                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1845         else
1846                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1847         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1848                    smmu->pgsize_bitmap);
1849
1850
1851         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1852                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1853                            smmu->va_size, smmu->ipa_size);
1854
1855         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1856                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1857                            smmu->ipa_size, smmu->pa_size);
1858
1859         if (smmu->impl && smmu->impl->cfg_probe)
1860                 return smmu->impl->cfg_probe(smmu);
1861
1862         return 0;
1863 }
1864
1865 struct arm_smmu_match_data {
1866         enum arm_smmu_arch_version version;
1867         enum arm_smmu_implementation model;
1868 };
1869
1870 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1871 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1872
1873 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1874 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1875 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1876 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1877 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1878 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1879
1880 static const struct of_device_id arm_smmu_of_match[] = {
1881         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1882         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1883         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1884         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1885         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1886         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1887         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1888         { },
1889 };
1890
1891 #ifdef CONFIG_ACPI
1892 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1893 {
1894         int ret = 0;
1895
1896         switch (model) {
1897         case ACPI_IORT_SMMU_V1:
1898         case ACPI_IORT_SMMU_CORELINK_MMU400:
1899                 smmu->version = ARM_SMMU_V1;
1900                 smmu->model = GENERIC_SMMU;
1901                 break;
1902         case ACPI_IORT_SMMU_CORELINK_MMU401:
1903                 smmu->version = ARM_SMMU_V1_64K;
1904                 smmu->model = GENERIC_SMMU;
1905                 break;
1906         case ACPI_IORT_SMMU_V2:
1907                 smmu->version = ARM_SMMU_V2;
1908                 smmu->model = GENERIC_SMMU;
1909                 break;
1910         case ACPI_IORT_SMMU_CORELINK_MMU500:
1911                 smmu->version = ARM_SMMU_V2;
1912                 smmu->model = ARM_MMU500;
1913                 break;
1914         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1915                 smmu->version = ARM_SMMU_V2;
1916                 smmu->model = CAVIUM_SMMUV2;
1917                 break;
1918         default:
1919                 ret = -ENODEV;
1920         }
1921
1922         return ret;
1923 }
1924
1925 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1926                                       struct arm_smmu_device *smmu)
1927 {
1928         struct device *dev = smmu->dev;
1929         struct acpi_iort_node *node =
1930                 *(struct acpi_iort_node **)dev_get_platdata(dev);
1931         struct acpi_iort_smmu *iort_smmu;
1932         int ret;
1933
1934         /* Retrieve SMMU1/2 specific data */
1935         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1936
1937         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1938         if (ret < 0)
1939                 return ret;
1940
1941         /* Ignore the configuration access interrupt */
1942         smmu->num_global_irqs = 1;
1943
1944         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1945                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1946
1947         return 0;
1948 }
1949 #else
1950 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1951                                              struct arm_smmu_device *smmu)
1952 {
1953         return -ENODEV;
1954 }
1955 #endif
1956
1957 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
1958                                     struct arm_smmu_device *smmu)
1959 {
1960         const struct arm_smmu_match_data *data;
1961         struct device *dev = &pdev->dev;
1962         bool legacy_binding;
1963
1964         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1965                                  &smmu->num_global_irqs)) {
1966                 dev_err(dev, "missing #global-interrupts property\n");
1967                 return -ENODEV;
1968         }
1969
1970         data = of_device_get_match_data(dev);
1971         smmu->version = data->version;
1972         smmu->model = data->model;
1973
1974         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1975         if (legacy_binding && !using_generic_binding) {
1976                 if (!using_legacy_binding)
1977                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
1978                 using_legacy_binding = true;
1979         } else if (!legacy_binding && !using_legacy_binding) {
1980                 using_generic_binding = true;
1981         } else {
1982                 dev_err(dev, "not probing due to mismatched DT properties\n");
1983                 return -ENODEV;
1984         }
1985
1986         if (of_dma_is_coherent(dev->of_node))
1987                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1988
1989         return 0;
1990 }
1991
1992 static void arm_smmu_bus_init(void)
1993 {
1994         /* Oh, for a proper bus abstraction */
1995         if (!iommu_present(&platform_bus_type))
1996                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
1997 #ifdef CONFIG_ARM_AMBA
1998         if (!iommu_present(&amba_bustype))
1999                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2000 #endif
2001 #ifdef CONFIG_PCI
2002         if (!iommu_present(&pci_bus_type)) {
2003                 pci_request_acs();
2004                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2005         }
2006 #endif
2007 #ifdef CONFIG_FSL_MC_BUS
2008         if (!iommu_present(&fsl_mc_bus_type))
2009                 bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
2010 #endif
2011 }
2012
2013 static int arm_smmu_device_probe(struct platform_device *pdev)
2014 {
2015         struct resource *res;
2016         resource_size_t ioaddr;
2017         struct arm_smmu_device *smmu;
2018         struct device *dev = &pdev->dev;
2019         int num_irqs, i, err;
2020
2021         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2022         if (!smmu) {
2023                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2024                 return -ENOMEM;
2025         }
2026         smmu->dev = dev;
2027
2028         if (dev->of_node)
2029                 err = arm_smmu_device_dt_probe(pdev, smmu);
2030         else
2031                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2032
2033         if (err)
2034                 return err;
2035
2036         smmu = arm_smmu_impl_init(smmu);
2037         if (IS_ERR(smmu))
2038                 return PTR_ERR(smmu);
2039
2040         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2041         ioaddr = res->start;
2042         smmu->base = devm_ioremap_resource(dev, res);
2043         if (IS_ERR(smmu->base))
2044                 return PTR_ERR(smmu->base);
2045         /*
2046          * The resource size should effectively match the value of SMMU_TOP;
2047          * stash that temporarily until we know PAGESIZE to validate it with.
2048          */
2049         smmu->numpage = resource_size(res);
2050
2051         num_irqs = 0;
2052         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2053                 num_irqs++;
2054                 if (num_irqs > smmu->num_global_irqs)
2055                         smmu->num_context_irqs++;
2056         }
2057
2058         if (!smmu->num_context_irqs) {
2059                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2060                         num_irqs, smmu->num_global_irqs + 1);
2061                 return -ENODEV;
2062         }
2063
2064         smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2065                                   GFP_KERNEL);
2066         if (!smmu->irqs) {
2067                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2068                 return -ENOMEM;
2069         }
2070
2071         for (i = 0; i < num_irqs; ++i) {
2072                 int irq = platform_get_irq(pdev, i);
2073
2074                 if (irq < 0) {
2075                         dev_err(dev, "failed to get irq index %d\n", i);
2076                         return -ENODEV;
2077                 }
2078                 smmu->irqs[i] = irq;
2079         }
2080
2081         err = devm_clk_bulk_get_all(dev, &smmu->clks);
2082         if (err < 0) {
2083                 dev_err(dev, "failed to get clocks %d\n", err);
2084                 return err;
2085         }
2086         smmu->num_clks = err;
2087
2088         err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2089         if (err)
2090                 return err;
2091
2092         err = arm_smmu_device_cfg_probe(smmu);
2093         if (err)
2094                 return err;
2095
2096         if (smmu->version == ARM_SMMU_V2) {
2097                 if (smmu->num_context_banks > smmu->num_context_irqs) {
2098                         dev_err(dev,
2099                               "found only %d context irq(s) but %d required\n",
2100                               smmu->num_context_irqs, smmu->num_context_banks);
2101                         return -ENODEV;
2102                 }
2103
2104                 /* Ignore superfluous interrupts */
2105                 smmu->num_context_irqs = smmu->num_context_banks;
2106         }
2107
2108         for (i = 0; i < smmu->num_global_irqs; ++i) {
2109                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2110                                        arm_smmu_global_fault,
2111                                        IRQF_SHARED,
2112                                        "arm-smmu global fault",
2113                                        smmu);
2114                 if (err) {
2115                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2116                                 i, smmu->irqs[i]);
2117                         return err;
2118                 }
2119         }
2120
2121         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2122                                      "smmu.%pa", &ioaddr);
2123         if (err) {
2124                 dev_err(dev, "Failed to register iommu in sysfs\n");
2125                 return err;
2126         }
2127
2128         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2129         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2130
2131         err = iommu_device_register(&smmu->iommu);
2132         if (err) {
2133                 dev_err(dev, "Failed to register iommu\n");
2134                 return err;
2135         }
2136
2137         platform_set_drvdata(pdev, smmu);
2138         arm_smmu_device_reset(smmu);
2139         arm_smmu_test_smr_masks(smmu);
2140
2141         /*
2142          * We want to avoid touching dev->power.lock in fastpaths unless
2143          * it's really going to do something useful - pm_runtime_enabled()
2144          * can serve as an ideal proxy for that decision. So, conditionally
2145          * enable pm_runtime.
2146          */
2147         if (dev->pm_domain) {
2148                 pm_runtime_set_active(dev);
2149                 pm_runtime_enable(dev);
2150         }
2151
2152         /*
2153          * For ACPI and generic DT bindings, an SMMU will be probed before
2154          * any device which might need it, so we want the bus ops in place
2155          * ready to handle default domain setup as soon as any SMMU exists.
2156          */
2157         if (!using_legacy_binding)
2158                 arm_smmu_bus_init();
2159
2160         return 0;
2161 }
2162
2163 /*
2164  * With the legacy DT binding in play, though, we have no guarantees about
2165  * probe order, but then we're also not doing default domains, so we can
2166  * delay setting bus ops until we're sure every possible SMMU is ready,
2167  * and that way ensure that no add_device() calls get missed.
2168  */
2169 static int arm_smmu_legacy_bus_init(void)
2170 {
2171         if (using_legacy_binding)
2172                 arm_smmu_bus_init();
2173         return 0;
2174 }
2175 device_initcall_sync(arm_smmu_legacy_bus_init);
2176
2177 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2178 {
2179         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2180
2181         if (!smmu)
2182                 return;
2183
2184         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2185                 dev_err(&pdev->dev, "removing device with active domains!\n");
2186
2187         arm_smmu_rpm_get(smmu);
2188         /* Turn the thing off */
2189         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, sCR0_CLIENTPD);
2190         arm_smmu_rpm_put(smmu);
2191
2192         if (pm_runtime_enabled(smmu->dev))
2193                 pm_runtime_force_suspend(smmu->dev);
2194         else
2195                 clk_bulk_disable(smmu->num_clks, smmu->clks);
2196
2197         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2198 }
2199
2200 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2201 {
2202         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2203         int ret;
2204
2205         ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2206         if (ret)
2207                 return ret;
2208
2209         arm_smmu_device_reset(smmu);
2210
2211         return 0;
2212 }
2213
2214 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2215 {
2216         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2217
2218         clk_bulk_disable(smmu->num_clks, smmu->clks);
2219
2220         return 0;
2221 }
2222
2223 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2224 {
2225         if (pm_runtime_suspended(dev))
2226                 return 0;
2227
2228         return arm_smmu_runtime_resume(dev);
2229 }
2230
2231 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2232 {
2233         if (pm_runtime_suspended(dev))
2234                 return 0;
2235
2236         return arm_smmu_runtime_suspend(dev);
2237 }
2238
2239 static const struct dev_pm_ops arm_smmu_pm_ops = {
2240         SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2241         SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2242                            arm_smmu_runtime_resume, NULL)
2243 };
2244
2245 static struct platform_driver arm_smmu_driver = {
2246         .driver = {
2247                 .name                   = "arm-smmu",
2248                 .of_match_table         = of_match_ptr(arm_smmu_of_match),
2249                 .pm                     = &arm_smmu_pm_ops,
2250                 .suppress_bind_attrs    = true,
2251         },
2252         .probe  = arm_smmu_device_probe,
2253         .shutdown = arm_smmu_device_shutdown,
2254 };
2255 builtin_platform_driver(arm_smmu_driver);