1 // SPDX-License-Identifier: GPL-2.0-only
3 * IOMMU API for ARM architected SMMU implementations.
5 * Copyright (C) 2013 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
18 #define pr_fmt(fmt) "arm-smmu: " fmt
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
38 #include <linux/fsl/mc.h>
41 #include "../../dma-iommu.h"
44 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
45 * global register space are still, in fact, using a hypervisor to mediate it
46 * by trapping and emulating register accesses. Sadly, some deployed versions
47 * of said trapping code have bugs wherein they go horribly wrong for stores
48 * using r31 (i.e. XZR/WZR) as the source register.
50 #define QCOM_DUMMY_VAL -1
52 #define MSI_IOVA_BASE 0x8000000
53 #define MSI_IOVA_LENGTH 0x100000
55 static int force_stage;
56 module_param(force_stage, int, S_IRUGO);
57 MODULE_PARM_DESC(force_stage,
58 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
59 static bool disable_bypass =
60 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
61 module_param(disable_bypass, bool, S_IRUGO);
62 MODULE_PARM_DESC(disable_bypass,
63 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
65 #define s2cr_init_val (struct arm_smmu_s2cr){ \
66 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
69 static bool using_legacy_binding, using_generic_binding;
71 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
73 if (pm_runtime_enabled(smmu->dev))
74 return pm_runtime_resume_and_get(smmu->dev);
79 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
81 if (pm_runtime_enabled(smmu->dev))
82 pm_runtime_put_autosuspend(smmu->dev);
85 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
87 return container_of(dom, struct arm_smmu_domain, domain);
90 static struct platform_driver arm_smmu_driver;
91 static struct iommu_ops arm_smmu_ops;
93 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
94 static struct device_node *dev_get_dev_node(struct device *dev)
96 if (dev_is_pci(dev)) {
97 struct pci_bus *bus = to_pci_dev(dev)->bus;
99 while (!pci_is_root_bus(bus))
101 return of_node_get(bus->bridge->parent->of_node);
104 return of_node_get(dev->of_node);
107 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
109 *((__be32 *)data) = cpu_to_be32(alias);
110 return 0; /* Continue walking */
113 static int __find_legacy_master_phandle(struct device *dev, void *data)
115 struct of_phandle_iterator *it = *(void **)data;
116 struct device_node *np = it->node;
119 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
120 "#stream-id-cells", -1)
121 if (it->node == np) {
122 *(void **)data = dev;
126 return err == -ENOENT ? 0 : err;
129 static int arm_smmu_register_legacy_master(struct device *dev,
130 struct arm_smmu_device **smmu)
132 struct device *smmu_dev;
133 struct device_node *np;
134 struct of_phandle_iterator it;
140 np = dev_get_dev_node(dev);
141 if (!np || !of_property_present(np, "#stream-id-cells")) {
147 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
148 __find_legacy_master_phandle);
156 if (dev_is_pci(dev)) {
157 /* "mmu-masters" assumes Stream ID == Requester ID */
158 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
164 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
169 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
173 *smmu = dev_get_drvdata(smmu_dev);
174 of_phandle_iterator_args(&it, sids, it.cur_count);
175 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
180 static int arm_smmu_register_legacy_master(struct device *dev,
181 struct arm_smmu_device **smmu)
185 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
187 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
192 /* Wait for any pending TLB invalidations to complete */
193 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
194 int sync, int status)
196 unsigned int spin_cnt, delay;
199 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
200 return smmu->impl->tlb_sync(smmu, page, sync, status);
202 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
203 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
204 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
205 reg = arm_smmu_readl(smmu, page, status);
206 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
212 dev_err_ratelimited(smmu->dev,
213 "TLB sync timed out -- SMMU may be deadlocked\n");
216 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
220 spin_lock_irqsave(&smmu->global_sync_lock, flags);
221 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
222 ARM_SMMU_GR0_sTLBGSTATUS);
223 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
226 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
228 struct arm_smmu_device *smmu = smmu_domain->smmu;
231 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
232 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
233 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
234 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
237 static void arm_smmu_tlb_inv_context_s1(void *cookie)
239 struct arm_smmu_domain *smmu_domain = cookie;
241 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
242 * current CPU are visible beforehand.
245 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
246 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
247 arm_smmu_tlb_sync_context(smmu_domain);
250 static void arm_smmu_tlb_inv_context_s2(void *cookie)
252 struct arm_smmu_domain *smmu_domain = cookie;
253 struct arm_smmu_device *smmu = smmu_domain->smmu;
257 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
258 arm_smmu_tlb_sync_global(smmu);
261 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
262 size_t granule, void *cookie, int reg)
264 struct arm_smmu_domain *smmu_domain = cookie;
265 struct arm_smmu_device *smmu = smmu_domain->smmu;
266 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
267 int idx = cfg->cbndx;
269 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
272 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
273 iova = (iova >> 12) << 12;
276 arm_smmu_cb_write(smmu, idx, reg, iova);
278 } while (size -= granule);
281 iova |= (u64)cfg->asid << 48;
283 arm_smmu_cb_writeq(smmu, idx, reg, iova);
284 iova += granule >> 12;
285 } while (size -= granule);
289 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
290 size_t granule, void *cookie, int reg)
292 struct arm_smmu_domain *smmu_domain = cookie;
293 struct arm_smmu_device *smmu = smmu_domain->smmu;
294 int idx = smmu_domain->cfg.cbndx;
296 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
301 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
302 arm_smmu_cb_writeq(smmu, idx, reg, iova);
304 arm_smmu_cb_write(smmu, idx, reg, iova);
305 iova += granule >> 12;
306 } while (size -= granule);
309 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
310 size_t granule, void *cookie)
312 struct arm_smmu_domain *smmu_domain = cookie;
313 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
315 if (cfg->flush_walk_prefer_tlbiasid) {
316 arm_smmu_tlb_inv_context_s1(cookie);
318 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
319 ARM_SMMU_CB_S1_TLBIVA);
320 arm_smmu_tlb_sync_context(cookie);
324 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
325 unsigned long iova, size_t granule,
328 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
329 ARM_SMMU_CB_S1_TLBIVAL);
332 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
333 size_t granule, void *cookie)
335 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
336 ARM_SMMU_CB_S2_TLBIIPAS2);
337 arm_smmu_tlb_sync_context(cookie);
340 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
341 unsigned long iova, size_t granule,
344 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
345 ARM_SMMU_CB_S2_TLBIIPAS2L);
348 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
349 size_t granule, void *cookie)
351 arm_smmu_tlb_inv_context_s2(cookie);
354 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
355 * almost negligible, but the benefit of getting the first one in as far ahead
356 * of the sync as possible is significant, hence we don't just make this a
357 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
360 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
361 unsigned long iova, size_t granule,
364 struct arm_smmu_domain *smmu_domain = cookie;
365 struct arm_smmu_device *smmu = smmu_domain->smmu;
367 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
370 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
373 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
374 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
375 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
376 .tlb_add_page = arm_smmu_tlb_add_page_s1,
379 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
380 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
381 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
382 .tlb_add_page = arm_smmu_tlb_add_page_s2,
385 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
386 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
387 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
388 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
391 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
393 u32 fsr, fsynr, cbfrsynra;
395 struct iommu_domain *domain = dev;
396 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
397 struct arm_smmu_device *smmu = smmu_domain->smmu;
398 int idx = smmu_domain->cfg.cbndx;
401 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
402 if (!(fsr & ARM_SMMU_FSR_FAULT))
405 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
406 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
407 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
409 ret = report_iommu_fault(domain, NULL, iova,
410 fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
413 dev_err_ratelimited(smmu->dev,
414 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
415 fsr, iova, fsynr, cbfrsynra, idx);
417 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
421 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
423 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
424 struct arm_smmu_device *smmu = dev;
425 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
426 DEFAULT_RATELIMIT_BURST);
428 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
429 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
430 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
431 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
436 if (__ratelimit(&rs)) {
437 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
438 (gfsr & ARM_SMMU_sGFSR_USF))
440 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
444 "Unexpected global fault, this could be serious\n");
446 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
447 gfsr, gfsynr0, gfsynr1, gfsynr2);
450 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
454 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
455 struct io_pgtable_cfg *pgtbl_cfg)
457 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
458 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
459 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
465 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
466 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
468 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
469 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
470 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
471 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
473 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
476 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
481 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
482 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
485 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
487 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
490 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
491 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
493 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
496 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
499 /* MAIRs (stage-1 only) */
501 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
502 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
503 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
505 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
506 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
511 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
515 struct arm_smmu_cb *cb = &smmu->cbs[idx];
516 struct arm_smmu_cfg *cfg = cb->cfg;
518 /* Unassigned context banks only need disabling */
520 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
524 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
527 if (smmu->version > ARM_SMMU_V1) {
528 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
529 reg = ARM_SMMU_CBA2R_VA64;
532 /* 16-bit VMIDs live in CBA2R */
533 if (smmu->features & ARM_SMMU_FEAT_VMID16)
534 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
536 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
540 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
541 if (smmu->version < ARM_SMMU_V2)
542 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
545 * Use the weakest shareability/memory types, so they are
546 * overridden by the ttbcr/pte.
549 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
550 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
551 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
552 ARM_SMMU_CBAR_S1_MEMATTR_WB);
553 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
554 /* 8-bit VMIDs live in CBAR */
555 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
557 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
561 * We must write this before the TTBRs, since it determines the
562 * access behaviour of some fields (in particular, ASID[15:8]).
564 if (stage1 && smmu->version > ARM_SMMU_V1)
565 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
566 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
569 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
570 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
571 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
572 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
574 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
576 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
580 /* MAIRs (stage-1 only) */
582 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
583 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
587 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
588 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
590 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
591 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
592 reg |= ARM_SMMU_SCTLR_E;
594 if (smmu->impl && smmu->impl->write_sctlr)
595 smmu->impl->write_sctlr(smmu, idx, reg);
597 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
600 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
601 struct arm_smmu_device *smmu,
602 struct device *dev, unsigned int start)
604 if (smmu->impl && smmu->impl->alloc_context_bank)
605 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
607 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
610 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
611 struct arm_smmu_device *smmu,
614 int irq, start, ret = 0;
615 unsigned long ias, oas;
616 struct io_pgtable_ops *pgtbl_ops;
617 struct io_pgtable_cfg pgtbl_cfg;
618 enum io_pgtable_fmt fmt;
619 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
620 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
621 irqreturn_t (*context_fault)(int irq, void *dev);
623 mutex_lock(&smmu_domain->init_mutex);
624 if (smmu_domain->smmu)
627 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
628 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
629 smmu_domain->smmu = smmu;
634 * Mapping the requested stage onto what we support is surprisingly
635 * complicated, mainly because the spec allows S1+S2 SMMUs without
636 * support for nested translation. That means we end up with the
639 * Requested Supported Actual
649 * Note that you can't actually request stage-2 mappings.
651 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
652 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
653 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
654 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
657 * Choosing a suitable context format is even more fiddly. Until we
658 * grow some way for the caller to express a preference, and/or move
659 * the decision into the io-pgtable code where it arguably belongs,
660 * just aim for the closest thing to the rest of the system, and hope
661 * that the hardware isn't esoteric enough that we can't assume AArch64
662 * support to be a superset of AArch32 support...
664 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
665 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
666 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
667 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
668 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
669 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
670 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
671 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
672 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
673 ARM_SMMU_FEAT_FMT_AARCH64_16K |
674 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
675 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
677 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
682 switch (smmu_domain->stage) {
683 case ARM_SMMU_DOMAIN_S1:
684 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
685 start = smmu->num_s2_context_banks;
687 oas = smmu->ipa_size;
688 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
689 fmt = ARM_64_LPAE_S1;
690 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
691 fmt = ARM_32_LPAE_S1;
692 ias = min(ias, 32UL);
693 oas = min(oas, 40UL);
696 ias = min(ias, 32UL);
697 oas = min(oas, 32UL);
699 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
701 case ARM_SMMU_DOMAIN_NESTED:
703 * We will likely want to change this if/when KVM gets
706 case ARM_SMMU_DOMAIN_S2:
707 cfg->cbar = CBAR_TYPE_S2_TRANS;
709 ias = smmu->ipa_size;
711 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
712 fmt = ARM_64_LPAE_S2;
714 fmt = ARM_32_LPAE_S2;
715 ias = min(ias, 40UL);
716 oas = min(oas, 40UL);
718 if (smmu->version == ARM_SMMU_V2)
719 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
721 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
728 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
733 smmu_domain->smmu = smmu;
736 if (smmu->version < ARM_SMMU_V2) {
737 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
738 cfg->irptndx %= smmu->num_context_irqs;
740 cfg->irptndx = cfg->cbndx;
743 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
744 cfg->vmid = cfg->cbndx + 1;
746 cfg->asid = cfg->cbndx;
748 pgtbl_cfg = (struct io_pgtable_cfg) {
749 .pgsize_bitmap = smmu->pgsize_bitmap,
752 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
753 .tlb = smmu_domain->flush_ops,
754 .iommu_dev = smmu->dev,
757 if (smmu->impl && smmu->impl->init_context) {
758 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
763 if (smmu_domain->pgtbl_quirks)
764 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
766 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
772 /* Update the domain's page sizes to reflect the page table format */
773 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
775 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
776 domain->geometry.aperture_start = ~0UL << ias;
777 domain->geometry.aperture_end = ~0UL;
779 domain->geometry.aperture_end = (1UL << ias) - 1;
782 domain->geometry.force_aperture = true;
784 /* Initialise the context bank with our page table cfg */
785 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
786 arm_smmu_write_context_bank(smmu, cfg->cbndx);
789 * Request context fault interrupt. Do this last to avoid the
790 * handler seeing a half-initialised domain state.
792 irq = smmu->irqs[cfg->irptndx];
794 if (smmu->impl && smmu->impl->context_fault)
795 context_fault = smmu->impl->context_fault;
797 context_fault = arm_smmu_context_fault;
799 ret = devm_request_irq(smmu->dev, irq, context_fault,
800 IRQF_SHARED, "arm-smmu-context-fault", domain);
802 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
804 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
807 mutex_unlock(&smmu_domain->init_mutex);
809 /* Publish page table ops for map/unmap */
810 smmu_domain->pgtbl_ops = pgtbl_ops;
814 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
815 smmu_domain->smmu = NULL;
817 mutex_unlock(&smmu_domain->init_mutex);
821 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
823 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
824 struct arm_smmu_device *smmu = smmu_domain->smmu;
825 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
828 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
831 ret = arm_smmu_rpm_get(smmu);
836 * Disable the context bank and free the page tables before freeing
839 smmu->cbs[cfg->cbndx].cfg = NULL;
840 arm_smmu_write_context_bank(smmu, cfg->cbndx);
842 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
843 irq = smmu->irqs[cfg->irptndx];
844 devm_free_irq(smmu->dev, irq, domain);
847 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
848 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
850 arm_smmu_rpm_put(smmu);
853 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
855 struct arm_smmu_domain *smmu_domain;
857 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_IDENTITY) {
858 if (using_legacy_binding || type != IOMMU_DOMAIN_DMA)
862 * Allocate the domain and initialise some of its data structures.
863 * We can't really do anything meaningful until we've added a
866 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
870 mutex_init(&smmu_domain->init_mutex);
871 spin_lock_init(&smmu_domain->cb_lock);
873 return &smmu_domain->domain;
876 static void arm_smmu_domain_free(struct iommu_domain *domain)
878 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
881 * Free the domain resources. We assume that all devices have
882 * already been detached.
884 arm_smmu_destroy_domain_context(domain);
888 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
890 struct arm_smmu_smr *smr = smmu->smrs + idx;
891 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
892 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
894 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
895 reg |= ARM_SMMU_SMR_VALID;
896 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
899 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
901 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
904 if (smmu->impl && smmu->impl->write_s2cr) {
905 smmu->impl->write_s2cr(smmu, idx);
909 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
910 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
911 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
913 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
914 smmu->smrs[idx].valid)
915 reg |= ARM_SMMU_S2CR_EXIDVALID;
916 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
919 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
921 arm_smmu_write_s2cr(smmu, idx);
923 arm_smmu_write_smr(smmu, idx);
927 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
928 * should be called after sCR0 is written.
930 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
938 * If we've had to accommodate firmware memory regions, we may
939 * have live SMRs by now; tread carefully...
941 * Somewhat perversely, not having a free SMR for this test implies we
942 * can get away without it anyway, as we'll only be able to 'allocate'
943 * these SMRs for the ID/mask values we're already trusting to be OK.
945 for (i = 0; i < smmu->num_mapping_groups; i++)
946 if (!smmu->smrs[i].valid)
951 * SMR.ID bits may not be preserved if the corresponding MASK
952 * bits are set, so check each one separately. We can reject
953 * masters later if they try to claim IDs outside these masks.
955 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
956 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
957 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
958 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
960 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
961 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
962 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
963 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
966 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
968 struct arm_smmu_smr *smrs = smmu->smrs;
969 int i, free_idx = -ENOSPC;
971 /* Stream indexing is blissfully easy */
975 /* Validating SMRs is... less so */
976 for (i = 0; i < smmu->num_mapping_groups; ++i) {
977 if (!smrs[i].valid) {
979 * Note the first free entry we come across, which
980 * we'll claim in the end if nothing else matches.
987 * If the new entry is _entirely_ matched by an existing entry,
988 * then reuse that, with the guarantee that there also cannot
989 * be any subsequent conflicting entries. In normal use we'd
990 * expect simply identical entries for this case, but there's
991 * no harm in accommodating the generalisation.
993 if ((mask & smrs[i].mask) == mask &&
994 !((id ^ smrs[i].id) & ~smrs[i].mask))
997 * If the new entry has any other overlap with an existing one,
998 * though, then there always exists at least one stream ID
999 * which would cause a conflict, and we can't allow that risk.
1001 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1008 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1010 if (--smmu->s2crs[idx].count)
1013 smmu->s2crs[idx] = s2cr_init_val;
1015 smmu->smrs[idx].valid = false;
1020 static int arm_smmu_master_alloc_smes(struct device *dev)
1022 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1023 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1024 struct arm_smmu_device *smmu = cfg->smmu;
1025 struct arm_smmu_smr *smrs = smmu->smrs;
1028 mutex_lock(&smmu->stream_map_mutex);
1029 /* Figure out a viable stream map entry allocation */
1030 for_each_cfg_sme(cfg, fwspec, i, idx) {
1031 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1032 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1034 if (idx != INVALID_SMENDX) {
1039 ret = arm_smmu_find_sme(smmu, sid, mask);
1044 if (smrs && smmu->s2crs[idx].count == 0) {
1046 smrs[idx].mask = mask;
1047 smrs[idx].valid = true;
1049 smmu->s2crs[idx].count++;
1050 cfg->smendx[i] = (s16)idx;
1053 /* It worked! Now, poke the actual hardware */
1054 for_each_cfg_sme(cfg, fwspec, i, idx)
1055 arm_smmu_write_sme(smmu, idx);
1057 mutex_unlock(&smmu->stream_map_mutex);
1062 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1063 cfg->smendx[i] = INVALID_SMENDX;
1065 mutex_unlock(&smmu->stream_map_mutex);
1069 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1070 struct iommu_fwspec *fwspec)
1072 struct arm_smmu_device *smmu = cfg->smmu;
1075 mutex_lock(&smmu->stream_map_mutex);
1076 for_each_cfg_sme(cfg, fwspec, i, idx) {
1077 if (arm_smmu_free_sme(smmu, idx))
1078 arm_smmu_write_sme(smmu, idx);
1079 cfg->smendx[i] = INVALID_SMENDX;
1081 mutex_unlock(&smmu->stream_map_mutex);
1084 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1085 struct arm_smmu_master_cfg *cfg,
1086 struct iommu_fwspec *fwspec)
1088 struct arm_smmu_device *smmu = smmu_domain->smmu;
1089 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1090 u8 cbndx = smmu_domain->cfg.cbndx;
1091 enum arm_smmu_s2cr_type type;
1094 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1095 type = S2CR_TYPE_BYPASS;
1097 type = S2CR_TYPE_TRANS;
1099 for_each_cfg_sme(cfg, fwspec, i, idx) {
1100 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1103 s2cr[idx].type = type;
1104 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1105 s2cr[idx].cbndx = cbndx;
1106 arm_smmu_write_s2cr(smmu, idx);
1111 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1113 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1114 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1115 struct arm_smmu_master_cfg *cfg;
1116 struct arm_smmu_device *smmu;
1119 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1120 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1125 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1126 * domains between of_xlate() and probe_device() - we have no way to cope
1127 * with that, so until ARM gets converted to rely on groups and default
1128 * domains, just say no (but more politely than by dereferencing NULL).
1129 * This should be at least a WARN_ON once that's sorted.
1131 cfg = dev_iommu_priv_get(dev);
1137 ret = arm_smmu_rpm_get(smmu);
1141 /* Ensure that the domain is finalised */
1142 ret = arm_smmu_init_domain_context(domain, smmu, dev);
1147 * Sanity check the domain. We don't support domains across
1150 if (smmu_domain->smmu != smmu) {
1155 /* Looks ok, so add the device to the domain */
1156 ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1159 * Setup an autosuspend delay to avoid bouncing runpm state.
1160 * Otherwise, if a driver for a suspended consumer device
1161 * unmaps buffers, it will runpm resume/suspend for each one.
1163 * For example, when used by a GPU device, when an application
1164 * or game exits, it can trigger unmapping 100s or 1000s of
1165 * buffers. With a runpm cycle for each buffer, that adds up
1166 * to 5-10sec worth of reprogramming the context bank, while
1167 * the system appears to be locked up to the user.
1169 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1170 pm_runtime_use_autosuspend(smmu->dev);
1173 arm_smmu_rpm_put(smmu);
1177 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1178 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1179 int prot, gfp_t gfp, size_t *mapped)
1181 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1182 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1188 arm_smmu_rpm_get(smmu);
1189 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1190 arm_smmu_rpm_put(smmu);
1195 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1196 size_t pgsize, size_t pgcount,
1197 struct iommu_iotlb_gather *iotlb_gather)
1199 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1200 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1206 arm_smmu_rpm_get(smmu);
1207 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1208 arm_smmu_rpm_put(smmu);
1213 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1215 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1216 struct arm_smmu_device *smmu = smmu_domain->smmu;
1218 if (smmu_domain->flush_ops) {
1219 arm_smmu_rpm_get(smmu);
1220 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1221 arm_smmu_rpm_put(smmu);
1225 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1226 struct iommu_iotlb_gather *gather)
1228 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1229 struct arm_smmu_device *smmu = smmu_domain->smmu;
1234 arm_smmu_rpm_get(smmu);
1235 if (smmu->version == ARM_SMMU_V2 ||
1236 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1237 arm_smmu_tlb_sync_context(smmu_domain);
1239 arm_smmu_tlb_sync_global(smmu);
1240 arm_smmu_rpm_put(smmu);
1243 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1246 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1247 struct arm_smmu_device *smmu = smmu_domain->smmu;
1248 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1249 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1250 struct device *dev = smmu->dev;
1254 unsigned long va, flags;
1255 int ret, idx = cfg->cbndx;
1256 phys_addr_t addr = 0;
1258 ret = arm_smmu_rpm_get(smmu);
1262 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1263 va = iova & ~0xfffUL;
1264 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1265 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1267 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1269 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1270 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1272 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1274 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1276 arm_smmu_rpm_put(smmu);
1277 return ops->iova_to_phys(ops, iova);
1280 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1281 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1282 if (phys & ARM_SMMU_CB_PAR_F) {
1283 dev_err(dev, "translation fault!\n");
1284 dev_err(dev, "PAR = 0x%llx\n", phys);
1288 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1290 arm_smmu_rpm_put(smmu);
1295 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1298 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1299 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1304 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1305 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1306 return arm_smmu_iova_to_phys_hard(domain, iova);
1308 return ops->iova_to_phys(ops, iova);
1311 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1313 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1316 case IOMMU_CAP_CACHE_COHERENCY:
1318 * It's overwhelmingly the case in practice that when the pagetable
1319 * walk interface is connected to a coherent interconnect, all the
1320 * translation interfaces are too. Furthermore if the device is
1321 * natively coherent, then its translation interface must also be.
1323 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1324 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1325 case IOMMU_CAP_NOEXEC:
1326 case IOMMU_CAP_DEFERRED_FLUSH:
1334 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1336 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1339 return dev ? dev_get_drvdata(dev) : NULL;
1342 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1344 struct arm_smmu_device *smmu = NULL;
1345 struct arm_smmu_master_cfg *cfg;
1346 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1349 if (using_legacy_binding) {
1350 ret = arm_smmu_register_legacy_master(dev, &smmu);
1353 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1354 * will allocate/initialise a new one. Thus we need to update fwspec for
1357 fwspec = dev_iommu_fwspec_get(dev);
1360 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1361 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1363 return ERR_PTR(-ENODEV);
1367 for (i = 0; i < fwspec->num_ids; i++) {
1368 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1369 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1371 if (sid & ~smmu->streamid_mask) {
1372 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1373 sid, smmu->streamid_mask);
1376 if (mask & ~smmu->smr_mask_mask) {
1377 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1378 mask, smmu->smr_mask_mask);
1384 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1390 dev_iommu_priv_set(dev, cfg);
1392 cfg->smendx[i] = INVALID_SMENDX;
1394 ret = arm_smmu_rpm_get(smmu);
1398 ret = arm_smmu_master_alloc_smes(dev);
1399 arm_smmu_rpm_put(smmu);
1404 device_link_add(dev, smmu->dev,
1405 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1407 return &smmu->iommu;
1412 iommu_fwspec_free(dev);
1413 return ERR_PTR(ret);
1416 static void arm_smmu_release_device(struct device *dev)
1418 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1419 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1422 ret = arm_smmu_rpm_get(cfg->smmu);
1426 arm_smmu_master_free_smes(cfg, fwspec);
1428 arm_smmu_rpm_put(cfg->smmu);
1430 dev_iommu_priv_set(dev, NULL);
1434 static void arm_smmu_probe_finalize(struct device *dev)
1436 struct arm_smmu_master_cfg *cfg;
1437 struct arm_smmu_device *smmu;
1439 cfg = dev_iommu_priv_get(dev);
1442 if (smmu->impl && smmu->impl->probe_finalize)
1443 smmu->impl->probe_finalize(smmu, dev);
1446 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1448 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1449 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1450 struct arm_smmu_device *smmu = cfg->smmu;
1451 struct iommu_group *group = NULL;
1454 mutex_lock(&smmu->stream_map_mutex);
1455 for_each_cfg_sme(cfg, fwspec, i, idx) {
1456 if (group && smmu->s2crs[idx].group &&
1457 group != smmu->s2crs[idx].group) {
1458 mutex_unlock(&smmu->stream_map_mutex);
1459 return ERR_PTR(-EINVAL);
1462 group = smmu->s2crs[idx].group;
1466 mutex_unlock(&smmu->stream_map_mutex);
1467 return iommu_group_ref_get(group);
1470 if (dev_is_pci(dev))
1471 group = pci_device_group(dev);
1472 else if (dev_is_fsl_mc(dev))
1473 group = fsl_mc_device_group(dev);
1475 group = generic_device_group(dev);
1477 /* Remember group for faster lookups */
1479 for_each_cfg_sme(cfg, fwspec, i, idx)
1480 smmu->s2crs[idx].group = group;
1482 mutex_unlock(&smmu->stream_map_mutex);
1486 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
1488 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1491 mutex_lock(&smmu_domain->init_mutex);
1492 if (smmu_domain->smmu)
1495 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1496 mutex_unlock(&smmu_domain->init_mutex);
1501 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1502 unsigned long quirks)
1504 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1507 mutex_lock(&smmu_domain->init_mutex);
1508 if (smmu_domain->smmu)
1511 smmu_domain->pgtbl_quirks = quirks;
1512 mutex_unlock(&smmu_domain->init_mutex);
1517 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1521 if (args->args_count > 0)
1522 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1524 if (args->args_count > 1)
1525 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1526 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1527 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1529 return iommu_fwspec_add_ids(dev, &fwid, 1);
1532 static void arm_smmu_get_resv_regions(struct device *dev,
1533 struct list_head *head)
1535 struct iommu_resv_region *region;
1536 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1538 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1539 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1543 list_add_tail(®ion->list, head);
1545 iommu_dma_get_resv_regions(dev, head);
1548 static int arm_smmu_def_domain_type(struct device *dev)
1550 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1551 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1553 if (using_legacy_binding)
1554 return IOMMU_DOMAIN_IDENTITY;
1556 if (impl && impl->def_domain_type)
1557 return impl->def_domain_type(dev);
1562 static struct iommu_ops arm_smmu_ops = {
1563 .capable = arm_smmu_capable,
1564 .domain_alloc = arm_smmu_domain_alloc,
1565 .probe_device = arm_smmu_probe_device,
1566 .release_device = arm_smmu_release_device,
1567 .probe_finalize = arm_smmu_probe_finalize,
1568 .device_group = arm_smmu_device_group,
1569 .of_xlate = arm_smmu_of_xlate,
1570 .get_resv_regions = arm_smmu_get_resv_regions,
1571 .def_domain_type = arm_smmu_def_domain_type,
1572 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1573 .owner = THIS_MODULE,
1574 .default_domain_ops = &(const struct iommu_domain_ops) {
1575 .attach_dev = arm_smmu_attach_dev,
1576 .map_pages = arm_smmu_map_pages,
1577 .unmap_pages = arm_smmu_unmap_pages,
1578 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1579 .iotlb_sync = arm_smmu_iotlb_sync,
1580 .iova_to_phys = arm_smmu_iova_to_phys,
1581 .enable_nesting = arm_smmu_enable_nesting,
1582 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1583 .free = arm_smmu_domain_free,
1587 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1592 /* clear global FSR */
1593 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1594 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1597 * Reset stream mapping groups: Initial values mark all SMRn as
1598 * invalid and all S2CRn as bypass unless overridden.
1600 for (i = 0; i < smmu->num_mapping_groups; ++i)
1601 arm_smmu_write_sme(smmu, i);
1603 /* Make sure all context banks are disabled and clear CB_FSR */
1604 for (i = 0; i < smmu->num_context_banks; ++i) {
1605 arm_smmu_write_context_bank(smmu, i);
1606 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1609 /* Invalidate the TLB, just in case */
1610 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1611 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1613 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1615 /* Enable fault reporting */
1616 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1617 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1619 /* Disable TLB broadcasting. */
1620 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1622 /* Enable client access, handling unmatched streams as appropriate */
1623 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1625 reg |= ARM_SMMU_sCR0_USFCFG;
1627 reg &= ~ARM_SMMU_sCR0_USFCFG;
1629 /* Disable forced broadcasting */
1630 reg &= ~ARM_SMMU_sCR0_FB;
1632 /* Don't upgrade barriers */
1633 reg &= ~(ARM_SMMU_sCR0_BSU);
1635 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1636 reg |= ARM_SMMU_sCR0_VMID16EN;
1638 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1639 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1641 if (smmu->impl && smmu->impl->reset)
1642 smmu->impl->reset(smmu);
1644 /* Push the button */
1645 arm_smmu_tlb_sync_global(smmu);
1646 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1649 static int arm_smmu_id_size_to_bits(int size)
1668 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1672 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1675 dev_notice(smmu->dev, "probing hardware configuration...\n");
1676 dev_notice(smmu->dev, "SMMUv%d with:\n",
1677 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1680 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1682 /* Restrict available stages based on module parameter */
1683 if (force_stage == 1)
1684 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1685 else if (force_stage == 2)
1686 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1688 if (id & ARM_SMMU_ID0_S1TS) {
1689 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1690 dev_notice(smmu->dev, "\tstage 1 translation\n");
1693 if (id & ARM_SMMU_ID0_S2TS) {
1694 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1695 dev_notice(smmu->dev, "\tstage 2 translation\n");
1698 if (id & ARM_SMMU_ID0_NTS) {
1699 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1700 dev_notice(smmu->dev, "\tnested translation\n");
1703 if (!(smmu->features &
1704 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1705 dev_err(smmu->dev, "\tno translation support!\n");
1709 if ((id & ARM_SMMU_ID0_S1TS) &&
1710 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1711 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1712 dev_notice(smmu->dev, "\taddress translation ops\n");
1716 * In order for DMA API calls to work properly, we must defer to what
1717 * the FW says about coherency, regardless of what the hardware claims.
1718 * Fortunately, this also opens up a workaround for systems where the
1719 * ID register value has ended up configured incorrectly.
1721 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1722 if (cttw_fw || cttw_reg)
1723 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1724 cttw_fw ? "" : "non-");
1725 if (cttw_fw != cttw_reg)
1726 dev_notice(smmu->dev,
1727 "\t(IDR0.CTTW overridden by FW configuration)\n");
1729 /* Max. number of entries we have for stream matching/indexing */
1730 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1731 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1734 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1736 smmu->streamid_mask = size - 1;
1737 if (id & ARM_SMMU_ID0_SMS) {
1738 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1739 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1742 "stream-matching supported, but no SMRs present!\n");
1746 /* Zero-initialised to mark as invalid */
1747 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1752 dev_notice(smmu->dev,
1753 "\tstream matching with %u register groups", size);
1755 /* s2cr->type == 0 means translation, so initialise explicitly */
1756 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1760 for (i = 0; i < size; i++)
1761 smmu->s2crs[i] = s2cr_init_val;
1763 smmu->num_mapping_groups = size;
1764 mutex_init(&smmu->stream_map_mutex);
1765 spin_lock_init(&smmu->global_sync_lock);
1767 if (smmu->version < ARM_SMMU_V2 ||
1768 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1769 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1770 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1771 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1775 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1776 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1778 /* Check for size mismatch of SMMU address space from mapped region */
1779 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1780 if (smmu->numpage != 2 * size << smmu->pgshift)
1782 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1783 2 * size << smmu->pgshift, smmu->numpage);
1784 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1785 smmu->numpage = size;
1787 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1788 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1789 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1790 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1793 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1794 smmu->num_context_banks, smmu->num_s2_context_banks);
1795 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1796 sizeof(*smmu->cbs), GFP_KERNEL);
1801 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1802 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1803 smmu->ipa_size = size;
1805 /* The output mask is also applied for bypass */
1806 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1807 smmu->pa_size = size;
1809 if (id & ARM_SMMU_ID2_VMID16)
1810 smmu->features |= ARM_SMMU_FEAT_VMID16;
1813 * What the page table walker can address actually depends on which
1814 * descriptor format is in use, but since a) we don't know that yet,
1815 * and b) it can vary per context bank, this will have to do...
1817 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1819 "failed to set DMA mask for table walker\n");
1821 if (smmu->version < ARM_SMMU_V2) {
1822 smmu->va_size = smmu->ipa_size;
1823 if (smmu->version == ARM_SMMU_V1_64K)
1824 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1826 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1827 smmu->va_size = arm_smmu_id_size_to_bits(size);
1828 if (id & ARM_SMMU_ID2_PTFS_4K)
1829 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1830 if (id & ARM_SMMU_ID2_PTFS_16K)
1831 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1832 if (id & ARM_SMMU_ID2_PTFS_64K)
1833 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1836 if (smmu->impl && smmu->impl->cfg_probe) {
1837 ret = smmu->impl->cfg_probe(smmu);
1842 /* Now we've corralled the various formats, what'll it do? */
1843 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1844 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1845 if (smmu->features &
1846 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1847 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1848 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1849 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1850 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1851 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1853 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1854 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1856 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1857 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1858 smmu->pgsize_bitmap);
1861 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1862 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1863 smmu->va_size, smmu->ipa_size);
1865 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1866 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1867 smmu->ipa_size, smmu->pa_size);
1872 struct arm_smmu_match_data {
1873 enum arm_smmu_arch_version version;
1874 enum arm_smmu_implementation model;
1877 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1878 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1880 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1881 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1882 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1883 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1884 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1885 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1887 static const struct of_device_id arm_smmu_of_match[] = {
1888 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1889 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1890 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1891 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1892 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1893 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1894 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1895 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1898 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1901 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1906 case ACPI_IORT_SMMU_V1:
1907 case ACPI_IORT_SMMU_CORELINK_MMU400:
1908 smmu->version = ARM_SMMU_V1;
1909 smmu->model = GENERIC_SMMU;
1911 case ACPI_IORT_SMMU_CORELINK_MMU401:
1912 smmu->version = ARM_SMMU_V1_64K;
1913 smmu->model = GENERIC_SMMU;
1915 case ACPI_IORT_SMMU_V2:
1916 smmu->version = ARM_SMMU_V2;
1917 smmu->model = GENERIC_SMMU;
1919 case ACPI_IORT_SMMU_CORELINK_MMU500:
1920 smmu->version = ARM_SMMU_V2;
1921 smmu->model = ARM_MMU500;
1923 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1924 smmu->version = ARM_SMMU_V2;
1925 smmu->model = CAVIUM_SMMUV2;
1934 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1935 u32 *global_irqs, u32 *pmu_irqs)
1937 struct device *dev = smmu->dev;
1938 struct acpi_iort_node *node =
1939 *(struct acpi_iort_node **)dev_get_platdata(dev);
1940 struct acpi_iort_smmu *iort_smmu;
1943 /* Retrieve SMMU1/2 specific data */
1944 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1946 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1950 /* Ignore the configuration access interrupt */
1954 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1955 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1960 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1961 u32 *global_irqs, u32 *pmu_irqs)
1967 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
1968 u32 *global_irqs, u32 *pmu_irqs)
1970 const struct arm_smmu_match_data *data;
1971 struct device *dev = smmu->dev;
1972 bool legacy_binding;
1974 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
1975 return dev_err_probe(dev, -ENODEV,
1976 "missing #global-interrupts property\n");
1979 data = of_device_get_match_data(dev);
1980 smmu->version = data->version;
1981 smmu->model = data->model;
1983 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1984 if (legacy_binding && !using_generic_binding) {
1985 if (!using_legacy_binding) {
1986 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
1987 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
1989 using_legacy_binding = true;
1990 } else if (!legacy_binding && !using_legacy_binding) {
1991 using_generic_binding = true;
1993 dev_err(dev, "not probing due to mismatched DT properties\n");
1997 if (of_dma_is_coherent(dev->of_node))
1998 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2003 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2005 struct list_head rmr_list;
2006 struct iommu_resv_region *e;
2010 INIT_LIST_HEAD(&rmr_list);
2011 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2014 * Rather than trying to look at existing mappings that
2015 * are setup by the firmware and then invalidate the ones
2016 * that do no have matching RMR entries, just disable the
2017 * SMMU until it gets enabled again in the reset routine.
2019 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2020 reg |= ARM_SMMU_sCR0_CLIENTPD;
2021 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2023 list_for_each_entry(e, &rmr_list, list) {
2024 struct iommu_iort_rmr_data *rmr;
2027 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2028 for (i = 0; i < rmr->num_sids; i++) {
2029 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2033 if (smmu->s2crs[idx].count == 0) {
2034 smmu->smrs[idx].id = rmr->sids[i];
2035 smmu->smrs[idx].mask = 0;
2036 smmu->smrs[idx].valid = true;
2038 smmu->s2crs[idx].count++;
2039 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2040 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2046 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2047 cnt == 1 ? "" : "s");
2048 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2051 static int arm_smmu_device_probe(struct platform_device *pdev)
2053 struct resource *res;
2054 struct arm_smmu_device *smmu;
2055 struct device *dev = &pdev->dev;
2056 int num_irqs, i, err;
2057 u32 global_irqs, pmu_irqs;
2058 irqreturn_t (*global_fault)(int irq, void *dev);
2060 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2062 dev_err(dev, "failed to allocate arm_smmu_device\n");
2068 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2070 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2074 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2075 if (IS_ERR(smmu->base))
2076 return PTR_ERR(smmu->base);
2077 smmu->ioaddr = res->start;
2080 * The resource size should effectively match the value of SMMU_TOP;
2081 * stash that temporarily until we know PAGESIZE to validate it with.
2083 smmu->numpage = resource_size(res);
2085 smmu = arm_smmu_impl_init(smmu);
2087 return PTR_ERR(smmu);
2089 num_irqs = platform_irq_count(pdev);
2091 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2092 if (smmu->num_context_irqs <= 0)
2093 return dev_err_probe(dev, -ENODEV,
2094 "found %d interrupts but expected at least %d\n",
2095 num_irqs, global_irqs + pmu_irqs + 1);
2097 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2098 sizeof(*smmu->irqs), GFP_KERNEL);
2100 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2101 smmu->num_context_irqs);
2103 for (i = 0; i < smmu->num_context_irqs; i++) {
2104 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2108 smmu->irqs[i] = irq;
2111 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2113 dev_err(dev, "failed to get clocks %d\n", err);
2116 smmu->num_clks = err;
2118 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2122 err = arm_smmu_device_cfg_probe(smmu);
2126 if (smmu->version == ARM_SMMU_V2) {
2127 if (smmu->num_context_banks > smmu->num_context_irqs) {
2129 "found only %d context irq(s) but %d required\n",
2130 smmu->num_context_irqs, smmu->num_context_banks);
2134 /* Ignore superfluous interrupts */
2135 smmu->num_context_irqs = smmu->num_context_banks;
2138 if (smmu->impl && smmu->impl->global_fault)
2139 global_fault = smmu->impl->global_fault;
2141 global_fault = arm_smmu_global_fault;
2143 for (i = 0; i < global_irqs; i++) {
2144 int irq = platform_get_irq(pdev, i);
2149 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2150 "arm-smmu global fault", smmu);
2152 return dev_err_probe(dev, err,
2153 "failed to request global IRQ %d (%u)\n",
2157 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2158 "smmu.%pa", &smmu->ioaddr);
2160 dev_err(dev, "Failed to register iommu in sysfs\n");
2164 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
2166 dev_err(dev, "Failed to register iommu\n");
2167 iommu_device_sysfs_remove(&smmu->iommu);
2171 platform_set_drvdata(pdev, smmu);
2173 /* Check for RMRs and install bypass SMRs if any */
2174 arm_smmu_rmr_install_bypass_smr(smmu);
2176 arm_smmu_device_reset(smmu);
2177 arm_smmu_test_smr_masks(smmu);
2180 * We want to avoid touching dev->power.lock in fastpaths unless
2181 * it's really going to do something useful - pm_runtime_enabled()
2182 * can serve as an ideal proxy for that decision. So, conditionally
2183 * enable pm_runtime.
2185 if (dev->pm_domain) {
2186 pm_runtime_set_active(dev);
2187 pm_runtime_enable(dev);
2193 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2195 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2197 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2198 dev_notice(&pdev->dev, "disabling translation\n");
2200 arm_smmu_rpm_get(smmu);
2201 /* Turn the thing off */
2202 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2203 arm_smmu_rpm_put(smmu);
2205 if (pm_runtime_enabled(smmu->dev))
2206 pm_runtime_force_suspend(smmu->dev);
2208 clk_bulk_disable(smmu->num_clks, smmu->clks);
2210 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2213 static void arm_smmu_device_remove(struct platform_device *pdev)
2215 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2217 iommu_device_unregister(&smmu->iommu);
2218 iommu_device_sysfs_remove(&smmu->iommu);
2220 arm_smmu_device_shutdown(pdev);
2223 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2225 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2228 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2232 arm_smmu_device_reset(smmu);
2237 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2239 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2241 clk_bulk_disable(smmu->num_clks, smmu->clks);
2246 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2249 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2251 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2255 if (pm_runtime_suspended(dev))
2258 ret = arm_smmu_runtime_resume(dev);
2260 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2265 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2268 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2270 if (pm_runtime_suspended(dev))
2273 ret = arm_smmu_runtime_suspend(dev);
2278 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2282 static const struct dev_pm_ops arm_smmu_pm_ops = {
2283 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2284 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2285 arm_smmu_runtime_resume, NULL)
2288 static struct platform_driver arm_smmu_driver = {
2291 .of_match_table = arm_smmu_of_match,
2292 .pm = &arm_smmu_pm_ops,
2293 .suppress_bind_attrs = true,
2295 .probe = arm_smmu_device_probe,
2296 .remove_new = arm_smmu_device_remove,
2297 .shutdown = arm_smmu_device_shutdown,
2299 module_platform_driver(arm_smmu_driver);
2301 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2302 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2303 MODULE_ALIAS("platform:arm-smmu");
2304 MODULE_LICENSE("GPL v2");