1 // SPDX-License-Identifier: GPL-2.0
3 * IOMMU API for ARM architected SMMUv3 implementations.
5 * Copyright (C) 2015 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
9 * This driver is powered by bad coffee and bombay mix.
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitfield.h>
15 #include <linux/bitops.h>
16 #include <linux/crash_dump.h>
17 #include <linux/delay.h>
18 #include <linux/dma-iommu.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/module.h>
25 #include <linux/msi.h>
27 #include <linux/of_address.h>
28 #include <linux/of_iommu.h>
29 #include <linux/of_platform.h>
30 #include <linux/pci.h>
31 #include <linux/pci-ats.h>
32 #include <linux/platform_device.h>
34 #include <linux/amba/bus.h>
37 #define ARM_SMMU_IDR0 0x0
38 #define IDR0_ST_LVL GENMASK(28, 27)
39 #define IDR0_ST_LVL_2LVL 1
40 #define IDR0_STALL_MODEL GENMASK(25, 24)
41 #define IDR0_STALL_MODEL_STALL 0
42 #define IDR0_STALL_MODEL_FORCE 2
43 #define IDR0_TTENDIAN GENMASK(22, 21)
44 #define IDR0_TTENDIAN_MIXED 0
45 #define IDR0_TTENDIAN_LE 2
46 #define IDR0_TTENDIAN_BE 3
47 #define IDR0_CD2L (1 << 19)
48 #define IDR0_VMID16 (1 << 18)
49 #define IDR0_PRI (1 << 16)
50 #define IDR0_SEV (1 << 14)
51 #define IDR0_MSI (1 << 13)
52 #define IDR0_ASID16 (1 << 12)
53 #define IDR0_ATS (1 << 10)
54 #define IDR0_HYP (1 << 9)
55 #define IDR0_COHACC (1 << 4)
56 #define IDR0_TTF GENMASK(3, 2)
57 #define IDR0_TTF_AARCH64 2
58 #define IDR0_TTF_AARCH32_64 3
59 #define IDR0_S1P (1 << 1)
60 #define IDR0_S2P (1 << 0)
62 #define ARM_SMMU_IDR1 0x4
63 #define IDR1_TABLES_PRESET (1 << 30)
64 #define IDR1_QUEUES_PRESET (1 << 29)
65 #define IDR1_REL (1 << 28)
66 #define IDR1_CMDQS GENMASK(25, 21)
67 #define IDR1_EVTQS GENMASK(20, 16)
68 #define IDR1_PRIQS GENMASK(15, 11)
69 #define IDR1_SSIDSIZE GENMASK(10, 6)
70 #define IDR1_SIDSIZE GENMASK(5, 0)
72 #define ARM_SMMU_IDR3 0xc
73 #define IDR3_RIL (1 << 10)
75 #define ARM_SMMU_IDR5 0x14
76 #define IDR5_STALL_MAX GENMASK(31, 16)
77 #define IDR5_GRAN64K (1 << 6)
78 #define IDR5_GRAN16K (1 << 5)
79 #define IDR5_GRAN4K (1 << 4)
80 #define IDR5_OAS GENMASK(2, 0)
81 #define IDR5_OAS_32_BIT 0
82 #define IDR5_OAS_36_BIT 1
83 #define IDR5_OAS_40_BIT 2
84 #define IDR5_OAS_42_BIT 3
85 #define IDR5_OAS_44_BIT 4
86 #define IDR5_OAS_48_BIT 5
87 #define IDR5_OAS_52_BIT 6
88 #define IDR5_VAX GENMASK(11, 10)
89 #define IDR5_VAX_52_BIT 1
91 #define ARM_SMMU_CR0 0x20
92 #define CR0_ATSCHK (1 << 4)
93 #define CR0_CMDQEN (1 << 3)
94 #define CR0_EVTQEN (1 << 2)
95 #define CR0_PRIQEN (1 << 1)
96 #define CR0_SMMUEN (1 << 0)
98 #define ARM_SMMU_CR0ACK 0x24
100 #define ARM_SMMU_CR1 0x28
101 #define CR1_TABLE_SH GENMASK(11, 10)
102 #define CR1_TABLE_OC GENMASK(9, 8)
103 #define CR1_TABLE_IC GENMASK(7, 6)
104 #define CR1_QUEUE_SH GENMASK(5, 4)
105 #define CR1_QUEUE_OC GENMASK(3, 2)
106 #define CR1_QUEUE_IC GENMASK(1, 0)
107 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
108 #define CR1_CACHE_NC 0
109 #define CR1_CACHE_WB 1
110 #define CR1_CACHE_WT 2
112 #define ARM_SMMU_CR2 0x2c
113 #define CR2_PTM (1 << 2)
114 #define CR2_RECINVSID (1 << 1)
115 #define CR2_E2H (1 << 0)
117 #define ARM_SMMU_GBPA 0x44
118 #define GBPA_UPDATE (1 << 31)
119 #define GBPA_ABORT (1 << 20)
121 #define ARM_SMMU_IRQ_CTRL 0x50
122 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
123 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
124 #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
126 #define ARM_SMMU_IRQ_CTRLACK 0x54
128 #define ARM_SMMU_GERROR 0x60
129 #define GERROR_SFM_ERR (1 << 8)
130 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
131 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
132 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
133 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
134 #define GERROR_PRIQ_ABT_ERR (1 << 3)
135 #define GERROR_EVTQ_ABT_ERR (1 << 2)
136 #define GERROR_CMDQ_ERR (1 << 0)
137 #define GERROR_ERR_MASK 0xfd
139 #define ARM_SMMU_GERRORN 0x64
141 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
142 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
143 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
145 #define ARM_SMMU_STRTAB_BASE 0x80
146 #define STRTAB_BASE_RA (1UL << 62)
147 #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
149 #define ARM_SMMU_STRTAB_BASE_CFG 0x88
150 #define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
151 #define STRTAB_BASE_CFG_FMT_LINEAR 0
152 #define STRTAB_BASE_CFG_FMT_2LVL 1
153 #define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
154 #define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
156 #define ARM_SMMU_CMDQ_BASE 0x90
157 #define ARM_SMMU_CMDQ_PROD 0x98
158 #define ARM_SMMU_CMDQ_CONS 0x9c
160 #define ARM_SMMU_EVTQ_BASE 0xa0
161 #define ARM_SMMU_EVTQ_PROD 0x100a8
162 #define ARM_SMMU_EVTQ_CONS 0x100ac
163 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
164 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
165 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
167 #define ARM_SMMU_PRIQ_BASE 0xc0
168 #define ARM_SMMU_PRIQ_PROD 0x100c8
169 #define ARM_SMMU_PRIQ_CONS 0x100cc
170 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
171 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
172 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
174 /* Common MSI config fields */
175 #define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
176 #define MSI_CFG2_SH GENMASK(5, 4)
177 #define MSI_CFG2_MEMATTR GENMASK(3, 0)
179 /* Common memory attribute values */
180 #define ARM_SMMU_SH_NSH 0
181 #define ARM_SMMU_SH_OSH 2
182 #define ARM_SMMU_SH_ISH 3
183 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
184 #define ARM_SMMU_MEMATTR_OIWB 0xf
186 #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
187 #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
188 #define Q_OVERFLOW_FLAG (1U << 31)
189 #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
190 #define Q_ENT(q, p) ((q)->base + \
191 Q_IDX(&((q)->llq), p) * \
194 #define Q_BASE_RWA (1UL << 62)
195 #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
196 #define Q_BASE_LOG2SIZE GENMASK(4, 0)
198 /* Ensure DMA allocations are naturally aligned */
199 #ifdef CONFIG_CMA_ALIGNMENT
200 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
202 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1)
208 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
209 * 2lvl: 128k L1 entries,
210 * 256 lazy entries per table (each table covers a PCI bus)
212 #define STRTAB_L1_SZ_SHIFT 20
213 #define STRTAB_SPLIT 8
215 #define STRTAB_L1_DESC_DWORDS 1
216 #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
217 #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
219 #define STRTAB_STE_DWORDS 8
220 #define STRTAB_STE_0_V (1UL << 0)
221 #define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
222 #define STRTAB_STE_0_CFG_ABORT 0
223 #define STRTAB_STE_0_CFG_BYPASS 4
224 #define STRTAB_STE_0_CFG_S1_TRANS 5
225 #define STRTAB_STE_0_CFG_S2_TRANS 6
227 #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
228 #define STRTAB_STE_0_S1FMT_LINEAR 0
229 #define STRTAB_STE_0_S1FMT_64K_L2 2
230 #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
231 #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
233 #define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0)
234 #define STRTAB_STE_1_S1DSS_TERMINATE 0x0
235 #define STRTAB_STE_1_S1DSS_BYPASS 0x1
236 #define STRTAB_STE_1_S1DSS_SSID0 0x2
238 #define STRTAB_STE_1_S1C_CACHE_NC 0UL
239 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
240 #define STRTAB_STE_1_S1C_CACHE_WT 2UL
241 #define STRTAB_STE_1_S1C_CACHE_WB 3UL
242 #define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
243 #define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
244 #define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
246 #define STRTAB_STE_1_S1STALLD (1UL << 27)
248 #define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
249 #define STRTAB_STE_1_EATS_ABT 0UL
250 #define STRTAB_STE_1_EATS_TRANS 1UL
251 #define STRTAB_STE_1_EATS_S1CHK 2UL
253 #define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
254 #define STRTAB_STE_1_STRW_NSEL1 0UL
255 #define STRTAB_STE_1_STRW_EL2 2UL
257 #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
258 #define STRTAB_STE_1_SHCFG_INCOMING 1UL
260 #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
261 #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
262 #define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
263 #define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6)
264 #define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8)
265 #define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10)
266 #define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12)
267 #define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14)
268 #define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16)
269 #define STRTAB_STE_2_S2AA64 (1UL << 51)
270 #define STRTAB_STE_2_S2ENDI (1UL << 52)
271 #define STRTAB_STE_2_S2PTW (1UL << 54)
272 #define STRTAB_STE_2_S2R (1UL << 58)
274 #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
277 * Context descriptors.
279 * Linear: when less than 1024 SSIDs are supported
280 * 2lvl: at most 1024 L1 entries,
281 * 1024 lazy entries per table.
283 #define CTXDESC_SPLIT 10
284 #define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT)
286 #define CTXDESC_L1_DESC_DWORDS 1
287 #define CTXDESC_L1_DESC_V (1UL << 0)
288 #define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12)
290 #define CTXDESC_CD_DWORDS 8
291 #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
292 #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
293 #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
294 #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
295 #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
296 #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
297 #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
299 #define CTXDESC_CD_0_ENDI (1UL << 15)
300 #define CTXDESC_CD_0_V (1UL << 31)
302 #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
303 #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
305 #define CTXDESC_CD_0_AA64 (1UL << 41)
306 #define CTXDESC_CD_0_S (1UL << 44)
307 #define CTXDESC_CD_0_R (1UL << 45)
308 #define CTXDESC_CD_0_A (1UL << 46)
309 #define CTXDESC_CD_0_ASET (1UL << 47)
310 #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
312 #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
315 * When the SMMU only supports linear context descriptor tables, pick a
316 * reasonable size limit (64kB).
318 #define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
321 #define CMDQ_ENT_SZ_SHIFT 4
322 #define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
323 #define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
325 #define CMDQ_CONS_ERR GENMASK(30, 24)
326 #define CMDQ_ERR_CERROR_NONE_IDX 0
327 #define CMDQ_ERR_CERROR_ILL_IDX 1
328 #define CMDQ_ERR_CERROR_ABT_IDX 2
329 #define CMDQ_ERR_CERROR_ATC_INV_IDX 3
331 #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
334 * This is used to size the command queue and therefore must be at least
335 * BITS_PER_LONG so that the valid_map works correctly (it relies on the
336 * total number of queue entries being a multiple of BITS_PER_LONG).
338 #define CMDQ_BATCH_ENTRIES BITS_PER_LONG
340 #define CMDQ_0_OP GENMASK_ULL(7, 0)
341 #define CMDQ_0_SSV (1UL << 11)
343 #define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
344 #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
345 #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
347 #define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12)
348 #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
349 #define CMDQ_CFGI_1_LEAF (1UL << 0)
350 #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
352 #define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12)
353 #define CMDQ_TLBI_RANGE_NUM_MAX 31
354 #define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20)
355 #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
356 #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
357 #define CMDQ_TLBI_1_LEAF (1UL << 0)
358 #define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8)
359 #define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10)
360 #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
361 #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
363 #define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
364 #define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
365 #define CMDQ_ATC_0_GLOBAL (1UL << 9)
366 #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
367 #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
369 #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
370 #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
371 #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
372 #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
374 #define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
375 #define CMDQ_SYNC_0_CS_NONE 0
376 #define CMDQ_SYNC_0_CS_IRQ 1
377 #define CMDQ_SYNC_0_CS_SEV 2
378 #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
379 #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
380 #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
381 #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
384 #define EVTQ_ENT_SZ_SHIFT 5
385 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
386 #define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
388 #define EVTQ_0_ID GENMASK_ULL(7, 0)
391 #define PRIQ_ENT_SZ_SHIFT 4
392 #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
393 #define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
395 #define PRIQ_0_SID GENMASK_ULL(31, 0)
396 #define PRIQ_0_SSID GENMASK_ULL(51, 32)
397 #define PRIQ_0_PERM_PRIV (1UL << 58)
398 #define PRIQ_0_PERM_EXEC (1UL << 59)
399 #define PRIQ_0_PERM_READ (1UL << 60)
400 #define PRIQ_0_PERM_WRITE (1UL << 61)
401 #define PRIQ_0_PRG_LAST (1UL << 62)
402 #define PRIQ_0_SSID_V (1UL << 63)
404 #define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
405 #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
407 /* High-level queue structures */
408 #define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */
409 #define ARM_SMMU_POLL_SPIN_COUNT 10
411 #define MSI_IOVA_BASE 0x8000000
412 #define MSI_IOVA_LENGTH 0x100000
414 static bool disable_bypass = 1;
415 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
416 MODULE_PARM_DESC(disable_bypass,
417 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
425 enum arm_smmu_msi_index {
432 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
434 ARM_SMMU_EVTQ_IRQ_CFG0,
435 ARM_SMMU_EVTQ_IRQ_CFG1,
436 ARM_SMMU_EVTQ_IRQ_CFG2,
438 [GERROR_MSI_INDEX] = {
439 ARM_SMMU_GERROR_IRQ_CFG0,
440 ARM_SMMU_GERROR_IRQ_CFG1,
441 ARM_SMMU_GERROR_IRQ_CFG2,
444 ARM_SMMU_PRIQ_IRQ_CFG0,
445 ARM_SMMU_PRIQ_IRQ_CFG1,
446 ARM_SMMU_PRIQ_IRQ_CFG2,
450 struct arm_smmu_cmdq_ent {
453 bool substream_valid;
455 /* Command-specific fields */
457 #define CMDQ_OP_PREFETCH_CFG 0x1
464 #define CMDQ_OP_CFGI_STE 0x3
465 #define CMDQ_OP_CFGI_ALL 0x4
466 #define CMDQ_OP_CFGI_CD 0x5
467 #define CMDQ_OP_CFGI_CD_ALL 0x6
477 #define CMDQ_OP_TLBI_NH_ASID 0x11
478 #define CMDQ_OP_TLBI_NH_VA 0x12
479 #define CMDQ_OP_TLBI_EL2_ALL 0x20
480 #define CMDQ_OP_TLBI_S12_VMALL 0x28
481 #define CMDQ_OP_TLBI_S2_IPA 0x2a
482 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
494 #define CMDQ_OP_ATC_INV 0x40
495 #define ATC_INV_SIZE_ALL 52
504 #define CMDQ_OP_PRI_RESP 0x41
512 #define CMDQ_OP_CMD_SYNC 0x46
519 struct arm_smmu_ll_queue {
530 u8 __pad[SMP_CACHE_BYTES];
531 } ____cacheline_aligned_in_smp;
535 struct arm_smmu_queue {
536 struct arm_smmu_ll_queue llq;
537 int irq; /* Wired interrupt */
545 u32 __iomem *prod_reg;
546 u32 __iomem *cons_reg;
549 struct arm_smmu_queue_poll {
552 unsigned int spin_cnt;
556 struct arm_smmu_cmdq {
557 struct arm_smmu_queue q;
558 atomic_long_t *valid_map;
563 struct arm_smmu_cmdq_batch {
564 u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
568 struct arm_smmu_evtq {
569 struct arm_smmu_queue q;
573 struct arm_smmu_priq {
574 struct arm_smmu_queue q;
577 /* High-level stream table and context descriptor structures */
578 struct arm_smmu_strtab_l1_desc {
582 dma_addr_t l2ptr_dma;
585 struct arm_smmu_ctx_desc {
592 struct arm_smmu_l1_ctx_desc {
594 dma_addr_t l2ptr_dma;
597 struct arm_smmu_ctx_desc_cfg {
599 dma_addr_t cdtab_dma;
600 struct arm_smmu_l1_ctx_desc *l1_desc;
601 unsigned int num_l1_ents;
604 struct arm_smmu_s1_cfg {
605 struct arm_smmu_ctx_desc_cfg cdcfg;
606 struct arm_smmu_ctx_desc cd;
611 struct arm_smmu_s2_cfg {
617 struct arm_smmu_strtab_cfg {
619 dma_addr_t strtab_dma;
620 struct arm_smmu_strtab_l1_desc *l1_desc;
621 unsigned int num_l1_ents;
627 /* An SMMUv3 instance */
628 struct arm_smmu_device {
632 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
633 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
634 #define ARM_SMMU_FEAT_TT_LE (1 << 2)
635 #define ARM_SMMU_FEAT_TT_BE (1 << 3)
636 #define ARM_SMMU_FEAT_PRI (1 << 4)
637 #define ARM_SMMU_FEAT_ATS (1 << 5)
638 #define ARM_SMMU_FEAT_SEV (1 << 6)
639 #define ARM_SMMU_FEAT_MSI (1 << 7)
640 #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
641 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
642 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
643 #define ARM_SMMU_FEAT_STALLS (1 << 11)
644 #define ARM_SMMU_FEAT_HYP (1 << 12)
645 #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
646 #define ARM_SMMU_FEAT_VAX (1 << 14)
647 #define ARM_SMMU_FEAT_RANGE_INV (1 << 15)
650 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
651 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
654 struct arm_smmu_cmdq cmdq;
655 struct arm_smmu_evtq evtq;
656 struct arm_smmu_priq priq;
661 unsigned long ias; /* IPA */
662 unsigned long oas; /* PA */
663 unsigned long pgsize_bitmap;
665 #define ARM_SMMU_MAX_ASIDS (1 << 16)
666 unsigned int asid_bits;
667 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
669 #define ARM_SMMU_MAX_VMIDS (1 << 16)
670 unsigned int vmid_bits;
671 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
673 unsigned int ssid_bits;
674 unsigned int sid_bits;
676 struct arm_smmu_strtab_cfg strtab_cfg;
678 /* IOMMU core code handle */
679 struct iommu_device iommu;
682 /* SMMU private data for each master */
683 struct arm_smmu_master {
684 struct arm_smmu_device *smmu;
686 struct arm_smmu_domain *domain;
687 struct list_head domain_head;
689 unsigned int num_sids;
691 unsigned int ssid_bits;
694 /* SMMU private data for an IOMMU domain */
695 enum arm_smmu_domain_stage {
696 ARM_SMMU_DOMAIN_S1 = 0,
698 ARM_SMMU_DOMAIN_NESTED,
699 ARM_SMMU_DOMAIN_BYPASS,
702 struct arm_smmu_domain {
703 struct arm_smmu_device *smmu;
704 struct mutex init_mutex; /* Protects smmu pointer */
706 struct io_pgtable_ops *pgtbl_ops;
708 atomic_t nr_ats_masters;
710 enum arm_smmu_domain_stage stage;
712 struct arm_smmu_s1_cfg s1_cfg;
713 struct arm_smmu_s2_cfg s2_cfg;
716 struct iommu_domain domain;
718 struct list_head devices;
719 spinlock_t devices_lock;
722 struct arm_smmu_option_prop {
727 static struct arm_smmu_option_prop arm_smmu_options[] = {
728 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
729 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
733 static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset,
734 struct arm_smmu_device *smmu)
736 if ((offset > SZ_64K) &&
737 (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY))
740 return smmu->base + offset;
743 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
745 return container_of(dom, struct arm_smmu_domain, domain);
748 static void parse_driver_options(struct arm_smmu_device *smmu)
753 if (of_property_read_bool(smmu->dev->of_node,
754 arm_smmu_options[i].prop)) {
755 smmu->options |= arm_smmu_options[i].opt;
756 dev_notice(smmu->dev, "option %s\n",
757 arm_smmu_options[i].prop);
759 } while (arm_smmu_options[++i].opt);
762 /* Low-level queue manipulation functions */
763 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
765 u32 space, prod, cons;
767 prod = Q_IDX(q, q->prod);
768 cons = Q_IDX(q, q->cons);
770 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
771 space = (1 << q->max_n_shift) - (prod - cons);
778 static bool queue_full(struct arm_smmu_ll_queue *q)
780 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
781 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
784 static bool queue_empty(struct arm_smmu_ll_queue *q)
786 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
787 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
790 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
792 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
793 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
794 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
795 (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
798 static void queue_sync_cons_out(struct arm_smmu_queue *q)
801 * Ensure that all CPU accesses (reads and writes) to the queue
802 * are complete before we update the cons pointer.
805 writel_relaxed(q->llq.cons, q->cons_reg);
808 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
810 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
811 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
814 static int queue_sync_prod_in(struct arm_smmu_queue *q)
817 u32 prod = readl_relaxed(q->prod_reg);
819 if (Q_OVF(prod) != Q_OVF(q->llq.prod))
826 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
828 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
829 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
832 static void queue_poll_init(struct arm_smmu_device *smmu,
833 struct arm_smmu_queue_poll *qp)
837 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
838 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
841 static int queue_poll(struct arm_smmu_queue_poll *qp)
843 if (ktime_compare(ktime_get(), qp->timeout) > 0)
848 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
859 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
863 for (i = 0; i < n_dwords; ++i)
864 *dst++ = cpu_to_le64(*src++);
867 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
871 for (i = 0; i < n_dwords; ++i)
872 *dst++ = le64_to_cpu(*src++);
875 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
877 if (queue_empty(&q->llq))
880 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
881 queue_inc_cons(&q->llq);
882 queue_sync_cons_out(q);
886 /* High-level queue accessors */
887 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
889 memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
890 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
892 switch (ent->opcode) {
893 case CMDQ_OP_TLBI_EL2_ALL:
894 case CMDQ_OP_TLBI_NSNH_ALL:
896 case CMDQ_OP_PREFETCH_CFG:
897 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
898 cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size);
899 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
901 case CMDQ_OP_CFGI_CD:
902 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
904 case CMDQ_OP_CFGI_STE:
905 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
906 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
908 case CMDQ_OP_CFGI_CD_ALL:
909 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
911 case CMDQ_OP_CFGI_ALL:
912 /* Cover the entire SID range */
913 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
915 case CMDQ_OP_TLBI_NH_VA:
916 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
917 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
918 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
919 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
920 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
921 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
922 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
923 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
925 case CMDQ_OP_TLBI_S2_IPA:
926 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
927 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
928 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
929 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
930 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
931 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
932 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
934 case CMDQ_OP_TLBI_NH_ASID:
935 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
937 case CMDQ_OP_TLBI_S12_VMALL:
938 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
940 case CMDQ_OP_ATC_INV:
941 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
942 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
943 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
944 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
945 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
946 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
948 case CMDQ_OP_PRI_RESP:
949 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
950 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
951 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
952 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
953 switch (ent->pri.resp) {
961 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
963 case CMDQ_OP_CMD_SYNC:
964 if (ent->sync.msiaddr) {
965 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
966 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
968 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
970 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
971 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
980 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
983 struct arm_smmu_queue *q = &smmu->cmdq.q;
984 struct arm_smmu_cmdq_ent ent = {
985 .opcode = CMDQ_OP_CMD_SYNC,
989 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
990 * payload, so the write will zero the entire command on that platform.
992 if (smmu->features & ARM_SMMU_FEAT_MSI &&
993 smmu->features & ARM_SMMU_FEAT_COHERENCY) {
994 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
998 arm_smmu_cmdq_build_cmd(cmd, &ent);
1001 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
1003 static const char *cerror_str[] = {
1004 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
1005 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
1006 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
1007 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
1011 u64 cmd[CMDQ_ENT_DWORDS];
1012 struct arm_smmu_queue *q = &smmu->cmdq.q;
1013 u32 cons = readl_relaxed(q->cons_reg);
1014 u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
1015 struct arm_smmu_cmdq_ent cmd_sync = {
1016 .opcode = CMDQ_OP_CMD_SYNC,
1019 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
1020 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
1023 case CMDQ_ERR_CERROR_ABT_IDX:
1024 dev_err(smmu->dev, "retrying command fetch\n");
1025 case CMDQ_ERR_CERROR_NONE_IDX:
1027 case CMDQ_ERR_CERROR_ATC_INV_IDX:
1029 * ATC Invalidation Completion timeout. CONS is still pointing
1030 * at the CMD_SYNC. Attempt to complete other pending commands
1031 * by repeating the CMD_SYNC, though we might well end up back
1032 * here since the ATC invalidation may still be pending.
1035 case CMDQ_ERR_CERROR_ILL_IDX:
1042 * We may have concurrent producers, so we need to be careful
1043 * not to touch any of the shadow cmdq state.
1045 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
1046 dev_err(smmu->dev, "skipping command in error state:\n");
1047 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
1048 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
1050 /* Convert the erroneous command into a CMD_SYNC */
1051 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
1052 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
1056 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
1060 * Command queue locking.
1061 * This is a form of bastardised rwlock with the following major changes:
1063 * - The only LOCK routines are exclusive_trylock() and shared_lock().
1064 * Neither have barrier semantics, and instead provide only a control
1067 * - The UNLOCK routines are supplemented with shared_tryunlock(), which
1068 * fails if the caller appears to be the last lock holder (yes, this is
1069 * racy). All successful UNLOCK routines have RELEASE semantics.
1071 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
1076 * We can try to avoid the cmpxchg() loop by simply incrementing the
1077 * lock counter. When held in exclusive state, the lock counter is set
1078 * to INT_MIN so these increments won't hurt as the value will remain
1081 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
1085 val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
1086 } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
1089 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
1091 (void)atomic_dec_return_release(&cmdq->lock);
1094 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
1096 if (atomic_read(&cmdq->lock) == 1)
1099 arm_smmu_cmdq_shared_unlock(cmdq);
1103 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags) \
1106 local_irq_save(flags); \
1107 __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \
1109 local_irq_restore(flags); \
1113 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \
1115 atomic_set_release(&cmdq->lock, 0); \
1116 local_irq_restore(flags); \
1121 * Command queue insertion.
1122 * This is made fiddly by our attempts to achieve some sort of scalability
1123 * since there is one queue shared amongst all of the CPUs in the system. If
1124 * you like mixed-size concurrency, dependency ordering and relaxed atomics,
1125 * then you'll *love* this monstrosity.
1127 * The basic idea is to split the queue up into ranges of commands that are
1128 * owned by a given CPU; the owner may not have written all of the commands
1129 * itself, but is responsible for advancing the hardware prod pointer when
1130 * the time comes. The algorithm is roughly:
1132 * 1. Allocate some space in the queue. At this point we also discover
1133 * whether the head of the queue is currently owned by another CPU,
1134 * or whether we are the owner.
1136 * 2. Write our commands into our allocated slots in the queue.
1138 * 3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
1140 * 4. If we are an owner:
1141 * a. Wait for the previous owner to finish.
1142 * b. Mark the queue head as unowned, which tells us the range
1143 * that we are responsible for publishing.
1144 * c. Wait for all commands in our owned range to become valid.
1145 * d. Advance the hardware prod pointer.
1146 * e. Tell the next owner we've finished.
1148 * 5. If we are inserting a CMD_SYNC (we may or may not have been an
1149 * owner), then we need to stick around until it has completed:
1150 * a. If we have MSIs, the SMMU can write back into the CMD_SYNC
1151 * to clear the first 4 bytes.
1152 * b. Otherwise, we spin waiting for the hardware cons pointer to
1153 * advance past our command.
1155 * The devil is in the details, particularly the use of locking for handling
1156 * SYNC completion and freeing up space in the queue before we think that it is
1159 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
1160 u32 sprod, u32 eprod, bool set)
1162 u32 swidx, sbidx, ewidx, ebidx;
1163 struct arm_smmu_ll_queue llq = {
1164 .max_n_shift = cmdq->q.llq.max_n_shift,
1168 ewidx = BIT_WORD(Q_IDX(&llq, eprod));
1169 ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
1171 while (llq.prod != eprod) {
1174 u32 limit = BITS_PER_LONG;
1176 swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
1177 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
1179 ptr = &cmdq->valid_map[swidx];
1181 if ((swidx == ewidx) && (sbidx < ebidx))
1184 mask = GENMASK(limit - 1, sbidx);
1187 * The valid bit is the inverse of the wrap bit. This means
1188 * that a zero-initialised queue is invalid and, after marking
1189 * all entries as valid, they become invalid again when we
1193 atomic_long_xor(mask, ptr);
1195 unsigned long valid;
1197 valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
1198 atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
1201 llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
1205 /* Mark all entries in the range [sprod, eprod) as valid */
1206 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
1207 u32 sprod, u32 eprod)
1209 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
1212 /* Wait for all entries in the range [sprod, eprod) to become valid */
1213 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
1214 u32 sprod, u32 eprod)
1216 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
1219 /* Wait for the command queue to become non-full */
1220 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
1221 struct arm_smmu_ll_queue *llq)
1223 unsigned long flags;
1224 struct arm_smmu_queue_poll qp;
1225 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1229 * Try to update our copy of cons by grabbing exclusive cmdq access. If
1230 * that fails, spin until somebody else updates it for us.
1232 if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
1233 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
1234 arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
1235 llq->val = READ_ONCE(cmdq->q.llq.val);
1239 queue_poll_init(smmu, &qp);
1241 llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
1242 if (!queue_full(llq))
1245 ret = queue_poll(&qp);
1252 * Wait until the SMMU signals a CMD_SYNC completion MSI.
1253 * Must be called with the cmdq lock held in some capacity.
1255 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
1256 struct arm_smmu_ll_queue *llq)
1259 struct arm_smmu_queue_poll qp;
1260 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1261 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
1263 queue_poll_init(smmu, &qp);
1266 * The MSI won't generate an event, since it's being written back
1267 * into the command queue.
1270 smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
1271 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
1276 * Wait until the SMMU cons index passes llq->prod.
1277 * Must be called with the cmdq lock held in some capacity.
1279 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
1280 struct arm_smmu_ll_queue *llq)
1282 struct arm_smmu_queue_poll qp;
1283 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1284 u32 prod = llq->prod;
1287 queue_poll_init(smmu, &qp);
1288 llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
1290 if (queue_consumed(llq, prod))
1293 ret = queue_poll(&qp);
1296 * This needs to be a readl() so that our subsequent call
1297 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
1299 * Specifically, we need to ensure that we observe all
1300 * shared_lock()s by other CMD_SYNCs that share our owner,
1301 * so that a failing call to tryunlock() means that we're
1302 * the last one out and therefore we can safely advance
1303 * cmdq->q.llq.cons. Roughly speaking:
1305 * CPU 0 CPU1 CPU2 (us)
1315 * <control dependency>
1321 * Requires us to see CPU 0's shared_lock() acquisition.
1323 llq->cons = readl(cmdq->q.cons_reg);
1329 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
1330 struct arm_smmu_ll_queue *llq)
1332 if (smmu->features & ARM_SMMU_FEAT_MSI &&
1333 smmu->features & ARM_SMMU_FEAT_COHERENCY)
1334 return __arm_smmu_cmdq_poll_until_msi(smmu, llq);
1336 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);
1339 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
1343 struct arm_smmu_ll_queue llq = {
1344 .max_n_shift = cmdq->q.llq.max_n_shift,
1348 for (i = 0; i < n; ++i) {
1349 u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
1351 prod = queue_inc_prod_n(&llq, i);
1352 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
1357 * This is the actual insertion function, and provides the following
1358 * ordering guarantees to callers:
1360 * - There is a dma_wmb() before publishing any commands to the queue.
1361 * This can be relied upon to order prior writes to data structures
1362 * in memory (such as a CD or an STE) before the command.
1364 * - On completion of a CMD_SYNC, there is a control dependency.
1365 * This can be relied upon to order subsequent writes to memory (e.g.
1366 * freeing an IOVA) after completion of the CMD_SYNC.
1368 * - Command insertion is totally ordered, so if two CPUs each race to
1369 * insert their own list of commands then all of the commands from one
1370 * CPU will appear before any of the commands from the other CPU.
1372 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
1373 u64 *cmds, int n, bool sync)
1375 u64 cmd_sync[CMDQ_ENT_DWORDS];
1377 unsigned long flags;
1379 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1380 struct arm_smmu_ll_queue llq = {
1381 .max_n_shift = cmdq->q.llq.max_n_shift,
1385 /* 1. Allocate some space in the queue */
1386 local_irq_save(flags);
1387 llq.val = READ_ONCE(cmdq->q.llq.val);
1391 while (!queue_has_space(&llq, n + sync)) {
1392 local_irq_restore(flags);
1393 if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))
1394 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
1395 local_irq_save(flags);
1398 head.cons = llq.cons;
1399 head.prod = queue_inc_prod_n(&llq, n + sync) |
1400 CMDQ_PROD_OWNED_FLAG;
1402 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
1408 owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
1409 head.prod &= ~CMDQ_PROD_OWNED_FLAG;
1410 llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
1413 * 2. Write our commands into the queue
1414 * Dependency ordering from the cmpxchg() loop above.
1416 arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
1418 prod = queue_inc_prod_n(&llq, n);
1419 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod);
1420 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
1423 * In order to determine completion of our CMD_SYNC, we must
1424 * ensure that the queue can't wrap twice without us noticing.
1425 * We achieve that by taking the cmdq lock as shared before
1426 * marking our slot as valid.
1428 arm_smmu_cmdq_shared_lock(cmdq);
1431 /* 3. Mark our slots as valid, ensuring commands are visible first */
1433 arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
1435 /* 4. If we are the owner, take control of the SMMU hardware */
1437 /* a. Wait for previous owner to finish */
1438 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
1440 /* b. Stop gathering work by clearing the owned flag */
1441 prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
1442 &cmdq->q.llq.atomic.prod);
1443 prod &= ~CMDQ_PROD_OWNED_FLAG;
1446 * c. Wait for any gathered work to be written to the queue.
1447 * Note that we read our own entries so that we have the control
1448 * dependency required by (d).
1450 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
1453 * d. Advance the hardware prod pointer
1454 * Control dependency ordering from the entries becoming valid.
1456 writel_relaxed(prod, cmdq->q.prod_reg);
1459 * e. Tell the next owner we're done
1460 * Make sure we've updated the hardware first, so that we don't
1461 * race to update prod and potentially move it backwards.
1463 atomic_set_release(&cmdq->owner_prod, prod);
1466 /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
1468 llq.prod = queue_inc_prod_n(&llq, n);
1469 ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);
1471 dev_err_ratelimited(smmu->dev,
1472 "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
1474 readl_relaxed(cmdq->q.prod_reg),
1475 readl_relaxed(cmdq->q.cons_reg));
1479 * Try to unlock the cmq lock. This will fail if we're the last
1480 * reader, in which case we can safely update cmdq->q.llq.cons
1482 if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
1483 WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
1484 arm_smmu_cmdq_shared_unlock(cmdq);
1488 local_irq_restore(flags);
1492 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
1493 struct arm_smmu_cmdq_ent *ent)
1495 u64 cmd[CMDQ_ENT_DWORDS];
1497 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
1498 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
1503 return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, false);
1506 static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1508 return arm_smmu_cmdq_issue_cmdlist(smmu, NULL, 0, true);
1511 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
1512 struct arm_smmu_cmdq_batch *cmds,
1513 struct arm_smmu_cmdq_ent *cmd)
1515 if (cmds->num == CMDQ_BATCH_ENTRIES) {
1516 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false);
1519 arm_smmu_cmdq_build_cmd(&cmds->cmds[cmds->num * CMDQ_ENT_DWORDS], cmd);
1523 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
1524 struct arm_smmu_cmdq_batch *cmds)
1526 return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);
1529 /* Context descriptor manipulation functions */
1530 static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
1531 int ssid, bool leaf)
1534 unsigned long flags;
1535 struct arm_smmu_master *master;
1536 struct arm_smmu_cmdq_batch cmds = {};
1537 struct arm_smmu_device *smmu = smmu_domain->smmu;
1538 struct arm_smmu_cmdq_ent cmd = {
1539 .opcode = CMDQ_OP_CFGI_CD,
1546 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1547 list_for_each_entry(master, &smmu_domain->devices, domain_head) {
1548 for (i = 0; i < master->num_sids; i++) {
1549 cmd.cfgi.sid = master->sids[i];
1550 arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
1553 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1555 arm_smmu_cmdq_batch_submit(smmu, &cmds);
1558 static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
1559 struct arm_smmu_l1_ctx_desc *l1_desc)
1561 size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1563 l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size,
1564 &l1_desc->l2ptr_dma, GFP_KERNEL);
1565 if (!l1_desc->l2ptr) {
1567 "failed to allocate context descriptor table\n");
1573 static void arm_smmu_write_cd_l1_desc(__le64 *dst,
1574 struct arm_smmu_l1_ctx_desc *l1_desc)
1576 u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) |
1579 /* See comment in arm_smmu_write_ctx_desc() */
1580 WRITE_ONCE(*dst, cpu_to_le64(val));
1583 static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain,
1588 struct arm_smmu_l1_ctx_desc *l1_desc;
1589 struct arm_smmu_device *smmu = smmu_domain->smmu;
1590 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1592 if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1593 return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
1595 idx = ssid >> CTXDESC_SPLIT;
1596 l1_desc = &cdcfg->l1_desc[idx];
1597 if (!l1_desc->l2ptr) {
1598 if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc))
1601 l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
1602 arm_smmu_write_cd_l1_desc(l1ptr, l1_desc);
1603 /* An invalid L1CD can be cached */
1604 arm_smmu_sync_cd(smmu_domain, ssid, false);
1606 idx = ssid & (CTXDESC_L2_ENTRIES - 1);
1607 return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
1610 static int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain,
1611 int ssid, struct arm_smmu_ctx_desc *cd)
1614 * This function handles the following cases:
1616 * (1) Install primary CD, for normal DMA traffic (SSID = 0).
1617 * (2) Install a secondary CD, for SID+SSID traffic.
1618 * (3) Update ASID of a CD. Atomically write the first 64 bits of the
1619 * CD, then invalidate the old entry and mappings.
1620 * (4) Remove a secondary CD.
1625 struct arm_smmu_device *smmu = smmu_domain->smmu;
1627 if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))
1630 cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid);
1634 val = le64_to_cpu(cdptr[0]);
1635 cd_live = !!(val & CTXDESC_CD_0_V);
1637 if (!cd) { /* (4) */
1639 } else if (cd_live) { /* (3) */
1640 val &= ~CTXDESC_CD_0_ASID;
1641 val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid);
1643 * Until CD+TLB invalidation, both ASIDs may be used for tagging
1644 * this substream's traffic
1646 } else { /* (1) and (2) */
1647 cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK);
1649 cdptr[3] = cpu_to_le64(cd->mair);
1652 * STE is live, and the SMMU might read dwords of this CD in any
1653 * order. Ensure that it observes valid values before reading
1656 arm_smmu_sync_cd(smmu_domain, ssid, true);
1662 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET |
1664 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |
1667 /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
1668 if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
1669 val |= CTXDESC_CD_0_S;
1673 * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3
1674 * "Configuration structures and configuration invalidation completion"
1676 * The size of single-copy atomic reads made by the SMMU is
1677 * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single
1678 * field within an aligned 64-bit span of a structure can be altered
1679 * without first making the structure invalid.
1681 WRITE_ONCE(cdptr[0], cpu_to_le64(val));
1682 arm_smmu_sync_cd(smmu_domain, ssid, true);
1686 static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
1690 size_t max_contexts;
1691 struct arm_smmu_device *smmu = smmu_domain->smmu;
1692 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1693 struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
1695 max_contexts = 1 << cfg->s1cdmax;
1697 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1698 max_contexts <= CTXDESC_L2_ENTRIES) {
1699 cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1700 cdcfg->num_l1_ents = max_contexts;
1702 l1size = max_contexts * (CTXDESC_CD_DWORDS << 3);
1704 cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1705 cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,
1706 CTXDESC_L2_ENTRIES);
1708 cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,
1709 sizeof(*cdcfg->l1_desc),
1711 if (!cdcfg->l1_desc)
1714 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1717 cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,
1719 if (!cdcfg->cdtab) {
1720 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1728 if (cdcfg->l1_desc) {
1729 devm_kfree(smmu->dev, cdcfg->l1_desc);
1730 cdcfg->l1_desc = NULL;
1735 static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
1738 size_t size, l1size;
1739 struct arm_smmu_device *smmu = smmu_domain->smmu;
1740 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1742 if (cdcfg->l1_desc) {
1743 size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1745 for (i = 0; i < cdcfg->num_l1_ents; i++) {
1746 if (!cdcfg->l1_desc[i].l2ptr)
1749 dmam_free_coherent(smmu->dev, size,
1750 cdcfg->l1_desc[i].l2ptr,
1751 cdcfg->l1_desc[i].l2ptr_dma);
1753 devm_kfree(smmu->dev, cdcfg->l1_desc);
1754 cdcfg->l1_desc = NULL;
1756 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1758 l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
1761 dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);
1762 cdcfg->cdtab_dma = 0;
1763 cdcfg->cdtab = NULL;
1766 /* Stream table manipulation functions */
1768 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1772 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1773 val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1775 /* See comment in arm_smmu_write_ctx_desc() */
1776 WRITE_ONCE(*dst, cpu_to_le64(val));
1779 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1781 struct arm_smmu_cmdq_ent cmd = {
1782 .opcode = CMDQ_OP_CFGI_STE,
1789 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1790 arm_smmu_cmdq_issue_sync(smmu);
1793 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1797 * This is hideously complicated, but we only really care about
1798 * three cases at the moment:
1800 * 1. Invalid (all zero) -> bypass/fault (init)
1801 * 2. Bypass/fault -> translation/bypass (attach)
1802 * 3. Translation/bypass -> bypass/fault (detach)
1804 * Given that we can't update the STE atomically and the SMMU
1805 * doesn't read the thing in a defined order, that leaves us
1806 * with the following maintenance requirements:
1808 * 1. Update Config, return (init time STEs aren't live)
1809 * 2. Write everything apart from dword 0, sync, write dword 0, sync
1810 * 3. Update Config, sync
1812 u64 val = le64_to_cpu(dst[0]);
1813 bool ste_live = false;
1814 struct arm_smmu_device *smmu = NULL;
1815 struct arm_smmu_s1_cfg *s1_cfg = NULL;
1816 struct arm_smmu_s2_cfg *s2_cfg = NULL;
1817 struct arm_smmu_domain *smmu_domain = NULL;
1818 struct arm_smmu_cmdq_ent prefetch_cmd = {
1819 .opcode = CMDQ_OP_PREFETCH_CFG,
1826 smmu_domain = master->domain;
1827 smmu = master->smmu;
1831 switch (smmu_domain->stage) {
1832 case ARM_SMMU_DOMAIN_S1:
1833 s1_cfg = &smmu_domain->s1_cfg;
1835 case ARM_SMMU_DOMAIN_S2:
1836 case ARM_SMMU_DOMAIN_NESTED:
1837 s2_cfg = &smmu_domain->s2_cfg;
1844 if (val & STRTAB_STE_0_V) {
1845 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1846 case STRTAB_STE_0_CFG_BYPASS:
1848 case STRTAB_STE_0_CFG_S1_TRANS:
1849 case STRTAB_STE_0_CFG_S2_TRANS:
1852 case STRTAB_STE_0_CFG_ABORT:
1853 BUG_ON(!disable_bypass);
1856 BUG(); /* STE corruption */
1860 /* Nuke the existing STE_0 value, as we're going to rewrite it */
1861 val = STRTAB_STE_0_V;
1864 if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1865 if (!smmu_domain && disable_bypass)
1866 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1868 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1870 dst[0] = cpu_to_le64(val);
1871 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1872 STRTAB_STE_1_SHCFG_INCOMING));
1873 dst[2] = 0; /* Nuke the VMID */
1875 * The SMMU can perform negative caching, so we must sync
1876 * the STE regardless of whether the old value was live.
1879 arm_smmu_sync_ste_for_sid(smmu, sid);
1885 dst[1] = cpu_to_le64(
1886 FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) |
1887 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1888 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1889 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1890 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1892 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1893 !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1894 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1896 val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1897 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
1898 FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) |
1899 FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt);
1904 dst[2] = cpu_to_le64(
1905 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1906 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1908 STRTAB_STE_2_S2ENDI |
1910 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1913 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1915 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1918 if (master->ats_enabled)
1919 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1920 STRTAB_STE_1_EATS_TRANS));
1922 arm_smmu_sync_ste_for_sid(smmu, sid);
1923 /* See comment in arm_smmu_write_ctx_desc() */
1924 WRITE_ONCE(dst[0], cpu_to_le64(val));
1925 arm_smmu_sync_ste_for_sid(smmu, sid);
1927 /* It's likely that we'll want to use the new STE soon */
1928 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1929 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1932 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1936 for (i = 0; i < nent; ++i) {
1937 arm_smmu_write_strtab_ent(NULL, -1, strtab);
1938 strtab += STRTAB_STE_DWORDS;
1942 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1946 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1947 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1952 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1953 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1955 desc->span = STRTAB_SPLIT + 1;
1956 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1960 "failed to allocate l2 stream table for SID %u\n",
1965 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1966 arm_smmu_write_strtab_l1_desc(strtab, desc);
1970 /* IRQ and event handlers */
1971 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1974 struct arm_smmu_device *smmu = dev;
1975 struct arm_smmu_queue *q = &smmu->evtq.q;
1976 struct arm_smmu_ll_queue *llq = &q->llq;
1977 u64 evt[EVTQ_ENT_DWORDS];
1980 while (!queue_remove_raw(q, evt)) {
1981 u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
1983 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1984 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1985 dev_info(smmu->dev, "\t0x%016llx\n",
1986 (unsigned long long)evt[i]);
1991 * Not much we can do on overflow, so scream and pretend we're
1994 if (queue_sync_prod_in(q) == -EOVERFLOW)
1995 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1996 } while (!queue_empty(llq));
1998 /* Sync our overflow flag, as we believe we're up to speed */
1999 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
2000 Q_IDX(llq, llq->cons);
2004 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
2010 sid = FIELD_GET(PRIQ_0_SID, evt[0]);
2011 ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
2012 ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0;
2013 last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
2014 grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
2016 dev_info(smmu->dev, "unexpected PRI request received:\n");
2018 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
2019 sid, ssid, grpid, last ? "L" : "",
2020 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
2021 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
2022 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
2023 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
2024 evt[1] & PRIQ_1_ADDR_MASK);
2027 struct arm_smmu_cmdq_ent cmd = {
2028 .opcode = CMDQ_OP_PRI_RESP,
2029 .substream_valid = ssv,
2034 .resp = PRI_RESP_DENY,
2038 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2042 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
2044 struct arm_smmu_device *smmu = dev;
2045 struct arm_smmu_queue *q = &smmu->priq.q;
2046 struct arm_smmu_ll_queue *llq = &q->llq;
2047 u64 evt[PRIQ_ENT_DWORDS];
2050 while (!queue_remove_raw(q, evt))
2051 arm_smmu_handle_ppr(smmu, evt);
2053 if (queue_sync_prod_in(q) == -EOVERFLOW)
2054 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
2055 } while (!queue_empty(llq));
2057 /* Sync our overflow flag, as we believe we're up to speed */
2058 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
2059 Q_IDX(llq, llq->cons);
2060 queue_sync_cons_out(q);
2064 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
2066 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
2068 u32 gerror, gerrorn, active;
2069 struct arm_smmu_device *smmu = dev;
2071 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
2072 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
2074 active = gerror ^ gerrorn;
2075 if (!(active & GERROR_ERR_MASK))
2076 return IRQ_NONE; /* No errors pending */
2079 "unexpected global error reported (0x%08x), this could be serious\n",
2082 if (active & GERROR_SFM_ERR) {
2083 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
2084 arm_smmu_device_disable(smmu);
2087 if (active & GERROR_MSI_GERROR_ABT_ERR)
2088 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
2090 if (active & GERROR_MSI_PRIQ_ABT_ERR)
2091 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
2093 if (active & GERROR_MSI_EVTQ_ABT_ERR)
2094 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
2096 if (active & GERROR_MSI_CMDQ_ABT_ERR)
2097 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
2099 if (active & GERROR_PRIQ_ABT_ERR)
2100 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
2102 if (active & GERROR_EVTQ_ABT_ERR)
2103 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
2105 if (active & GERROR_CMDQ_ERR)
2106 arm_smmu_cmdq_skip_err(smmu);
2108 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
2112 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
2114 struct arm_smmu_device *smmu = dev;
2116 arm_smmu_evtq_thread(irq, dev);
2117 if (smmu->features & ARM_SMMU_FEAT_PRI)
2118 arm_smmu_priq_thread(irq, dev);
2123 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
2125 arm_smmu_gerror_handler(irq, dev);
2126 return IRQ_WAKE_THREAD;
2130 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
2131 struct arm_smmu_cmdq_ent *cmd)
2135 /* ATC invalidates are always on 4096-bytes pages */
2136 size_t inval_grain_shift = 12;
2137 unsigned long page_start, page_end;
2139 *cmd = (struct arm_smmu_cmdq_ent) {
2140 .opcode = CMDQ_OP_ATC_INV,
2141 .substream_valid = !!ssid,
2146 cmd->atc.size = ATC_INV_SIZE_ALL;
2150 page_start = iova >> inval_grain_shift;
2151 page_end = (iova + size - 1) >> inval_grain_shift;
2154 * In an ATS Invalidate Request, the address must be aligned on the
2155 * range size, which must be a power of two number of page sizes. We
2156 * thus have to choose between grossly over-invalidating the region, or
2157 * splitting the invalidation into multiple commands. For simplicity
2158 * we'll go with the first solution, but should refine it in the future
2159 * if multiple commands are shown to be more efficient.
2161 * Find the smallest power of two that covers the range. The most
2162 * significant differing bit between the start and end addresses,
2163 * fls(start ^ end), indicates the required span. For example:
2165 * We want to invalidate pages [8; 11]. This is already the ideal range:
2166 * x = 0b1000 ^ 0b1011 = 0b11
2167 * span = 1 << fls(x) = 4
2169 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
2170 * x = 0b0111 ^ 0b1010 = 0b1101
2171 * span = 1 << fls(x) = 16
2173 log2_span = fls_long(page_start ^ page_end);
2174 span_mask = (1ULL << log2_span) - 1;
2176 page_start &= ~span_mask;
2178 cmd->atc.addr = page_start << inval_grain_shift;
2179 cmd->atc.size = log2_span;
2182 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
2185 struct arm_smmu_cmdq_ent cmd;
2187 arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd);
2189 for (i = 0; i < master->num_sids; i++) {
2190 cmd.atc.sid = master->sids[i];
2191 arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
2194 return arm_smmu_cmdq_issue_sync(master->smmu);
2197 static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
2198 int ssid, unsigned long iova, size_t size)
2201 unsigned long flags;
2202 struct arm_smmu_cmdq_ent cmd;
2203 struct arm_smmu_master *master;
2204 struct arm_smmu_cmdq_batch cmds = {};
2206 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
2210 * Ensure that we've completed prior invalidation of the main TLBs
2211 * before we read 'nr_ats_masters' in case of a concurrent call to
2212 * arm_smmu_enable_ats():
2214 * // unmap() // arm_smmu_enable_ats()
2215 * TLBI+SYNC atomic_inc(&nr_ats_masters);
2217 * atomic_read(&nr_ats_masters); pci_enable_ats() // writel()
2219 * Ensures that we always see the incremented 'nr_ats_masters' count if
2220 * ATS was enabled at the PCI device before completion of the TLBI.
2223 if (!atomic_read(&smmu_domain->nr_ats_masters))
2226 arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
2228 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2229 list_for_each_entry(master, &smmu_domain->devices, domain_head) {
2230 if (!master->ats_enabled)
2233 for (i = 0; i < master->num_sids; i++) {
2234 cmd.atc.sid = master->sids[i];
2235 arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
2238 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2240 return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
2243 /* IO_PGTABLE API */
2244 static void arm_smmu_tlb_inv_context(void *cookie)
2246 struct arm_smmu_domain *smmu_domain = cookie;
2247 struct arm_smmu_device *smmu = smmu_domain->smmu;
2248 struct arm_smmu_cmdq_ent cmd;
2250 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2251 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
2252 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
2255 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
2256 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2260 * NOTE: when io-pgtable is in non-strict mode, we may get here with
2261 * PTEs previously cleared by unmaps on the current CPU not yet visible
2262 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
2263 * insertion to guarantee those are observed before the TLBI. Do be
2266 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2267 arm_smmu_cmdq_issue_sync(smmu);
2268 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
2271 static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
2272 size_t granule, bool leaf,
2273 struct arm_smmu_domain *smmu_domain)
2275 struct arm_smmu_device *smmu = smmu_domain->smmu;
2276 unsigned long start = iova, end = iova + size, num_pages = 0, tg = 0;
2277 size_t inv_range = granule;
2278 struct arm_smmu_cmdq_batch cmds = {};
2279 struct arm_smmu_cmdq_ent cmd = {
2288 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2289 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
2290 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
2292 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
2293 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2296 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2297 /* Get the leaf page size */
2298 tg = __ffs(smmu_domain->domain.pgsize_bitmap);
2300 /* Convert page size of 12,14,16 (log2) to 1,2,3 */
2301 cmd.tlbi.tg = (tg - 10) / 2;
2303 /* Determine what level the granule is at */
2304 cmd.tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
2306 num_pages = size >> tg;
2309 while (iova < end) {
2310 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2312 * On each iteration of the loop, the range is 5 bits
2313 * worth of the aligned size remaining.
2314 * The range in pages is:
2316 * range = (num_pages & (0x1f << __ffs(num_pages)))
2318 unsigned long scale, num;
2320 /* Determine the power of 2 multiple number of pages */
2321 scale = __ffs(num_pages);
2322 cmd.tlbi.scale = scale;
2324 /* Determine how many chunks of 2^scale size we have */
2325 num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
2326 cmd.tlbi.num = num - 1;
2328 /* range is num * 2^scale * pgsize */
2329 inv_range = num << (scale + tg);
2331 /* Clear out the lower order bits for the next iteration */
2332 num_pages -= num << scale;
2335 cmd.tlbi.addr = iova;
2336 arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
2339 arm_smmu_cmdq_batch_submit(smmu, &cmds);
2342 * Unfortunately, this can't be leaf-only since we may have
2343 * zapped an entire table.
2345 arm_smmu_atc_inv_domain(smmu_domain, 0, start, size);
2348 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
2349 unsigned long iova, size_t granule,
2352 struct arm_smmu_domain *smmu_domain = cookie;
2353 struct iommu_domain *domain = &smmu_domain->domain;
2355 iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2358 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2359 size_t granule, void *cookie)
2361 arm_smmu_tlb_inv_range(iova, size, granule, false, cookie);
2364 static void arm_smmu_tlb_inv_leaf(unsigned long iova, size_t size,
2365 size_t granule, void *cookie)
2367 arm_smmu_tlb_inv_range(iova, size, granule, true, cookie);
2370 static const struct iommu_flush_ops arm_smmu_flush_ops = {
2371 .tlb_flush_all = arm_smmu_tlb_inv_context,
2372 .tlb_flush_walk = arm_smmu_tlb_inv_walk,
2373 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf,
2374 .tlb_add_page = arm_smmu_tlb_inv_page_nosync,
2378 static bool arm_smmu_capable(enum iommu_cap cap)
2381 case IOMMU_CAP_CACHE_COHERENCY:
2383 case IOMMU_CAP_NOEXEC:
2390 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
2392 struct arm_smmu_domain *smmu_domain;
2394 if (type != IOMMU_DOMAIN_UNMANAGED &&
2395 type != IOMMU_DOMAIN_DMA &&
2396 type != IOMMU_DOMAIN_IDENTITY)
2400 * Allocate the domain and initialise some of its data structures.
2401 * We can't really do anything meaningful until we've added a
2404 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2408 if (type == IOMMU_DOMAIN_DMA &&
2409 iommu_get_dma_cookie(&smmu_domain->domain)) {
2414 mutex_init(&smmu_domain->init_mutex);
2415 INIT_LIST_HEAD(&smmu_domain->devices);
2416 spin_lock_init(&smmu_domain->devices_lock);
2418 return &smmu_domain->domain;
2421 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
2423 int idx, size = 1 << span;
2426 idx = find_first_zero_bit(map, size);
2429 } while (test_and_set_bit(idx, map));
2434 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
2436 clear_bit(idx, map);
2439 static void arm_smmu_domain_free(struct iommu_domain *domain)
2441 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2442 struct arm_smmu_device *smmu = smmu_domain->smmu;
2444 iommu_put_dma_cookie(domain);
2445 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2447 /* Free the CD and ASID, if we allocated them */
2448 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2449 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2451 if (cfg->cdcfg.cdtab) {
2452 arm_smmu_free_cd_tables(smmu_domain);
2453 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
2456 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2458 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
2464 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
2465 struct arm_smmu_master *master,
2466 struct io_pgtable_cfg *pgtbl_cfg)
2470 struct arm_smmu_device *smmu = smmu_domain->smmu;
2471 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2472 typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2474 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
2478 cfg->s1cdmax = master->ssid_bits;
2480 ret = arm_smmu_alloc_cd_tables(smmu_domain);
2484 cfg->cd.asid = (u16)asid;
2485 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
2486 cfg->cd.tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
2487 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
2488 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
2489 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
2490 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
2491 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
2492 CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
2493 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair;
2496 * Note that this will end up calling arm_smmu_sync_cd() before
2497 * the master has been added to the devices list for this domain.
2498 * This isn't an issue because the STE hasn't been installed yet.
2500 ret = arm_smmu_write_ctx_desc(smmu_domain, 0, &cfg->cd);
2502 goto out_free_cd_tables;
2507 arm_smmu_free_cd_tables(smmu_domain);
2509 arm_smmu_bitmap_free(smmu->asid_map, asid);
2513 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
2514 struct arm_smmu_master *master,
2515 struct io_pgtable_cfg *pgtbl_cfg)
2518 struct arm_smmu_device *smmu = smmu_domain->smmu;
2519 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2520 typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr;
2522 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
2526 vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
2527 cfg->vmid = (u16)vmid;
2528 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
2529 cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
2530 FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
2531 FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
2532 FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
2533 FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
2534 FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
2535 FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
2539 static int arm_smmu_domain_finalise(struct iommu_domain *domain,
2540 struct arm_smmu_master *master)
2543 unsigned long ias, oas;
2544 enum io_pgtable_fmt fmt;
2545 struct io_pgtable_cfg pgtbl_cfg;
2546 struct io_pgtable_ops *pgtbl_ops;
2547 int (*finalise_stage_fn)(struct arm_smmu_domain *,
2548 struct arm_smmu_master *,
2549 struct io_pgtable_cfg *);
2550 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2551 struct arm_smmu_device *smmu = smmu_domain->smmu;
2553 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
2554 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
2558 /* Restrict the stage to what we can actually support */
2559 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
2560 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
2561 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
2562 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2564 switch (smmu_domain->stage) {
2565 case ARM_SMMU_DOMAIN_S1:
2566 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
2567 ias = min_t(unsigned long, ias, VA_BITS);
2569 fmt = ARM_64_LPAE_S1;
2570 finalise_stage_fn = arm_smmu_domain_finalise_s1;
2572 case ARM_SMMU_DOMAIN_NESTED:
2573 case ARM_SMMU_DOMAIN_S2:
2576 fmt = ARM_64_LPAE_S2;
2577 finalise_stage_fn = arm_smmu_domain_finalise_s2;
2583 pgtbl_cfg = (struct io_pgtable_cfg) {
2584 .pgsize_bitmap = smmu->pgsize_bitmap,
2587 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY,
2588 .tlb = &arm_smmu_flush_ops,
2589 .iommu_dev = smmu->dev,
2592 if (smmu_domain->non_strict)
2593 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
2595 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2599 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2600 domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2601 domain->geometry.force_aperture = true;
2603 ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg);
2605 free_io_pgtable_ops(pgtbl_ops);
2609 smmu_domain->pgtbl_ops = pgtbl_ops;
2613 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2616 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2618 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2619 struct arm_smmu_strtab_l1_desc *l1_desc;
2622 /* Two-level walk */
2623 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
2624 l1_desc = &cfg->l1_desc[idx];
2625 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
2626 step = &l1_desc->l2ptr[idx];
2628 /* Simple linear lookup */
2629 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
2635 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
2638 struct arm_smmu_device *smmu = master->smmu;
2640 for (i = 0; i < master->num_sids; ++i) {
2641 u32 sid = master->sids[i];
2642 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
2644 /* Bridged PCI devices may end up with duplicated IDs */
2645 for (j = 0; j < i; j++)
2646 if (master->sids[j] == sid)
2651 arm_smmu_write_strtab_ent(master, sid, step);
2655 #ifdef CONFIG_PCI_ATS
2656 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2658 struct pci_dev *pdev;
2659 struct arm_smmu_device *smmu = master->smmu;
2660 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2662 if (!(smmu->features & ARM_SMMU_FEAT_ATS) || !dev_is_pci(master->dev) ||
2663 !(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS) || pci_ats_disabled())
2666 pdev = to_pci_dev(master->dev);
2667 return !pdev->untrusted && pdev->ats_cap;
2670 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2676 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2679 struct pci_dev *pdev;
2680 struct arm_smmu_device *smmu = master->smmu;
2681 struct arm_smmu_domain *smmu_domain = master->domain;
2683 /* Don't enable ATS at the endpoint if it's not enabled in the STE */
2684 if (!master->ats_enabled)
2687 /* Smallest Translation Unit: log2 of the smallest supported granule */
2688 stu = __ffs(smmu->pgsize_bitmap);
2689 pdev = to_pci_dev(master->dev);
2691 atomic_inc(&smmu_domain->nr_ats_masters);
2692 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
2693 if (pci_enable_ats(pdev, stu))
2694 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2697 static void arm_smmu_disable_ats(struct arm_smmu_master *master)
2699 struct arm_smmu_domain *smmu_domain = master->domain;
2701 if (!master->ats_enabled)
2704 pci_disable_ats(to_pci_dev(master->dev));
2706 * Ensure ATS is disabled at the endpoint before we issue the
2707 * ATC invalidation via the SMMU.
2710 arm_smmu_atc_inv_master(master);
2711 atomic_dec(&smmu_domain->nr_ats_masters);
2714 static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
2719 struct pci_dev *pdev;
2721 if (!dev_is_pci(master->dev))
2724 pdev = to_pci_dev(master->dev);
2726 features = pci_pasid_features(pdev);
2730 num_pasids = pci_max_pasids(pdev);
2731 if (num_pasids <= 0)
2734 ret = pci_enable_pasid(pdev, features);
2736 dev_err(&pdev->dev, "Failed to enable PASID\n");
2740 master->ssid_bits = min_t(u8, ilog2(num_pasids),
2741 master->smmu->ssid_bits);
2745 static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
2747 struct pci_dev *pdev;
2749 if (!dev_is_pci(master->dev))
2752 pdev = to_pci_dev(master->dev);
2754 if (!pdev->pasid_enabled)
2757 master->ssid_bits = 0;
2758 pci_disable_pasid(pdev);
2761 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
2763 unsigned long flags;
2764 struct arm_smmu_domain *smmu_domain = master->domain;
2769 arm_smmu_disable_ats(master);
2771 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2772 list_del(&master->domain_head);
2773 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2775 master->domain = NULL;
2776 master->ats_enabled = false;
2777 arm_smmu_install_ste_for_dev(master);
2780 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
2783 unsigned long flags;
2784 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2785 struct arm_smmu_device *smmu;
2786 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2787 struct arm_smmu_master *master;
2792 master = dev_iommu_priv_get(dev);
2793 smmu = master->smmu;
2795 arm_smmu_detach_dev(master);
2797 mutex_lock(&smmu_domain->init_mutex);
2799 if (!smmu_domain->smmu) {
2800 smmu_domain->smmu = smmu;
2801 ret = arm_smmu_domain_finalise(domain, master);
2803 smmu_domain->smmu = NULL;
2806 } else if (smmu_domain->smmu != smmu) {
2808 "cannot attach to SMMU %s (upstream of %s)\n",
2809 dev_name(smmu_domain->smmu->dev),
2810 dev_name(smmu->dev));
2813 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
2814 master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
2816 "cannot attach to incompatible domain (%u SSID bits != %u)\n",
2817 smmu_domain->s1_cfg.s1cdmax, master->ssid_bits);
2822 master->domain = smmu_domain;
2824 if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
2825 master->ats_enabled = arm_smmu_ats_supported(master);
2827 arm_smmu_install_ste_for_dev(master);
2829 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2830 list_add(&master->domain_head, &smmu_domain->devices);
2831 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2833 arm_smmu_enable_ats(master);
2836 mutex_unlock(&smmu_domain->init_mutex);
2840 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
2841 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2843 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2848 return ops->map(ops, iova, paddr, size, prot);
2851 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
2852 size_t size, struct iommu_iotlb_gather *gather)
2854 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2855 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
2860 return ops->unmap(ops, iova, size, gather);
2863 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
2865 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2867 if (smmu_domain->smmu)
2868 arm_smmu_tlb_inv_context(smmu_domain);
2871 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
2872 struct iommu_iotlb_gather *gather)
2874 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2876 arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start,
2877 gather->pgsize, true, smmu_domain);
2881 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2883 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2885 if (domain->type == IOMMU_DOMAIN_IDENTITY)
2891 return ops->iova_to_phys(ops, iova);
2894 static struct platform_driver arm_smmu_driver;
2897 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
2899 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
2902 return dev ? dev_get_drvdata(dev) : NULL;
2905 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
2907 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2909 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2910 limit *= 1UL << STRTAB_SPLIT;
2915 static struct iommu_ops arm_smmu_ops;
2917 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
2920 struct arm_smmu_device *smmu;
2921 struct arm_smmu_master *master;
2922 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2924 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2925 return ERR_PTR(-ENODEV);
2927 if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
2928 return ERR_PTR(-EBUSY);
2930 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2932 return ERR_PTR(-ENODEV);
2934 master = kzalloc(sizeof(*master), GFP_KERNEL);
2936 return ERR_PTR(-ENOMEM);
2939 master->smmu = smmu;
2940 master->sids = fwspec->ids;
2941 master->num_sids = fwspec->num_ids;
2942 dev_iommu_priv_set(dev, master);
2944 /* Check the SIDs are in range of the SMMU and our stream table */
2945 for (i = 0; i < master->num_sids; i++) {
2946 u32 sid = master->sids[i];
2948 if (!arm_smmu_sid_in_range(smmu, sid)) {
2950 goto err_free_master;
2953 /* Ensure l2 strtab is initialised */
2954 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2955 ret = arm_smmu_init_l2_strtab(smmu, sid);
2957 goto err_free_master;
2961 master->ssid_bits = min(smmu->ssid_bits, fwspec->num_pasid_bits);
2964 * Note that PASID must be enabled before, and disabled after ATS:
2965 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
2967 * Behavior is undefined if this bit is Set and the value of the PASID
2968 * Enable, Execute Requested Enable, or Privileged Mode Requested bits
2971 arm_smmu_enable_pasid(master);
2973 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
2974 master->ssid_bits = min_t(u8, master->ssid_bits,
2975 CTXDESC_LINEAR_CDMAX);
2977 return &smmu->iommu;
2981 dev_iommu_priv_set(dev, NULL);
2982 return ERR_PTR(ret);
2985 static void arm_smmu_release_device(struct device *dev)
2987 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2988 struct arm_smmu_master *master;
2989 struct arm_smmu_device *smmu;
2991 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2994 master = dev_iommu_priv_get(dev);
2995 smmu = master->smmu;
2996 arm_smmu_detach_dev(master);
2997 arm_smmu_disable_pasid(master);
2999 iommu_fwspec_free(dev);
3002 static struct iommu_group *arm_smmu_device_group(struct device *dev)
3004 struct iommu_group *group;
3007 * We don't support devices sharing stream IDs other than PCI RID
3008 * aliases, since the necessary ID-to-device lookup becomes rather
3009 * impractical given a potential sparse 32-bit stream ID space.
3011 if (dev_is_pci(dev))
3012 group = pci_device_group(dev);
3014 group = generic_device_group(dev);
3019 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
3020 enum iommu_attr attr, void *data)
3022 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3024 switch (domain->type) {
3025 case IOMMU_DOMAIN_UNMANAGED:
3027 case DOMAIN_ATTR_NESTING:
3028 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
3034 case IOMMU_DOMAIN_DMA:
3036 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
3037 *(int *)data = smmu_domain->non_strict;
3048 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
3049 enum iommu_attr attr, void *data)
3052 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3054 mutex_lock(&smmu_domain->init_mutex);
3056 switch (domain->type) {
3057 case IOMMU_DOMAIN_UNMANAGED:
3059 case DOMAIN_ATTR_NESTING:
3060 if (smmu_domain->smmu) {
3066 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
3068 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
3074 case IOMMU_DOMAIN_DMA:
3076 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
3077 smmu_domain->non_strict = *(int *)data;
3088 mutex_unlock(&smmu_domain->init_mutex);
3092 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
3094 return iommu_fwspec_add_ids(dev, args->args, 1);
3097 static void arm_smmu_get_resv_regions(struct device *dev,
3098 struct list_head *head)
3100 struct iommu_resv_region *region;
3101 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
3103 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
3104 prot, IOMMU_RESV_SW_MSI);
3108 list_add_tail(®ion->list, head);
3110 iommu_dma_get_resv_regions(dev, head);
3113 static struct iommu_ops arm_smmu_ops = {
3114 .capable = arm_smmu_capable,
3115 .domain_alloc = arm_smmu_domain_alloc,
3116 .domain_free = arm_smmu_domain_free,
3117 .attach_dev = arm_smmu_attach_dev,
3118 .map = arm_smmu_map,
3119 .unmap = arm_smmu_unmap,
3120 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
3121 .iotlb_sync = arm_smmu_iotlb_sync,
3122 .iova_to_phys = arm_smmu_iova_to_phys,
3123 .probe_device = arm_smmu_probe_device,
3124 .release_device = arm_smmu_release_device,
3125 .device_group = arm_smmu_device_group,
3126 .domain_get_attr = arm_smmu_domain_get_attr,
3127 .domain_set_attr = arm_smmu_domain_set_attr,
3128 .of_xlate = arm_smmu_of_xlate,
3129 .get_resv_regions = arm_smmu_get_resv_regions,
3130 .put_resv_regions = generic_iommu_put_resv_regions,
3131 .pgsize_bitmap = -1UL, /* Restricted during device attach */
3134 /* Probing and initialisation functions */
3135 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
3136 struct arm_smmu_queue *q,
3137 unsigned long prod_off,
3138 unsigned long cons_off,
3139 size_t dwords, const char *name)
3144 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
3145 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
3147 if (q->base || qsz < PAGE_SIZE)
3150 q->llq.max_n_shift--;
3155 "failed to allocate queue (0x%zx bytes) for %s\n",
3160 if (!WARN_ON(q->base_dma & (qsz - 1))) {
3161 dev_info(smmu->dev, "allocated %u entries for %s\n",
3162 1 << q->llq.max_n_shift, name);
3165 q->prod_reg = arm_smmu_page1_fixup(prod_off, smmu);
3166 q->cons_reg = arm_smmu_page1_fixup(cons_off, smmu);
3167 q->ent_dwords = dwords;
3169 q->q_base = Q_BASE_RWA;
3170 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
3171 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
3173 q->llq.prod = q->llq.cons = 0;
3177 static void arm_smmu_cmdq_free_bitmap(void *data)
3179 unsigned long *bitmap = data;
3180 bitmap_free(bitmap);
3183 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)
3186 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
3187 unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
3188 atomic_long_t *bitmap;
3190 atomic_set(&cmdq->owner_prod, 0);
3191 atomic_set(&cmdq->lock, 0);
3193 bitmap = (atomic_long_t *)bitmap_zalloc(nents, GFP_KERNEL);
3195 dev_err(smmu->dev, "failed to allocate cmdq bitmap\n");
3198 cmdq->valid_map = bitmap;
3199 devm_add_action(smmu->dev, arm_smmu_cmdq_free_bitmap, bitmap);
3205 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
3210 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
3211 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS,
3216 ret = arm_smmu_cmdq_init(smmu);
3221 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
3222 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS,
3228 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
3231 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
3232 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS,
3236 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
3239 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3240 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
3241 void *strtab = smmu->strtab_cfg.strtab;
3243 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
3244 if (!cfg->l1_desc) {
3245 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
3249 for (i = 0; i < cfg->num_l1_ents; ++i) {
3250 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
3251 strtab += STRTAB_L1_DESC_DWORDS << 3;
3257 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
3262 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3264 /* Calculate the L1 size, capped to the SIDSIZE. */
3265 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
3266 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
3267 cfg->num_l1_ents = 1 << size;
3269 size += STRTAB_SPLIT;
3270 if (size < smmu->sid_bits)
3272 "2-level strtab only covers %u/%u bits of SID\n",
3273 size, smmu->sid_bits);
3275 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
3276 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
3280 "failed to allocate l1 stream table (%u bytes)\n",
3284 cfg->strtab = strtab;
3286 /* Configure strtab_base_cfg for 2 levels */
3287 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL);
3288 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size);
3289 reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
3290 cfg->strtab_base_cfg = reg;
3292 return arm_smmu_init_l1_strtab(smmu);
3295 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
3300 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3302 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
3303 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
3307 "failed to allocate linear stream table (%u bytes)\n",
3311 cfg->strtab = strtab;
3312 cfg->num_l1_ents = 1 << smmu->sid_bits;
3314 /* Configure strtab_base_cfg for a linear table covering all SIDs */
3315 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR);
3316 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
3317 cfg->strtab_base_cfg = reg;
3319 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
3323 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
3328 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3329 ret = arm_smmu_init_strtab_2lvl(smmu);
3331 ret = arm_smmu_init_strtab_linear(smmu);
3336 /* Set the strtab base address */
3337 reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
3338 reg |= STRTAB_BASE_RA;
3339 smmu->strtab_cfg.strtab_base = reg;
3341 /* Allocate the first VMID for stage-2 bypass STEs */
3342 set_bit(0, smmu->vmid_map);
3346 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
3350 ret = arm_smmu_init_queues(smmu);
3354 return arm_smmu_init_strtab(smmu);
3357 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
3358 unsigned int reg_off, unsigned int ack_off)
3362 writel_relaxed(val, smmu->base + reg_off);
3363 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3364 1, ARM_SMMU_POLL_TIMEOUT_US);
3367 /* GBPA is "special" */
3368 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
3371 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3373 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3374 1, ARM_SMMU_POLL_TIMEOUT_US);
3380 writel_relaxed(reg | GBPA_UPDATE, gbpa);
3381 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3382 1, ARM_SMMU_POLL_TIMEOUT_US);
3385 dev_err(smmu->dev, "GBPA not responding to update\n");
3389 static void arm_smmu_free_msis(void *data)
3391 struct device *dev = data;
3392 platform_msi_domain_free_irqs(dev);
3395 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
3397 phys_addr_t doorbell;
3398 struct device *dev = msi_desc_to_dev(desc);
3399 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
3400 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
3402 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3403 doorbell &= MSI_CFG0_ADDR_MASK;
3405 writeq_relaxed(doorbell, smmu->base + cfg[0]);
3406 writel_relaxed(msg->data, smmu->base + cfg[1]);
3407 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3410 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
3412 struct msi_desc *desc;
3413 int ret, nvec = ARM_SMMU_MAX_MSIS;
3414 struct device *dev = smmu->dev;
3416 /* Clear the MSI address regs */
3417 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3418 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3420 if (smmu->features & ARM_SMMU_FEAT_PRI)
3421 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3425 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3428 if (!dev->msi_domain) {
3429 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3433 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
3434 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
3436 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3440 for_each_msi_entry(desc, dev) {
3441 switch (desc->platform.msi_index) {
3442 case EVTQ_MSI_INDEX:
3443 smmu->evtq.q.irq = desc->irq;
3445 case GERROR_MSI_INDEX:
3446 smmu->gerr_irq = desc->irq;
3448 case PRIQ_MSI_INDEX:
3449 smmu->priq.q.irq = desc->irq;
3451 default: /* Unknown */
3456 /* Add callback to free MSIs on teardown */
3457 devm_add_action(dev, arm_smmu_free_msis, dev);
3460 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
3464 arm_smmu_setup_msis(smmu);
3466 /* Request interrupt lines */
3467 irq = smmu->evtq.q.irq;
3469 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3470 arm_smmu_evtq_thread,
3472 "arm-smmu-v3-evtq", smmu);
3474 dev_warn(smmu->dev, "failed to enable evtq irq\n");
3476 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3479 irq = smmu->gerr_irq;
3481 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3482 0, "arm-smmu-v3-gerror", smmu);
3484 dev_warn(smmu->dev, "failed to enable gerror irq\n");
3486 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
3489 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3490 irq = smmu->priq.q.irq;
3492 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3493 arm_smmu_priq_thread,
3499 "failed to enable priq irq\n");
3501 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
3506 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
3509 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
3511 /* Disable IRQs first */
3512 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
3513 ARM_SMMU_IRQ_CTRLACK);
3515 dev_err(smmu->dev, "failed to disable irqs\n");
3519 irq = smmu->combined_irq;
3522 * Cavium ThunderX2 implementation doesn't support unique irq
3523 * lines. Use a single irq line for all the SMMUv3 interrupts.
3525 ret = devm_request_threaded_irq(smmu->dev, irq,
3526 arm_smmu_combined_irq_handler,
3527 arm_smmu_combined_irq_thread,
3529 "arm-smmu-v3-combined-irq", smmu);
3531 dev_warn(smmu->dev, "failed to enable combined irq\n");
3533 arm_smmu_setup_unique_irqs(smmu);
3535 if (smmu->features & ARM_SMMU_FEAT_PRI)
3536 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
3538 /* Enable interrupt generation on the SMMU */
3539 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
3540 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
3542 dev_warn(smmu->dev, "failed to enable irqs\n");
3547 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
3551 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
3553 dev_err(smmu->dev, "failed to clear cr0\n");
3558 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
3562 struct arm_smmu_cmdq_ent cmd;
3564 /* Clear CR0 and sync (disables SMMU and queue processing) */
3565 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
3566 if (reg & CR0_SMMUEN) {
3567 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
3568 WARN_ON(is_kdump_kernel() && !disable_bypass);
3569 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
3572 ret = arm_smmu_device_disable(smmu);
3576 /* CR1 (table and queue memory attributes) */
3577 reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
3578 FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
3579 FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
3580 FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
3581 FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
3582 FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
3583 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
3585 /* CR2 (random crap) */
3586 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
3587 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
3590 writeq_relaxed(smmu->strtab_cfg.strtab_base,
3591 smmu->base + ARM_SMMU_STRTAB_BASE);
3592 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
3593 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
3596 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
3597 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
3598 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
3600 enables = CR0_CMDQEN;
3601 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3604 dev_err(smmu->dev, "failed to enable command queue\n");
3608 /* Invalidate any cached configuration */
3609 cmd.opcode = CMDQ_OP_CFGI_ALL;
3610 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3611 arm_smmu_cmdq_issue_sync(smmu);
3613 /* Invalidate any stale TLB entries */
3614 if (smmu->features & ARM_SMMU_FEAT_HYP) {
3615 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
3616 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3619 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
3620 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3621 arm_smmu_cmdq_issue_sync(smmu);
3624 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
3625 writel_relaxed(smmu->evtq.q.llq.prod,
3626 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD, smmu));
3627 writel_relaxed(smmu->evtq.q.llq.cons,
3628 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS, smmu));
3630 enables |= CR0_EVTQEN;
3631 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3634 dev_err(smmu->dev, "failed to enable event queue\n");
3639 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3640 writeq_relaxed(smmu->priq.q.q_base,
3641 smmu->base + ARM_SMMU_PRIQ_BASE);
3642 writel_relaxed(smmu->priq.q.llq.prod,
3643 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD, smmu));
3644 writel_relaxed(smmu->priq.q.llq.cons,
3645 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS, smmu));
3647 enables |= CR0_PRIQEN;
3648 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3651 dev_err(smmu->dev, "failed to enable PRI queue\n");
3656 if (smmu->features & ARM_SMMU_FEAT_ATS) {
3657 enables |= CR0_ATSCHK;
3658 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3661 dev_err(smmu->dev, "failed to enable ATS check\n");
3666 ret = arm_smmu_setup_irqs(smmu);
3668 dev_err(smmu->dev, "failed to setup irqs\n");
3672 if (is_kdump_kernel())
3673 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
3675 /* Enable the SMMU interface, or ensure bypass */
3676 if (!bypass || disable_bypass) {
3677 enables |= CR0_SMMUEN;
3679 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
3683 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3686 dev_err(smmu->dev, "failed to enable SMMU interface\n");
3693 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
3696 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
3699 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
3701 /* 2-level structures */
3702 if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
3703 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
3705 if (reg & IDR0_CD2L)
3706 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
3709 * Translation table endianness.
3710 * We currently require the same endianness as the CPU, but this
3711 * could be changed later by adding a new IO_PGTABLE_QUIRK.
3713 switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
3714 case IDR0_TTENDIAN_MIXED:
3715 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
3718 case IDR0_TTENDIAN_BE:
3719 smmu->features |= ARM_SMMU_FEAT_TT_BE;
3722 case IDR0_TTENDIAN_LE:
3723 smmu->features |= ARM_SMMU_FEAT_TT_LE;
3727 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
3731 /* Boolean feature flags */
3732 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
3733 smmu->features |= ARM_SMMU_FEAT_PRI;
3735 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
3736 smmu->features |= ARM_SMMU_FEAT_ATS;
3739 smmu->features |= ARM_SMMU_FEAT_SEV;
3742 smmu->features |= ARM_SMMU_FEAT_MSI;
3745 smmu->features |= ARM_SMMU_FEAT_HYP;
3748 * The coherency feature as set by FW is used in preference to the ID
3749 * register, but warn on mismatch.
3751 if (!!(reg & IDR0_COHACC) != coherent)
3752 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
3753 coherent ? "true" : "false");
3755 switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
3756 case IDR0_STALL_MODEL_FORCE:
3757 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
3759 case IDR0_STALL_MODEL_STALL:
3760 smmu->features |= ARM_SMMU_FEAT_STALLS;
3764 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
3767 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
3769 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
3770 dev_err(smmu->dev, "no translation support!\n");
3774 /* We only support the AArch64 table format at present */
3775 switch (FIELD_GET(IDR0_TTF, reg)) {
3776 case IDR0_TTF_AARCH32_64:
3779 case IDR0_TTF_AARCH64:
3782 dev_err(smmu->dev, "AArch64 table format not supported!\n");
3786 /* ASID/VMID sizes */
3787 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
3788 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
3791 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
3792 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
3793 dev_err(smmu->dev, "embedded implementation not supported\n");
3797 /* Queue sizes, capped to ensure natural alignment */
3798 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
3799 FIELD_GET(IDR1_CMDQS, reg));
3800 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
3802 * We don't support splitting up batches, so one batch of
3803 * commands plus an extra sync needs to fit inside the command
3804 * queue. There's also no way we can handle the weird alignment
3805 * restrictions on the base pointer for a unit-length queue.
3807 dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
3808 CMDQ_BATCH_ENTRIES);
3812 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
3813 FIELD_GET(IDR1_EVTQS, reg));
3814 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
3815 FIELD_GET(IDR1_PRIQS, reg));
3817 /* SID/SSID sizes */
3818 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
3819 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
3822 * If the SMMU supports fewer bits than would fill a single L2 stream
3823 * table, use a linear table instead.
3825 if (smmu->sid_bits <= STRTAB_SPLIT)
3826 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
3829 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
3830 if (FIELD_GET(IDR3_RIL, reg))
3831 smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
3834 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
3836 /* Maximum number of outstanding stalls */
3837 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
3840 if (reg & IDR5_GRAN64K)
3841 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
3842 if (reg & IDR5_GRAN16K)
3843 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
3844 if (reg & IDR5_GRAN4K)
3845 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
3847 /* Input address size */
3848 if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
3849 smmu->features |= ARM_SMMU_FEAT_VAX;
3851 /* Output address size */
3852 switch (FIELD_GET(IDR5_OAS, reg)) {
3853 case IDR5_OAS_32_BIT:
3856 case IDR5_OAS_36_BIT:
3859 case IDR5_OAS_40_BIT:
3862 case IDR5_OAS_42_BIT:
3865 case IDR5_OAS_44_BIT:
3868 case IDR5_OAS_52_BIT:
3870 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
3874 "unknown output address size. Truncating to 48-bit\n");
3876 case IDR5_OAS_48_BIT:
3880 if (arm_smmu_ops.pgsize_bitmap == -1UL)
3881 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
3883 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
3885 /* Set the DMA mask for our table walker */
3886 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
3888 "failed to set DMA mask for table walker\n");
3890 smmu->ias = max(smmu->ias, smmu->oas);
3892 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
3893 smmu->ias, smmu->oas, smmu->features);
3898 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
3901 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
3902 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
3904 case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
3905 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
3909 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
3912 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3913 struct arm_smmu_device *smmu)
3915 struct acpi_iort_smmu_v3 *iort_smmu;
3916 struct device *dev = smmu->dev;
3917 struct acpi_iort_node *node;
3919 node = *(struct acpi_iort_node **)dev_get_platdata(dev);
3921 /* Retrieve SMMUv3 specific data */
3922 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3924 acpi_smmu_get_options(iort_smmu->model, smmu);
3926 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3927 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3932 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3933 struct arm_smmu_device *smmu)
3939 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
3940 struct arm_smmu_device *smmu)
3942 struct device *dev = &pdev->dev;
3946 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3947 dev_err(dev, "missing #iommu-cells property\n");
3948 else if (cells != 1)
3949 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3953 parse_driver_options(smmu);
3955 if (of_dma_is_coherent(dev->of_node))
3956 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3961 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
3963 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3969 static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
3974 if (pci_bus_type.iommu_ops != ops) {
3975 err = bus_set_iommu(&pci_bus_type, ops);
3980 #ifdef CONFIG_ARM_AMBA
3981 if (amba_bustype.iommu_ops != ops) {
3982 err = bus_set_iommu(&amba_bustype, ops);
3984 goto err_reset_pci_ops;
3987 if (platform_bus_type.iommu_ops != ops) {
3988 err = bus_set_iommu(&platform_bus_type, ops);
3990 goto err_reset_amba_ops;
3996 #ifdef CONFIG_ARM_AMBA
3997 bus_set_iommu(&amba_bustype, NULL);
3999 err_reset_pci_ops: __maybe_unused;
4001 bus_set_iommu(&pci_bus_type, NULL);
4006 static int arm_smmu_device_probe(struct platform_device *pdev)
4009 struct resource *res;
4010 resource_size_t ioaddr;
4011 struct arm_smmu_device *smmu;
4012 struct device *dev = &pdev->dev;
4015 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
4017 dev_err(dev, "failed to allocate arm_smmu_device\n");
4023 ret = arm_smmu_device_dt_probe(pdev, smmu);
4025 ret = arm_smmu_device_acpi_probe(pdev, smmu);
4030 /* Set bypass mode according to firmware probing result */
4034 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4035 if (resource_size(res) < arm_smmu_resource_size(smmu)) {
4036 dev_err(dev, "MMIO region too small (%pr)\n", res);
4039 ioaddr = res->start;
4041 smmu->base = devm_ioremap_resource(dev, res);
4042 if (IS_ERR(smmu->base))
4043 return PTR_ERR(smmu->base);
4045 /* Interrupt lines */
4047 irq = platform_get_irq_byname_optional(pdev, "combined");
4049 smmu->combined_irq = irq;
4051 irq = platform_get_irq_byname_optional(pdev, "eventq");
4053 smmu->evtq.q.irq = irq;
4055 irq = platform_get_irq_byname_optional(pdev, "priq");
4057 smmu->priq.q.irq = irq;
4059 irq = platform_get_irq_byname_optional(pdev, "gerror");
4061 smmu->gerr_irq = irq;
4064 ret = arm_smmu_device_hw_probe(smmu);
4068 /* Initialise in-memory data structures */
4069 ret = arm_smmu_init_structures(smmu);
4073 /* Record our private device structure */
4074 platform_set_drvdata(pdev, smmu);
4076 /* Reset the device */
4077 ret = arm_smmu_device_reset(smmu, bypass);
4081 /* And we're up. Go go go! */
4082 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
4083 "smmu3.%pa", &ioaddr);
4087 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
4088 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
4090 ret = iommu_device_register(&smmu->iommu);
4092 dev_err(dev, "Failed to register iommu\n");
4096 return arm_smmu_set_bus_ops(&arm_smmu_ops);
4099 static int arm_smmu_device_remove(struct platform_device *pdev)
4101 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4103 arm_smmu_set_bus_ops(NULL);
4104 iommu_device_unregister(&smmu->iommu);
4105 iommu_device_sysfs_remove(&smmu->iommu);
4106 arm_smmu_device_disable(smmu);
4111 static void arm_smmu_device_shutdown(struct platform_device *pdev)
4113 arm_smmu_device_remove(pdev);
4116 static const struct of_device_id arm_smmu_of_match[] = {
4117 { .compatible = "arm,smmu-v3", },
4120 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
4122 static struct platform_driver arm_smmu_driver = {
4124 .name = "arm-smmu-v3",
4125 .of_match_table = arm_smmu_of_match,
4126 .suppress_bind_attrs = true,
4128 .probe = arm_smmu_device_probe,
4129 .remove = arm_smmu_device_remove,
4130 .shutdown = arm_smmu_device_shutdown,
4132 module_platform_driver(arm_smmu_driver);
4134 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
4135 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
4136 MODULE_ALIAS("platform:arm-smmu-v3");
4137 MODULE_LICENSE("GPL v2");