1 // SPDX-License-Identifier: GPL-2.0
3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2020 Linaro Ltd.
7 #include <linux/types.h>
8 #include <linux/bitfield.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/iommu.h>
13 #include <linux/soc/qcom/smem.h>
20 #include "ipa_table.h"
21 #include "gsi_trans.h"
23 /* "Canary" value placed between memory regions to detect overflow */
24 #define IPA_MEM_CANARY_VAL cpu_to_le32(0xdeadbeef)
26 /* SMEM host id representing the modem. */
27 #define QCOM_SMEM_HOST_MODEM 1
29 /* Add an immediate command to a transaction that zeroes a memory region */
31 ipa_mem_zero_region_add(struct gsi_trans *trans, const struct ipa_mem *mem)
33 struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi);
34 dma_addr_t addr = ipa->zero_addr;
39 ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true);
43 * ipa_mem_setup() - Set up IPA AP and modem shared memory areas
45 * Set up the shared memory regions in IPA local memory. This involves
46 * zero-filling memory regions, and in the case of header memory, telling
47 * the IPA where it's located.
49 * This function performs the initial setup of this memory. If the modem
50 * crashes, its regions are re-zeroed in ipa_mem_zero_modem().
52 * The AP informs the modem where its portions of memory are located
53 * in a QMI exchange that occurs at modem startup.
55 * @Return: 0 if successful, or a negative error code
57 int ipa_mem_setup(struct ipa *ipa)
59 dma_addr_t addr = ipa->zero_addr;
60 struct gsi_trans *trans;
64 /* Get a transaction to define the header memory region and to zero
65 * the processing context and modem memory regions.
67 trans = ipa_cmd_trans_alloc(ipa, 4);
69 dev_err(&ipa->pdev->dev, "no transaction for memory setup\n");
73 /* Initialize IPA-local header memory. The modem and AP header
74 * regions are contiguous, and initialized together.
76 offset = ipa->mem[IPA_MEM_MODEM_HEADER].offset;
77 size = ipa->mem[IPA_MEM_MODEM_HEADER].size;
78 size += ipa->mem[IPA_MEM_AP_HEADER].size;
80 ipa_cmd_hdr_init_local_add(trans, offset, size, addr);
82 ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_MODEM_PROC_CTX]);
84 ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_AP_PROC_CTX]);
86 ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_MODEM]);
88 gsi_trans_commit_wait(trans);
90 /* Tell the hardware where the processing context area is located */
91 iowrite32(ipa->mem_offset + offset,
92 ipa->reg_virt + IPA_REG_LOCAL_PKT_PROC_CNTXT_BASE_OFFSET);
97 void ipa_mem_teardown(struct ipa *ipa)
104 static bool ipa_mem_valid(struct ipa *ipa, enum ipa_mem_id mem_id)
106 const struct ipa_mem *mem = &ipa->mem[mem_id];
107 struct device *dev = &ipa->pdev->dev;
110 /* Other than modem memory, sizes must be a multiple of 8 */
111 size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8;
112 if (mem->size % size_multiple)
113 dev_err(dev, "region %u size not a multiple of %u bytes\n",
114 mem_id, size_multiple);
115 else if (mem->offset % 8)
116 dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id);
117 else if (mem->offset < mem->canary_count * sizeof(__le32))
118 dev_err(dev, "region %u offset too small for %hu canaries\n",
119 mem_id, mem->canary_count);
120 else if (mem->offset + mem->size > ipa->mem_size)
121 dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n",
122 mem_id, ipa->mem_size);
129 #else /* !IPA_VALIDATE */
131 static bool ipa_mem_valid(struct ipa *ipa, enum ipa_mem_id mem_id)
136 #endif /*! IPA_VALIDATE */
139 * ipa_mem_config() - Configure IPA shared memory
141 * @Return: 0 if successful, or a negative error code
143 int ipa_mem_config(struct ipa *ipa)
145 struct device *dev = &ipa->pdev->dev;
146 enum ipa_mem_id mem_id;
152 /* Check the advertised location and size of the shared memory area */
153 val = ioread32(ipa->reg_virt + IPA_REG_SHARED_MEM_SIZE_OFFSET);
155 /* The fields in the register are in 8 byte units */
156 ipa->mem_offset = 8 * u32_get_bits(val, SHARED_MEM_BADDR_FMASK);
157 /* Make sure the end is within the region's mapped space */
158 mem_size = 8 * u32_get_bits(val, SHARED_MEM_SIZE_FMASK);
160 /* If the sizes don't match, issue a warning */
161 if (ipa->mem_offset + mem_size > ipa->mem_size) {
162 dev_warn(dev, "ignoring larger reported memory size: 0x%08x\n",
164 } else if (ipa->mem_offset + mem_size < ipa->mem_size) {
165 dev_warn(dev, "limiting IPA memory size to 0x%08x\n",
167 ipa->mem_size = mem_size;
170 /* Prealloc DMA memory for zeroing regions */
171 virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL);
174 ipa->zero_addr = addr;
175 ipa->zero_virt = virt;
176 ipa->zero_size = IPA_MEM_MAX;
178 /* Verify each defined memory region is valid, and if indicated
179 * for the region, write "canary" values in the space prior to
180 * the region's base address.
182 for (mem_id = 0; mem_id < IPA_MEM_COUNT; mem_id++) {
183 const struct ipa_mem *mem = &ipa->mem[mem_id];
187 /* Validate all regions (even undefined ones) */
188 if (!ipa_mem_valid(ipa, mem_id))
191 /* Skip over undefined regions */
192 if (!mem->offset && !mem->size)
195 canary_count = mem->canary_count;
199 /* Write canary values in the space before the region */
200 canary = ipa->mem_virt + ipa->mem_offset + mem->offset;
202 *--canary = IPA_MEM_CANARY_VAL;
203 while (--canary_count);
206 /* Make sure filter and route table memory regions are valid */
207 if (!ipa_table_valid(ipa))
210 /* Validate memory-related properties relevant to immediate commands */
211 if (!ipa_cmd_data_valid(ipa))
214 /* Verify the microcontroller ring alignment (0 is OK too) */
215 if (ipa->mem[IPA_MEM_UC_EVENT_RING].offset % 1024) {
216 dev_err(dev, "microcontroller ring not 1024-byte aligned\n");
223 dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr);
228 /* Inverse of ipa_mem_config() */
229 void ipa_mem_deconfig(struct ipa *ipa)
231 struct device *dev = &ipa->pdev->dev;
233 dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr);
235 ipa->zero_virt = NULL;
240 * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem
242 * Zero regions of IPA-local memory used by the modem. These are configured
243 * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and
244 * restarts via SSR we need to re-initialize them. A QMI message tells the
245 * modem where to find regions of IPA local memory it needs to know about
248 int ipa_mem_zero_modem(struct ipa *ipa)
250 struct gsi_trans *trans;
252 /* Get a transaction to zero the modem memory, modem header,
253 * and modem processing context regions.
255 trans = ipa_cmd_trans_alloc(ipa, 3);
257 dev_err(&ipa->pdev->dev,
258 "no transaction to zero modem memory\n");
262 ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_MODEM_HEADER]);
264 ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_MODEM_PROC_CTX]);
266 ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_MODEM]);
268 gsi_trans_commit_wait(trans);
274 * ipa_imem_init() - Initialize IMEM memory used by the IPA
276 * @addr: Physical address of the IPA region in IMEM
277 * @size: Size (bytes) of the IPA region in IMEM
279 * IMEM is a block of shared memory separate from system DRAM, and
280 * a portion of this memory is available for the IPA to use. The
281 * modem accesses this memory directly, but the IPA accesses it
282 * via the IOMMU, using the AP's credentials.
284 * If this region exists (size > 0) we map it for read/write access
285 * through the IOMMU using the IPA device.
287 * Note: @addr and @size are not guaranteed to be page-aligned.
289 static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size)
291 struct device *dev = &ipa->pdev->dev;
292 struct iommu_domain *domain;
298 return 0; /* IMEM memory not used */
300 domain = iommu_get_domain_for_dev(dev);
302 dev_err(dev, "no IOMMU domain found for IMEM\n");
306 /* Align the address down and the size up to page boundaries */
307 phys = addr & PAGE_MASK;
308 size = PAGE_ALIGN(size + addr - phys);
309 iova = phys; /* We just want a direct mapping */
311 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE);
315 ipa->imem_iova = iova;
316 ipa->imem_size = size;
321 static void ipa_imem_exit(struct ipa *ipa)
323 struct iommu_domain *domain;
329 dev = &ipa->pdev->dev;
330 domain = iommu_get_domain_for_dev(dev);
334 size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size);
335 if (size != ipa->imem_size)
336 dev_warn(dev, "unmapped %zu IMEM bytes, expected %lu\n",
337 size, ipa->imem_size);
339 dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n");
347 * ipa_smem_init() - Initialize SMEM memory used by the IPA
349 * @item: Item ID of SMEM memory
350 * @size: Size (bytes) of SMEM memory region
352 * SMEM is a managed block of shared DRAM, from which numbered "items"
353 * can be allocated. One item is designated for use by the IPA.
355 * The modem accesses SMEM memory directly, but the IPA accesses it
356 * via the IOMMU, using the AP's credentials.
358 * If size provided is non-zero, we allocate it and map it for
359 * access through the IOMMU.
361 * Note: @size and the item address are is not guaranteed to be page-aligned.
363 static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size)
365 struct device *dev = &ipa->pdev->dev;
366 struct iommu_domain *domain;
375 return 0; /* SMEM memory not used */
377 /* SMEM is memory shared between the AP and another system entity
378 * (in this case, the modem). An allocation from SMEM is persistent
379 * until the AP reboots; there is no way to free an allocated SMEM
380 * region. Allocation only reserves the space; to use it you need
381 * to "get" a pointer it (this implies no reference counting).
382 * The item might have already been allocated, in which case we
383 * use it unless the size isn't what we expect.
385 ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, item, size);
386 if (ret && ret != -EEXIST) {
387 dev_err(dev, "error %d allocating size %zu SMEM item %u\n",
392 /* Now get the address of the SMEM memory region */
393 virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, item, &actual);
396 dev_err(dev, "error %d getting SMEM item %u\n", ret, item);
400 /* In case the region was already allocated, verify the size */
401 if (ret && actual != size) {
402 dev_err(dev, "SMEM item %u has size %zu, expected %zu\n",
407 domain = iommu_get_domain_for_dev(dev);
409 dev_err(dev, "no IOMMU domain found for SMEM\n");
413 /* Align the address down and the size up to a page boundary */
414 addr = qcom_smem_virt_to_phys(virt) & PAGE_MASK;
415 phys = addr & PAGE_MASK;
416 size = PAGE_ALIGN(size + addr - phys);
417 iova = phys; /* We just want a direct mapping */
419 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE);
423 ipa->smem_iova = iova;
424 ipa->smem_size = size;
429 static void ipa_smem_exit(struct ipa *ipa)
431 struct device *dev = &ipa->pdev->dev;
432 struct iommu_domain *domain;
434 domain = iommu_get_domain_for_dev(dev);
438 size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size);
439 if (size != ipa->smem_size)
440 dev_warn(dev, "unmapped %zu SMEM bytes, expected %lu\n",
441 size, ipa->smem_size);
444 dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n");
451 /* Perform memory region-related initialization */
452 int ipa_mem_init(struct ipa *ipa, const struct ipa_mem_data *mem_data)
454 struct device *dev = &ipa->pdev->dev;
455 struct resource *res;
458 if (mem_data->local_count > IPA_MEM_COUNT) {
459 dev_err(dev, "to many memory regions (%u > %u)\n",
460 mem_data->local_count, IPA_MEM_COUNT);
464 ret = dma_set_mask_and_coherent(&ipa->pdev->dev, DMA_BIT_MASK(64));
466 dev_err(dev, "error %d setting DMA mask\n", ret);
470 res = platform_get_resource_byname(ipa->pdev, IORESOURCE_MEM,
474 "DT error getting \"ipa-shared\" memory property\n");
478 ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC);
479 if (!ipa->mem_virt) {
480 dev_err(dev, "unable to remap \"ipa-shared\" memory\n");
484 ipa->mem_addr = res->start;
485 ipa->mem_size = resource_size(res);
487 /* The ipa->mem[] array is indexed by enum ipa_mem_id values */
488 ipa->mem = mem_data->local;
490 ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size);
494 ret = ipa_smem_init(ipa, mem_data->smem_id, mem_data->smem_size);
503 memunmap(ipa->mem_virt);
508 /* Inverse of ipa_mem_init() */
509 void ipa_mem_exit(struct ipa *ipa)
513 memunmap(ipa->mem_virt);