1 // SPDX-License-Identifier: GPL-2.0
3 * Helpers for IOMMU drivers implementing SVA
5 #include <linux/mmu_context.h>
6 #include <linux/mutex.h>
7 #include <linux/sched/mm.h>
8 #include <linux/iommu.h>
10 #include "iommu-sva.h"
12 static DEFINE_MUTEX(iommu_sva_lock);
13 static DEFINE_IDA(iommu_global_pasid_ida);
15 /* Allocate a PASID for the mm within range (inclusive) */
16 static int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max)
20 if (min == IOMMU_PASID_INVALID ||
21 max == IOMMU_PASID_INVALID ||
22 min == 0 || max < min)
25 if (!arch_pgtable_dma_compat(mm))
28 mutex_lock(&iommu_sva_lock);
29 /* Is a PASID already associated with this mm? */
30 if (mm_valid_pasid(mm)) {
31 if (mm->pasid < min || mm->pasid > max)
36 ret = ida_alloc_range(&iommu_global_pasid_ida, min, max, GFP_KERNEL);
42 mutex_unlock(&iommu_sva_lock);
47 * iommu_sva_bind_device() - Bind a process address space to a device
49 * @mm: the mm to bind, caller must hold a reference to mm_users
51 * Create a bond between device and address space, allowing the device to
52 * access the mm using the PASID returned by iommu_sva_get_pasid(). If a
53 * bond already exists between @device and @mm, an additional internal
54 * reference is taken. Caller must call iommu_sva_unbind_device()
55 * to release each reference.
57 * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
58 * initialize the required SVA features.
60 * On error, returns an ERR_PTR value.
62 struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
64 struct iommu_domain *domain;
65 struct iommu_sva *handle;
69 max_pasids = dev->iommu->max_pasids;
71 return ERR_PTR(-EOPNOTSUPP);
73 /* Allocate mm->pasid if necessary. */
74 ret = iommu_sva_alloc_pasid(mm, 1, max_pasids - 1);
78 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
80 return ERR_PTR(-ENOMEM);
82 mutex_lock(&iommu_sva_lock);
83 /* Search for an existing domain. */
84 domain = iommu_get_domain_for_dev_pasid(dev, mm->pasid,
87 ret = PTR_ERR(domain);
96 /* Allocate a new domain and set it on device pasid. */
97 domain = iommu_sva_domain_alloc(dev, mm);
103 ret = iommu_attach_device_pasid(domain, dev, mm->pasid);
105 goto out_free_domain;
108 mutex_unlock(&iommu_sva_lock);
110 handle->domain = domain;
115 iommu_domain_free(domain);
117 mutex_unlock(&iommu_sva_lock);
122 EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
125 * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
126 * @handle: the handle returned by iommu_sva_bind_device()
128 * Put reference to a bond between device and address space. The device should
129 * not be issuing any more transaction for this PASID. All outstanding page
130 * requests for this PASID must have been flushed to the IOMMU.
132 void iommu_sva_unbind_device(struct iommu_sva *handle)
134 struct iommu_domain *domain = handle->domain;
135 ioasid_t pasid = domain->mm->pasid;
136 struct device *dev = handle->dev;
138 mutex_lock(&iommu_sva_lock);
139 if (--domain->users == 0) {
140 iommu_detach_device_pasid(domain, dev, pasid);
141 iommu_domain_free(domain);
143 mutex_unlock(&iommu_sva_lock);
146 EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
148 u32 iommu_sva_get_pasid(struct iommu_sva *handle)
150 struct iommu_domain *domain = handle->domain;
152 return domain->mm->pasid;
154 EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);
157 * I/O page fault handler for SVA
159 enum iommu_page_response_code
160 iommu_sva_handle_iopf(struct iommu_fault *fault, void *data)
163 struct vm_area_struct *vma;
164 struct mm_struct *mm = data;
165 unsigned int access_flags = 0;
166 unsigned int fault_flags = FAULT_FLAG_REMOTE;
167 struct iommu_fault_page_request *prm = &fault->prm;
168 enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID;
170 if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID))
173 if (!mmget_not_zero(mm))
178 vma = find_extend_vma(mm, prm->addr);
183 if (prm->perm & IOMMU_FAULT_PERM_READ)
184 access_flags |= VM_READ;
186 if (prm->perm & IOMMU_FAULT_PERM_WRITE) {
187 access_flags |= VM_WRITE;
188 fault_flags |= FAULT_FLAG_WRITE;
191 if (prm->perm & IOMMU_FAULT_PERM_EXEC) {
192 access_flags |= VM_EXEC;
193 fault_flags |= FAULT_FLAG_INSTRUCTION;
196 if (!(prm->perm & IOMMU_FAULT_PERM_PRIV))
197 fault_flags |= FAULT_FLAG_USER;
199 if (access_flags & ~vma->vm_flags)
203 ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL);
204 status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID :
205 IOMMU_PAGE_RESP_SUCCESS;
208 mmap_read_unlock(mm);
214 void mm_pasid_drop(struct mm_struct *mm)
216 if (likely(!mm_valid_pasid(mm)))
219 ida_free(&iommu_global_pasid_ida, mm->pasid);