1 // SPDX-License-Identifier: GPL-2.0
3 * Device driver to expose SGX enclave memory to KVM guests.
5 * Copyright(c) 2021 Intel Corporation.
8 #include <linux/miscdevice.h>
10 #include <linux/mman.h>
11 #include <linux/sched/mm.h>
12 #include <linux/sched/signal.h>
13 #include <linux/slab.h>
14 #include <linux/xarray.h>
16 #include <uapi/asm/sgx.h>
22 struct xarray page_array;
27 * Temporary SECS pages that cannot be EREMOVE'd due to having child in other
28 * virtual EPC instances, and the lock to protect it.
30 static struct mutex zombie_secs_pages_lock;
31 static struct list_head zombie_secs_pages;
33 static int __sgx_vepc_fault(struct sgx_vepc *vepc,
34 struct vm_area_struct *vma, unsigned long addr)
36 struct sgx_epc_page *epc_page;
37 unsigned long index, pfn;
40 WARN_ON(!mutex_is_locked(&vepc->lock));
42 /* Calculate index of EPC page in virtual EPC's page_array */
43 index = vma->vm_pgoff + PFN_DOWN(addr - vma->vm_start);
45 epc_page = xa_load(&vepc->page_array, index);
49 epc_page = sgx_alloc_epc_page(vepc, false);
51 return PTR_ERR(epc_page);
53 ret = xa_err(xa_store(&vepc->page_array, index, epc_page, GFP_KERNEL));
57 pfn = PFN_DOWN(sgx_get_epc_phys_addr(epc_page));
59 ret = vmf_insert_pfn(vma, addr, pfn);
60 if (ret != VM_FAULT_NOPAGE) {
68 xa_erase(&vepc->page_array, index);
70 sgx_free_epc_page(epc_page);
74 static vm_fault_t sgx_vepc_fault(struct vm_fault *vmf)
76 struct vm_area_struct *vma = vmf->vma;
77 struct sgx_vepc *vepc = vma->vm_private_data;
80 mutex_lock(&vepc->lock);
81 ret = __sgx_vepc_fault(vepc, vma, vmf->address);
82 mutex_unlock(&vepc->lock);
85 return VM_FAULT_NOPAGE;
87 if (ret == -EBUSY && (vmf->flags & FAULT_FLAG_ALLOW_RETRY)) {
88 mmap_read_unlock(vma->vm_mm);
89 return VM_FAULT_RETRY;
92 return VM_FAULT_SIGBUS;
95 static const struct vm_operations_struct sgx_vepc_vm_ops = {
96 .fault = sgx_vepc_fault,
99 static int sgx_vepc_mmap(struct file *file, struct vm_area_struct *vma)
101 struct sgx_vepc *vepc = file->private_data;
103 if (!(vma->vm_flags & VM_SHARED))
106 vma->vm_ops = &sgx_vepc_vm_ops;
107 /* Don't copy VMA in fork() */
108 vma->vm_flags |= VM_PFNMAP | VM_IO | VM_DONTDUMP | VM_DONTCOPY;
109 vma->vm_private_data = vepc;
114 static int sgx_vepc_remove_page(struct sgx_epc_page *epc_page)
117 * Take a previously guest-owned EPC page and return it to the
118 * general EPC page pool.
120 * Guests can not be trusted to have left this page in a good
121 * state, so run EREMOVE on the page unconditionally. In the
122 * case that a guest properly EREMOVE'd this page, a superfluous
123 * EREMOVE is harmless.
125 return __eremove(sgx_get_epc_virt_addr(epc_page));
128 static int sgx_vepc_free_page(struct sgx_epc_page *epc_page)
130 int ret = sgx_vepc_remove_page(epc_page);
133 * Only SGX_CHILD_PRESENT is expected, which is because of
134 * EREMOVE'ing an SECS still with child, in which case it can
135 * be handled by EREMOVE'ing the SECS again after all pages in
136 * virtual EPC have been EREMOVE'd. See comments in below in
137 * sgx_vepc_release().
139 * The user of virtual EPC (KVM) needs to guarantee there's no
140 * logical processor is still running in the enclave in guest,
141 * otherwise EREMOVE will get SGX_ENCLAVE_ACT which cannot be
144 WARN_ONCE(ret != SGX_CHILD_PRESENT, EREMOVE_ERROR_MESSAGE,
149 sgx_free_epc_page(epc_page);
153 static long sgx_vepc_remove_all(struct sgx_vepc *vepc)
155 struct sgx_epc_page *entry;
159 xa_for_each(&vepc->page_array, index, entry) {
160 int ret = sgx_vepc_remove_page(entry);
162 if (ret == SGX_CHILD_PRESENT) {
163 /* The page is a SECS, userspace will retry. */
167 * Report errors due to #GP or SGX_ENCLAVE_ACT; do not
168 * WARN, as userspace can induce said failures by
169 * calling the ioctl concurrently on multiple vEPCs or
170 * while one or more CPUs is running the enclave. Only
171 * a #PF on EREMOVE indicates a kernel/hardware issue.
173 WARN_ON_ONCE(encls_faulted(ret) &&
174 ENCLS_TRAPNR(ret) != X86_TRAP_GP);
182 * Return the number of SECS pages that failed to be removed, so
183 * userspace knows that it has to retry.
188 static int sgx_vepc_release(struct inode *inode, struct file *file)
190 struct sgx_vepc *vepc = file->private_data;
191 struct sgx_epc_page *epc_page, *tmp, *entry;
194 LIST_HEAD(secs_pages);
196 xa_for_each(&vepc->page_array, index, entry) {
198 * Remove all normal, child pages. sgx_vepc_free_page()
199 * will fail if EREMOVE fails, but this is OK and expected on
200 * SECS pages. Those can only be EREMOVE'd *after* all their
201 * child pages. Retries below will clean them up.
203 if (sgx_vepc_free_page(entry))
206 xa_erase(&vepc->page_array, index);
210 * Retry EREMOVE'ing pages. This will clean up any SECS pages that
211 * only had children in this 'epc' area.
213 xa_for_each(&vepc->page_array, index, entry) {
216 * An EREMOVE failure here means that the SECS page still
217 * has children. But, since all children in this 'sgx_vepc'
218 * have been removed, the SECS page must have a child on
221 if (sgx_vepc_free_page(epc_page))
222 list_add_tail(&epc_page->list, &secs_pages);
224 xa_erase(&vepc->page_array, index);
228 * SECS pages are "pinned" by child pages, and "unpinned" once all
229 * children have been EREMOVE'd. A child page in this instance
230 * may have pinned an SECS page encountered in an earlier release(),
231 * creating a zombie. Since some children were EREMOVE'd above,
232 * try to EREMOVE all zombies in the hopes that one was unpinned.
234 mutex_lock(&zombie_secs_pages_lock);
235 list_for_each_entry_safe(epc_page, tmp, &zombie_secs_pages, list) {
237 * Speculatively remove the page from the list of zombies,
238 * if the page is successfully EREMOVE'd it will be added to
239 * the list of free pages. If EREMOVE fails, throw the page
240 * on the local list, which will be spliced on at the end.
242 list_del(&epc_page->list);
244 if (sgx_vepc_free_page(epc_page))
245 list_add_tail(&epc_page->list, &secs_pages);
248 if (!list_empty(&secs_pages))
249 list_splice_tail(&secs_pages, &zombie_secs_pages);
250 mutex_unlock(&zombie_secs_pages_lock);
252 xa_destroy(&vepc->page_array);
258 static int sgx_vepc_open(struct inode *inode, struct file *file)
260 struct sgx_vepc *vepc;
262 vepc = kzalloc(sizeof(struct sgx_vepc), GFP_KERNEL);
265 mutex_init(&vepc->lock);
266 xa_init(&vepc->page_array);
268 file->private_data = vepc;
273 static long sgx_vepc_ioctl(struct file *file,
274 unsigned int cmd, unsigned long arg)
276 struct sgx_vepc *vepc = file->private_data;
279 case SGX_IOC_VEPC_REMOVE_ALL:
282 return sgx_vepc_remove_all(vepc);
289 static const struct file_operations sgx_vepc_fops = {
290 .owner = THIS_MODULE,
291 .open = sgx_vepc_open,
292 .unlocked_ioctl = sgx_vepc_ioctl,
293 .compat_ioctl = sgx_vepc_ioctl,
294 .release = sgx_vepc_release,
295 .mmap = sgx_vepc_mmap,
298 static struct miscdevice sgx_vepc_dev = {
299 .minor = MISC_DYNAMIC_MINOR,
301 .nodename = "sgx_vepc",
302 .fops = &sgx_vepc_fops,
305 int __init sgx_vepc_init(void)
307 /* SGX virtualization requires KVM to work */
308 if (!cpu_feature_enabled(X86_FEATURE_VMX))
311 INIT_LIST_HEAD(&zombie_secs_pages);
312 mutex_init(&zombie_secs_pages_lock);
314 return misc_register(&sgx_vepc_dev);
318 * sgx_virt_ecreate() - Run ECREATE on behalf of guest
319 * @pageinfo: Pointer to PAGEINFO structure
320 * @secs: Userspace pointer to SECS page
321 * @trapnr: trap number injected to guest in case of ECREATE error
323 * Run ECREATE on behalf of guest after KVM traps ECREATE for the purpose
324 * of enforcing policies of guest's enclaves, and return the trap number
325 * which should be injected to guest in case of any ECREATE error.
328 * - 0: ECREATE was successful.
331 int sgx_virt_ecreate(struct sgx_pageinfo *pageinfo, void __user *secs,
337 * @secs is an untrusted, userspace-provided address. It comes from
338 * KVM and is assumed to be a valid pointer which points somewhere in
339 * userspace. This can fault and call SGX or other fault handlers when
340 * userspace mapping @secs doesn't exist.
342 * Add a WARN() to make sure @secs is already valid userspace pointer
343 * from caller (KVM), who should already have handled invalid pointer
344 * case (for instance, made by malicious guest). All other checks,
345 * such as alignment of @secs, are deferred to ENCLS itself.
347 if (WARN_ON_ONCE(!access_ok(secs, PAGE_SIZE)))
351 ret = __ecreate(pageinfo, (void *)secs);
354 if (encls_faulted(ret)) {
355 *trapnr = ENCLS_TRAPNR(ret);
359 /* ECREATE doesn't return an error code, it faults or succeeds. */
363 EXPORT_SYMBOL_GPL(sgx_virt_ecreate);
365 static int __sgx_virt_einit(void __user *sigstruct, void __user *token,
371 * Make sure all userspace pointers from caller (KVM) are valid.
372 * All other checks deferred to ENCLS itself. Also see comment
373 * for @secs in sgx_virt_ecreate().
375 #define SGX_EINITTOKEN_SIZE 304
376 if (WARN_ON_ONCE(!access_ok(sigstruct, sizeof(struct sgx_sigstruct)) ||
377 !access_ok(token, SGX_EINITTOKEN_SIZE) ||
378 !access_ok(secs, PAGE_SIZE)))
382 ret = __einit((void *)sigstruct, (void *)token, (void *)secs);
389 * sgx_virt_einit() - Run EINIT on behalf of guest
390 * @sigstruct: Userspace pointer to SIGSTRUCT structure
391 * @token: Userspace pointer to EINITTOKEN structure
392 * @secs: Userspace pointer to SECS page
393 * @lepubkeyhash: Pointer to guest's *virtual* SGX_LEPUBKEYHASH MSR values
394 * @trapnr: trap number injected to guest in case of EINIT error
396 * Run EINIT on behalf of guest after KVM traps EINIT. If SGX_LC is available
397 * in host, SGX driver may rewrite the hardware values at wish, therefore KVM
398 * needs to update hardware values to guest's virtual MSR values in order to
399 * ensure EINIT is executed with expected hardware values.
402 * - 0: EINIT was successful.
405 int sgx_virt_einit(void __user *sigstruct, void __user *token,
406 void __user *secs, u64 *lepubkeyhash, int *trapnr)
410 if (!cpu_feature_enabled(X86_FEATURE_SGX_LC)) {
411 ret = __sgx_virt_einit(sigstruct, token, secs);
415 sgx_update_lepubkeyhash(lepubkeyhash);
417 ret = __sgx_virt_einit(sigstruct, token, secs);
421 /* Propagate up the error from the WARN_ON_ONCE in __sgx_virt_einit() */
425 if (encls_faulted(ret)) {
426 *trapnr = ENCLS_TRAPNR(ret);
432 EXPORT_SYMBOL_GPL(sgx_virt_einit);