Merge tag 'driver-core-5.13-rc6' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6-microblaze.git] / drivers / tee / optee / call.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2015, Linaro Limited
4  */
5 #include <linux/arm-smccc.h>
6 #include <linux/device.h>
7 #include <linux/err.h>
8 #include <linux/errno.h>
9 #include <linux/mm.h>
10 #include <linux/sched.h>
11 #include <linux/slab.h>
12 #include <linux/tee_drv.h>
13 #include <linux/types.h>
14 #include <linux/uaccess.h>
15 #include "optee_private.h"
16 #include "optee_smc.h"
17 #define CREATE_TRACE_POINTS
18 #include "optee_trace.h"
19
20 struct optee_call_waiter {
21         struct list_head list_node;
22         struct completion c;
23 };
24
25 static void optee_cq_wait_init(struct optee_call_queue *cq,
26                                struct optee_call_waiter *w)
27 {
28         /*
29          * We're preparing to make a call to secure world. In case we can't
30          * allocate a thread in secure world we'll end up waiting in
31          * optee_cq_wait_for_completion().
32          *
33          * Normally if there's no contention in secure world the call will
34          * complete and we can cleanup directly with optee_cq_wait_final().
35          */
36         mutex_lock(&cq->mutex);
37
38         /*
39          * We add ourselves to the queue, but we don't wait. This
40          * guarantees that we don't lose a completion if secure world
41          * returns busy and another thread just exited and try to complete
42          * someone.
43          */
44         init_completion(&w->c);
45         list_add_tail(&w->list_node, &cq->waiters);
46
47         mutex_unlock(&cq->mutex);
48 }
49
50 static void optee_cq_wait_for_completion(struct optee_call_queue *cq,
51                                          struct optee_call_waiter *w)
52 {
53         wait_for_completion(&w->c);
54
55         mutex_lock(&cq->mutex);
56
57         /* Move to end of list to get out of the way for other waiters */
58         list_del(&w->list_node);
59         reinit_completion(&w->c);
60         list_add_tail(&w->list_node, &cq->waiters);
61
62         mutex_unlock(&cq->mutex);
63 }
64
65 static void optee_cq_complete_one(struct optee_call_queue *cq)
66 {
67         struct optee_call_waiter *w;
68
69         list_for_each_entry(w, &cq->waiters, list_node) {
70                 if (!completion_done(&w->c)) {
71                         complete(&w->c);
72                         break;
73                 }
74         }
75 }
76
77 static void optee_cq_wait_final(struct optee_call_queue *cq,
78                                 struct optee_call_waiter *w)
79 {
80         /*
81          * We're done with the call to secure world. The thread in secure
82          * world that was used for this call is now available for some
83          * other task to use.
84          */
85         mutex_lock(&cq->mutex);
86
87         /* Get out of the list */
88         list_del(&w->list_node);
89
90         /* Wake up one eventual waiting task */
91         optee_cq_complete_one(cq);
92
93         /*
94          * If we're completed we've got a completion from another task that
95          * was just done with its call to secure world. Since yet another
96          * thread now is available in secure world wake up another eventual
97          * waiting task.
98          */
99         if (completion_done(&w->c))
100                 optee_cq_complete_one(cq);
101
102         mutex_unlock(&cq->mutex);
103 }
104
105 /* Requires the filpstate mutex to be held */
106 static struct optee_session *find_session(struct optee_context_data *ctxdata,
107                                           u32 session_id)
108 {
109         struct optee_session *sess;
110
111         list_for_each_entry(sess, &ctxdata->sess_list, list_node)
112                 if (sess->session_id == session_id)
113                         return sess;
114
115         return NULL;
116 }
117
118 /**
119  * optee_do_call_with_arg() - Do an SMC to OP-TEE in secure world
120  * @ctx:        calling context
121  * @parg:       physical address of message to pass to secure world
122  *
123  * Does and SMC to OP-TEE in secure world and handles eventual resulting
124  * Remote Procedure Calls (RPC) from OP-TEE.
125  *
126  * Returns return code from secure world, 0 is OK
127  */
128 u32 optee_do_call_with_arg(struct tee_context *ctx, phys_addr_t parg)
129 {
130         struct optee *optee = tee_get_drvdata(ctx->teedev);
131         struct optee_call_waiter w;
132         struct optee_rpc_param param = { };
133         struct optee_call_ctx call_ctx = { };
134         u32 ret;
135
136         param.a0 = OPTEE_SMC_CALL_WITH_ARG;
137         reg_pair_from_64(&param.a1, &param.a2, parg);
138         /* Initialize waiter */
139         optee_cq_wait_init(&optee->call_queue, &w);
140         while (true) {
141                 struct arm_smccc_res res;
142
143                 trace_optee_invoke_fn_begin(&param);
144                 optee->invoke_fn(param.a0, param.a1, param.a2, param.a3,
145                                  param.a4, param.a5, param.a6, param.a7,
146                                  &res);
147                 trace_optee_invoke_fn_end(&param, &res);
148
149                 if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {
150                         /*
151                          * Out of threads in secure world, wait for a thread
152                          * become available.
153                          */
154                         optee_cq_wait_for_completion(&optee->call_queue, &w);
155                 } else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {
156                         cond_resched();
157                         param.a0 = res.a0;
158                         param.a1 = res.a1;
159                         param.a2 = res.a2;
160                         param.a3 = res.a3;
161                         optee_handle_rpc(ctx, &param, &call_ctx);
162                 } else {
163                         ret = res.a0;
164                         break;
165                 }
166         }
167
168         optee_rpc_finalize_call(&call_ctx);
169         /*
170          * We're done with our thread in secure world, if there's any
171          * thread waiters wake up one.
172          */
173         optee_cq_wait_final(&optee->call_queue, &w);
174
175         return ret;
176 }
177
178 static struct tee_shm *get_msg_arg(struct tee_context *ctx, size_t num_params,
179                                    struct optee_msg_arg **msg_arg,
180                                    phys_addr_t *msg_parg)
181 {
182         int rc;
183         struct tee_shm *shm;
184         struct optee_msg_arg *ma;
185
186         shm = tee_shm_alloc(ctx, OPTEE_MSG_GET_ARG_SIZE(num_params),
187                             TEE_SHM_MAPPED);
188         if (IS_ERR(shm))
189                 return shm;
190
191         ma = tee_shm_get_va(shm, 0);
192         if (IS_ERR(ma)) {
193                 rc = PTR_ERR(ma);
194                 goto out;
195         }
196
197         rc = tee_shm_get_pa(shm, 0, msg_parg);
198         if (rc)
199                 goto out;
200
201         memset(ma, 0, OPTEE_MSG_GET_ARG_SIZE(num_params));
202         ma->num_params = num_params;
203         *msg_arg = ma;
204 out:
205         if (rc) {
206                 tee_shm_free(shm);
207                 return ERR_PTR(rc);
208         }
209
210         return shm;
211 }
212
213 int optee_open_session(struct tee_context *ctx,
214                        struct tee_ioctl_open_session_arg *arg,
215                        struct tee_param *param)
216 {
217         struct optee_context_data *ctxdata = ctx->data;
218         int rc;
219         struct tee_shm *shm;
220         struct optee_msg_arg *msg_arg;
221         phys_addr_t msg_parg;
222         struct optee_session *sess = NULL;
223         uuid_t client_uuid;
224
225         /* +2 for the meta parameters added below */
226         shm = get_msg_arg(ctx, arg->num_params + 2, &msg_arg, &msg_parg);
227         if (IS_ERR(shm))
228                 return PTR_ERR(shm);
229
230         msg_arg->cmd = OPTEE_MSG_CMD_OPEN_SESSION;
231         msg_arg->cancel_id = arg->cancel_id;
232
233         /*
234          * Initialize and add the meta parameters needed when opening a
235          * session.
236          */
237         msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |
238                                   OPTEE_MSG_ATTR_META;
239         msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |
240                                   OPTEE_MSG_ATTR_META;
241         memcpy(&msg_arg->params[0].u.value, arg->uuid, sizeof(arg->uuid));
242         msg_arg->params[1].u.value.c = arg->clnt_login;
243
244         rc = tee_session_calc_client_uuid(&client_uuid, arg->clnt_login,
245                                           arg->clnt_uuid);
246         if (rc)
247                 goto out;
248         export_uuid(msg_arg->params[1].u.octets, &client_uuid);
249
250         rc = optee_to_msg_param(msg_arg->params + 2, arg->num_params, param);
251         if (rc)
252                 goto out;
253
254         sess = kzalloc(sizeof(*sess), GFP_KERNEL);
255         if (!sess) {
256                 rc = -ENOMEM;
257                 goto out;
258         }
259
260         if (optee_do_call_with_arg(ctx, msg_parg)) {
261                 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
262                 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
263         }
264
265         if (msg_arg->ret == TEEC_SUCCESS) {
266                 /* A new session has been created, add it to the list. */
267                 sess->session_id = msg_arg->session;
268                 mutex_lock(&ctxdata->mutex);
269                 list_add(&sess->list_node, &ctxdata->sess_list);
270                 mutex_unlock(&ctxdata->mutex);
271         } else {
272                 kfree(sess);
273         }
274
275         if (optee_from_msg_param(param, arg->num_params, msg_arg->params + 2)) {
276                 arg->ret = TEEC_ERROR_COMMUNICATION;
277                 arg->ret_origin = TEEC_ORIGIN_COMMS;
278                 /* Close session again to avoid leakage */
279                 optee_close_session(ctx, msg_arg->session);
280         } else {
281                 arg->session = msg_arg->session;
282                 arg->ret = msg_arg->ret;
283                 arg->ret_origin = msg_arg->ret_origin;
284         }
285 out:
286         tee_shm_free(shm);
287
288         return rc;
289 }
290
291 int optee_close_session(struct tee_context *ctx, u32 session)
292 {
293         struct optee_context_data *ctxdata = ctx->data;
294         struct tee_shm *shm;
295         struct optee_msg_arg *msg_arg;
296         phys_addr_t msg_parg;
297         struct optee_session *sess;
298
299         /* Check that the session is valid and remove it from the list */
300         mutex_lock(&ctxdata->mutex);
301         sess = find_session(ctxdata, session);
302         if (sess)
303                 list_del(&sess->list_node);
304         mutex_unlock(&ctxdata->mutex);
305         if (!sess)
306                 return -EINVAL;
307         kfree(sess);
308
309         shm = get_msg_arg(ctx, 0, &msg_arg, &msg_parg);
310         if (IS_ERR(shm))
311                 return PTR_ERR(shm);
312
313         msg_arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION;
314         msg_arg->session = session;
315         optee_do_call_with_arg(ctx, msg_parg);
316
317         tee_shm_free(shm);
318         return 0;
319 }
320
321 int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg,
322                       struct tee_param *param)
323 {
324         struct optee_context_data *ctxdata = ctx->data;
325         struct tee_shm *shm;
326         struct optee_msg_arg *msg_arg;
327         phys_addr_t msg_parg;
328         struct optee_session *sess;
329         int rc;
330
331         /* Check that the session is valid */
332         mutex_lock(&ctxdata->mutex);
333         sess = find_session(ctxdata, arg->session);
334         mutex_unlock(&ctxdata->mutex);
335         if (!sess)
336                 return -EINVAL;
337
338         shm = get_msg_arg(ctx, arg->num_params, &msg_arg, &msg_parg);
339         if (IS_ERR(shm))
340                 return PTR_ERR(shm);
341         msg_arg->cmd = OPTEE_MSG_CMD_INVOKE_COMMAND;
342         msg_arg->func = arg->func;
343         msg_arg->session = arg->session;
344         msg_arg->cancel_id = arg->cancel_id;
345
346         rc = optee_to_msg_param(msg_arg->params, arg->num_params, param);
347         if (rc)
348                 goto out;
349
350         if (optee_do_call_with_arg(ctx, msg_parg)) {
351                 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
352                 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
353         }
354
355         if (optee_from_msg_param(param, arg->num_params, msg_arg->params)) {
356                 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
357                 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
358         }
359
360         arg->ret = msg_arg->ret;
361         arg->ret_origin = msg_arg->ret_origin;
362 out:
363         tee_shm_free(shm);
364         return rc;
365 }
366
367 int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session)
368 {
369         struct optee_context_data *ctxdata = ctx->data;
370         struct tee_shm *shm;
371         struct optee_msg_arg *msg_arg;
372         phys_addr_t msg_parg;
373         struct optee_session *sess;
374
375         /* Check that the session is valid */
376         mutex_lock(&ctxdata->mutex);
377         sess = find_session(ctxdata, session);
378         mutex_unlock(&ctxdata->mutex);
379         if (!sess)
380                 return -EINVAL;
381
382         shm = get_msg_arg(ctx, 0, &msg_arg, &msg_parg);
383         if (IS_ERR(shm))
384                 return PTR_ERR(shm);
385
386         msg_arg->cmd = OPTEE_MSG_CMD_CANCEL;
387         msg_arg->session = session;
388         msg_arg->cancel_id = cancel_id;
389         optee_do_call_with_arg(ctx, msg_parg);
390
391         tee_shm_free(shm);
392         return 0;
393 }
394
395 /**
396  * optee_enable_shm_cache() - Enables caching of some shared memory allocation
397  *                            in OP-TEE
398  * @optee:      main service struct
399  */
400 void optee_enable_shm_cache(struct optee *optee)
401 {
402         struct optee_call_waiter w;
403
404         /* We need to retry until secure world isn't busy. */
405         optee_cq_wait_init(&optee->call_queue, &w);
406         while (true) {
407                 struct arm_smccc_res res;
408
409                 optee->invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE, 0, 0, 0, 0, 0, 0,
410                                  0, &res);
411                 if (res.a0 == OPTEE_SMC_RETURN_OK)
412                         break;
413                 optee_cq_wait_for_completion(&optee->call_queue, &w);
414         }
415         optee_cq_wait_final(&optee->call_queue, &w);
416 }
417
418 /**
419  * optee_disable_shm_cache() - Disables caching of some shared memory allocation
420  *                            in OP-TEE
421  * @optee:      main service struct
422  */
423 void optee_disable_shm_cache(struct optee *optee)
424 {
425         struct optee_call_waiter w;
426
427         /* We need to retry until secure world isn't busy. */
428         optee_cq_wait_init(&optee->call_queue, &w);
429         while (true) {
430                 union {
431                         struct arm_smccc_res smccc;
432                         struct optee_smc_disable_shm_cache_result result;
433                 } res;
434
435                 optee->invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE, 0, 0, 0, 0, 0, 0,
436                                  0, &res.smccc);
437                 if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)
438                         break; /* All shm's freed */
439                 if (res.result.status == OPTEE_SMC_RETURN_OK) {
440                         struct tee_shm *shm;
441
442                         shm = reg_pair_to_ptr(res.result.shm_upper32,
443                                               res.result.shm_lower32);
444                         tee_shm_free(shm);
445                 } else {
446                         optee_cq_wait_for_completion(&optee->call_queue, &w);
447                 }
448         }
449         optee_cq_wait_final(&optee->call_queue, &w);
450 }
451
452 #define PAGELIST_ENTRIES_PER_PAGE                               \
453         ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
454
455 /**
456  * optee_fill_pages_list() - write list of user pages to given shared
457  * buffer.
458  *
459  * @dst: page-aligned buffer where list of pages will be stored
460  * @pages: array of pages that represents shared buffer
461  * @num_pages: number of entries in @pages
462  * @page_offset: offset of user buffer from page start
463  *
464  * @dst should be big enough to hold list of user page addresses and
465  *      links to the next pages of buffer
466  */
467 void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,
468                            size_t page_offset)
469 {
470         int n = 0;
471         phys_addr_t optee_page;
472         /*
473          * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
474          * for details.
475          */
476         struct {
477                 u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];
478                 u64 next_page_data;
479         } *pages_data;
480
481         /*
482          * Currently OP-TEE uses 4k page size and it does not looks
483          * like this will change in the future.  On other hand, there are
484          * no know ARM architectures with page size < 4k.
485          * Thus the next built assert looks redundant. But the following
486          * code heavily relies on this assumption, so it is better be
487          * safe than sorry.
488          */
489         BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);
490
491         pages_data = (void *)dst;
492         /*
493          * If linux page is bigger than 4k, and user buffer offset is
494          * larger than 4k/8k/12k/etc this will skip first 4k pages,
495          * because they bear no value data for OP-TEE.
496          */
497         optee_page = page_to_phys(*pages) +
498                 round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
499
500         while (true) {
501                 pages_data->pages_list[n++] = optee_page;
502
503                 if (n == PAGELIST_ENTRIES_PER_PAGE) {
504                         pages_data->next_page_data =
505                                 virt_to_phys(pages_data + 1);
506                         pages_data++;
507                         n = 0;
508                 }
509
510                 optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;
511                 if (!(optee_page & ~PAGE_MASK)) {
512                         if (!--num_pages)
513                                 break;
514                         pages++;
515                         optee_page = page_to_phys(*pages);
516                 }
517         }
518 }
519
520 /*
521  * The final entry in each pagelist page is a pointer to the next
522  * pagelist page.
523  */
524 static size_t get_pages_list_size(size_t num_entries)
525 {
526         int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
527
528         return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
529 }
530
531 u64 *optee_allocate_pages_list(size_t num_entries)
532 {
533         return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL);
534 }
535
536 void optee_free_pages_list(void *list, size_t num_entries)
537 {
538         free_pages_exact(list, get_pages_list_size(num_entries));
539 }
540
541 static bool is_normal_memory(pgprot_t p)
542 {
543 #if defined(CONFIG_ARM)
544         return (((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEALLOC) ||
545                 ((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEBACK));
546 #elif defined(CONFIG_ARM64)
547         return (pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL);
548 #else
549 #error "Unuspported architecture"
550 #endif
551 }
552
553 static int __check_mem_type(struct vm_area_struct *vma, unsigned long end)
554 {
555         while (vma && is_normal_memory(vma->vm_page_prot)) {
556                 if (vma->vm_end >= end)
557                         return 0;
558                 vma = vma->vm_next;
559         }
560
561         return -EINVAL;
562 }
563
564 static int check_mem_type(unsigned long start, size_t num_pages)
565 {
566         struct mm_struct *mm = current->mm;
567         int rc;
568
569         /*
570          * Allow kernel address to register with OP-TEE as kernel
571          * pages are configured as normal memory only.
572          */
573         if (virt_addr_valid(start))
574                 return 0;
575
576         mmap_read_lock(mm);
577         rc = __check_mem_type(find_vma(mm, start),
578                               start + num_pages * PAGE_SIZE);
579         mmap_read_unlock(mm);
580
581         return rc;
582 }
583
584 int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
585                        struct page **pages, size_t num_pages,
586                        unsigned long start)
587 {
588         struct tee_shm *shm_arg = NULL;
589         struct optee_msg_arg *msg_arg;
590         u64 *pages_list;
591         phys_addr_t msg_parg;
592         int rc;
593
594         if (!num_pages)
595                 return -EINVAL;
596
597         rc = check_mem_type(start, num_pages);
598         if (rc)
599                 return rc;
600
601         pages_list = optee_allocate_pages_list(num_pages);
602         if (!pages_list)
603                 return -ENOMEM;
604
605         shm_arg = get_msg_arg(ctx, 1, &msg_arg, &msg_parg);
606         if (IS_ERR(shm_arg)) {
607                 rc = PTR_ERR(shm_arg);
608                 goto out;
609         }
610
611         optee_fill_pages_list(pages_list, pages, num_pages,
612                               tee_shm_get_page_offset(shm));
613
614         msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;
615         msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
616                                 OPTEE_MSG_ATTR_NONCONTIG;
617         msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;
618         msg_arg->params->u.tmem.size = tee_shm_get_size(shm);
619         /*
620          * In the least bits of msg_arg->params->u.tmem.buf_ptr we
621          * store buffer offset from 4k page, as described in OP-TEE ABI.
622          */
623         msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) |
624           (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
625
626         if (optee_do_call_with_arg(ctx, msg_parg) ||
627             msg_arg->ret != TEEC_SUCCESS)
628                 rc = -EINVAL;
629
630         tee_shm_free(shm_arg);
631 out:
632         optee_free_pages_list(pages_list, num_pages);
633         return rc;
634 }
635
636 int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
637 {
638         struct tee_shm *shm_arg;
639         struct optee_msg_arg *msg_arg;
640         phys_addr_t msg_parg;
641         int rc = 0;
642
643         shm_arg = get_msg_arg(ctx, 1, &msg_arg, &msg_parg);
644         if (IS_ERR(shm_arg))
645                 return PTR_ERR(shm_arg);
646
647         msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;
648
649         msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
650         msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
651
652         if (optee_do_call_with_arg(ctx, msg_parg) ||
653             msg_arg->ret != TEEC_SUCCESS)
654                 rc = -EINVAL;
655         tee_shm_free(shm_arg);
656         return rc;
657 }
658
659 int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,
660                             struct page **pages, size_t num_pages,
661                             unsigned long start)
662 {
663         /*
664          * We don't want to register supplicant memory in OP-TEE.
665          * Instead information about it will be passed in RPC code.
666          */
667         return check_mem_type(start, num_pages);
668 }
669
670 int optee_shm_unregister_supp(struct tee_context *ctx, struct tee_shm *shm)
671 {
672         return 0;
673 }