Merge tag 'omap-for-v5.13/fixes-sata' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6-microblaze.git] / drivers / tee / optee / call.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2015, Linaro Limited
4  */
5 #include <linux/arm-smccc.h>
6 #include <linux/device.h>
7 #include <linux/err.h>
8 #include <linux/errno.h>
9 #include <linux/mm.h>
10 #include <linux/sched.h>
11 #include <linux/slab.h>
12 #include <linux/tee_drv.h>
13 #include <linux/types.h>
14 #include <linux/uaccess.h>
15 #include "optee_private.h"
16 #include "optee_smc.h"
17 #define CREATE_TRACE_POINTS
18 #include "optee_trace.h"
19
20 struct optee_call_waiter {
21         struct list_head list_node;
22         struct completion c;
23 };
24
25 static void optee_cq_wait_init(struct optee_call_queue *cq,
26                                struct optee_call_waiter *w)
27 {
28         /*
29          * We're preparing to make a call to secure world. In case we can't
30          * allocate a thread in secure world we'll end up waiting in
31          * optee_cq_wait_for_completion().
32          *
33          * Normally if there's no contention in secure world the call will
34          * complete and we can cleanup directly with optee_cq_wait_final().
35          */
36         mutex_lock(&cq->mutex);
37
38         /*
39          * We add ourselves to the queue, but we don't wait. This
40          * guarantees that we don't lose a completion if secure world
41          * returns busy and another thread just exited and try to complete
42          * someone.
43          */
44         init_completion(&w->c);
45         list_add_tail(&w->list_node, &cq->waiters);
46
47         mutex_unlock(&cq->mutex);
48 }
49
50 static void optee_cq_wait_for_completion(struct optee_call_queue *cq,
51                                          struct optee_call_waiter *w)
52 {
53         wait_for_completion(&w->c);
54
55         mutex_lock(&cq->mutex);
56
57         /* Move to end of list to get out of the way for other waiters */
58         list_del(&w->list_node);
59         reinit_completion(&w->c);
60         list_add_tail(&w->list_node, &cq->waiters);
61
62         mutex_unlock(&cq->mutex);
63 }
64
65 static void optee_cq_complete_one(struct optee_call_queue *cq)
66 {
67         struct optee_call_waiter *w;
68
69         list_for_each_entry(w, &cq->waiters, list_node) {
70                 if (!completion_done(&w->c)) {
71                         complete(&w->c);
72                         break;
73                 }
74         }
75 }
76
77 static void optee_cq_wait_final(struct optee_call_queue *cq,
78                                 struct optee_call_waiter *w)
79 {
80         /*
81          * We're done with the call to secure world. The thread in secure
82          * world that was used for this call is now available for some
83          * other task to use.
84          */
85         mutex_lock(&cq->mutex);
86
87         /* Get out of the list */
88         list_del(&w->list_node);
89
90         /* Wake up one eventual waiting task */
91         optee_cq_complete_one(cq);
92
93         /*
94          * If we're completed we've got a completion from another task that
95          * was just done with its call to secure world. Since yet another
96          * thread now is available in secure world wake up another eventual
97          * waiting task.
98          */
99         if (completion_done(&w->c))
100                 optee_cq_complete_one(cq);
101
102         mutex_unlock(&cq->mutex);
103 }
104
105 /* Requires the filpstate mutex to be held */
106 static struct optee_session *find_session(struct optee_context_data *ctxdata,
107                                           u32 session_id)
108 {
109         struct optee_session *sess;
110
111         list_for_each_entry(sess, &ctxdata->sess_list, list_node)
112                 if (sess->session_id == session_id)
113                         return sess;
114
115         return NULL;
116 }
117
118 /**
119  * optee_do_call_with_arg() - Do an SMC to OP-TEE in secure world
120  * @ctx:        calling context
121  * @parg:       physical address of message to pass to secure world
122  *
123  * Does and SMC to OP-TEE in secure world and handles eventual resulting
124  * Remote Procedure Calls (RPC) from OP-TEE.
125  *
126  * Returns return code from secure world, 0 is OK
127  */
128 u32 optee_do_call_with_arg(struct tee_context *ctx, phys_addr_t parg)
129 {
130         struct optee *optee = tee_get_drvdata(ctx->teedev);
131         struct optee_call_waiter w;
132         struct optee_rpc_param param = { };
133         struct optee_call_ctx call_ctx = { };
134         u32 ret;
135
136         param.a0 = OPTEE_SMC_CALL_WITH_ARG;
137         reg_pair_from_64(&param.a1, &param.a2, parg);
138         /* Initialize waiter */
139         optee_cq_wait_init(&optee->call_queue, &w);
140         while (true) {
141                 struct arm_smccc_res res;
142
143                 trace_optee_invoke_fn_begin(&param);
144                 optee->invoke_fn(param.a0, param.a1, param.a2, param.a3,
145                                  param.a4, param.a5, param.a6, param.a7,
146                                  &res);
147                 trace_optee_invoke_fn_end(&param, &res);
148
149                 if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {
150                         /*
151                          * Out of threads in secure world, wait for a thread
152                          * become available.
153                          */
154                         optee_cq_wait_for_completion(&optee->call_queue, &w);
155                 } else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {
156                         cond_resched();
157                         param.a0 = res.a0;
158                         param.a1 = res.a1;
159                         param.a2 = res.a2;
160                         param.a3 = res.a3;
161                         optee_handle_rpc(ctx, &param, &call_ctx);
162                 } else {
163                         ret = res.a0;
164                         break;
165                 }
166         }
167
168         optee_rpc_finalize_call(&call_ctx);
169         /*
170          * We're done with our thread in secure world, if there's any
171          * thread waiters wake up one.
172          */
173         optee_cq_wait_final(&optee->call_queue, &w);
174
175         return ret;
176 }
177
178 static struct tee_shm *get_msg_arg(struct tee_context *ctx, size_t num_params,
179                                    struct optee_msg_arg **msg_arg,
180                                    phys_addr_t *msg_parg)
181 {
182         int rc;
183         struct tee_shm *shm;
184         struct optee_msg_arg *ma;
185
186         shm = tee_shm_alloc(ctx, OPTEE_MSG_GET_ARG_SIZE(num_params),
187                             TEE_SHM_MAPPED);
188         if (IS_ERR(shm))
189                 return shm;
190
191         ma = tee_shm_get_va(shm, 0);
192         if (IS_ERR(ma)) {
193                 rc = PTR_ERR(ma);
194                 goto out;
195         }
196
197         rc = tee_shm_get_pa(shm, 0, msg_parg);
198         if (rc)
199                 goto out;
200
201         memset(ma, 0, OPTEE_MSG_GET_ARG_SIZE(num_params));
202         ma->num_params = num_params;
203         *msg_arg = ma;
204 out:
205         if (rc) {
206                 tee_shm_free(shm);
207                 return ERR_PTR(rc);
208         }
209
210         return shm;
211 }
212
213 int optee_open_session(struct tee_context *ctx,
214                        struct tee_ioctl_open_session_arg *arg,
215                        struct tee_param *param)
216 {
217         struct optee_context_data *ctxdata = ctx->data;
218         int rc;
219         struct tee_shm *shm;
220         struct optee_msg_arg *msg_arg;
221         phys_addr_t msg_parg;
222         struct optee_session *sess = NULL;
223
224         /* +2 for the meta parameters added below */
225         shm = get_msg_arg(ctx, arg->num_params + 2, &msg_arg, &msg_parg);
226         if (IS_ERR(shm))
227                 return PTR_ERR(shm);
228
229         msg_arg->cmd = OPTEE_MSG_CMD_OPEN_SESSION;
230         msg_arg->cancel_id = arg->cancel_id;
231
232         /*
233          * Initialize and add the meta parameters needed when opening a
234          * session.
235          */
236         msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |
237                                   OPTEE_MSG_ATTR_META;
238         msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |
239                                   OPTEE_MSG_ATTR_META;
240         memcpy(&msg_arg->params[0].u.value, arg->uuid, sizeof(arg->uuid));
241         msg_arg->params[1].u.value.c = arg->clnt_login;
242
243         rc = tee_session_calc_client_uuid((uuid_t *)&msg_arg->params[1].u.value,
244                                           arg->clnt_login, arg->clnt_uuid);
245         if (rc)
246                 goto out;
247
248         rc = optee_to_msg_param(msg_arg->params + 2, arg->num_params, param);
249         if (rc)
250                 goto out;
251
252         sess = kzalloc(sizeof(*sess), GFP_KERNEL);
253         if (!sess) {
254                 rc = -ENOMEM;
255                 goto out;
256         }
257
258         if (optee_do_call_with_arg(ctx, msg_parg)) {
259                 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
260                 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
261         }
262
263         if (msg_arg->ret == TEEC_SUCCESS) {
264                 /* A new session has been created, add it to the list. */
265                 sess->session_id = msg_arg->session;
266                 mutex_lock(&ctxdata->mutex);
267                 list_add(&sess->list_node, &ctxdata->sess_list);
268                 mutex_unlock(&ctxdata->mutex);
269         } else {
270                 kfree(sess);
271         }
272
273         if (optee_from_msg_param(param, arg->num_params, msg_arg->params + 2)) {
274                 arg->ret = TEEC_ERROR_COMMUNICATION;
275                 arg->ret_origin = TEEC_ORIGIN_COMMS;
276                 /* Close session again to avoid leakage */
277                 optee_close_session(ctx, msg_arg->session);
278         } else {
279                 arg->session = msg_arg->session;
280                 arg->ret = msg_arg->ret;
281                 arg->ret_origin = msg_arg->ret_origin;
282         }
283 out:
284         tee_shm_free(shm);
285
286         return rc;
287 }
288
289 int optee_close_session(struct tee_context *ctx, u32 session)
290 {
291         struct optee_context_data *ctxdata = ctx->data;
292         struct tee_shm *shm;
293         struct optee_msg_arg *msg_arg;
294         phys_addr_t msg_parg;
295         struct optee_session *sess;
296
297         /* Check that the session is valid and remove it from the list */
298         mutex_lock(&ctxdata->mutex);
299         sess = find_session(ctxdata, session);
300         if (sess)
301                 list_del(&sess->list_node);
302         mutex_unlock(&ctxdata->mutex);
303         if (!sess)
304                 return -EINVAL;
305         kfree(sess);
306
307         shm = get_msg_arg(ctx, 0, &msg_arg, &msg_parg);
308         if (IS_ERR(shm))
309                 return PTR_ERR(shm);
310
311         msg_arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION;
312         msg_arg->session = session;
313         optee_do_call_with_arg(ctx, msg_parg);
314
315         tee_shm_free(shm);
316         return 0;
317 }
318
319 int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg,
320                       struct tee_param *param)
321 {
322         struct optee_context_data *ctxdata = ctx->data;
323         struct tee_shm *shm;
324         struct optee_msg_arg *msg_arg;
325         phys_addr_t msg_parg;
326         struct optee_session *sess;
327         int rc;
328
329         /* Check that the session is valid */
330         mutex_lock(&ctxdata->mutex);
331         sess = find_session(ctxdata, arg->session);
332         mutex_unlock(&ctxdata->mutex);
333         if (!sess)
334                 return -EINVAL;
335
336         shm = get_msg_arg(ctx, arg->num_params, &msg_arg, &msg_parg);
337         if (IS_ERR(shm))
338                 return PTR_ERR(shm);
339         msg_arg->cmd = OPTEE_MSG_CMD_INVOKE_COMMAND;
340         msg_arg->func = arg->func;
341         msg_arg->session = arg->session;
342         msg_arg->cancel_id = arg->cancel_id;
343
344         rc = optee_to_msg_param(msg_arg->params, arg->num_params, param);
345         if (rc)
346                 goto out;
347
348         if (optee_do_call_with_arg(ctx, msg_parg)) {
349                 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
350                 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
351         }
352
353         if (optee_from_msg_param(param, arg->num_params, msg_arg->params)) {
354                 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
355                 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
356         }
357
358         arg->ret = msg_arg->ret;
359         arg->ret_origin = msg_arg->ret_origin;
360 out:
361         tee_shm_free(shm);
362         return rc;
363 }
364
365 int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session)
366 {
367         struct optee_context_data *ctxdata = ctx->data;
368         struct tee_shm *shm;
369         struct optee_msg_arg *msg_arg;
370         phys_addr_t msg_parg;
371         struct optee_session *sess;
372
373         /* Check that the session is valid */
374         mutex_lock(&ctxdata->mutex);
375         sess = find_session(ctxdata, session);
376         mutex_unlock(&ctxdata->mutex);
377         if (!sess)
378                 return -EINVAL;
379
380         shm = get_msg_arg(ctx, 0, &msg_arg, &msg_parg);
381         if (IS_ERR(shm))
382                 return PTR_ERR(shm);
383
384         msg_arg->cmd = OPTEE_MSG_CMD_CANCEL;
385         msg_arg->session = session;
386         msg_arg->cancel_id = cancel_id;
387         optee_do_call_with_arg(ctx, msg_parg);
388
389         tee_shm_free(shm);
390         return 0;
391 }
392
393 /**
394  * optee_enable_shm_cache() - Enables caching of some shared memory allocation
395  *                            in OP-TEE
396  * @optee:      main service struct
397  */
398 void optee_enable_shm_cache(struct optee *optee)
399 {
400         struct optee_call_waiter w;
401
402         /* We need to retry until secure world isn't busy. */
403         optee_cq_wait_init(&optee->call_queue, &w);
404         while (true) {
405                 struct arm_smccc_res res;
406
407                 optee->invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE, 0, 0, 0, 0, 0, 0,
408                                  0, &res);
409                 if (res.a0 == OPTEE_SMC_RETURN_OK)
410                         break;
411                 optee_cq_wait_for_completion(&optee->call_queue, &w);
412         }
413         optee_cq_wait_final(&optee->call_queue, &w);
414 }
415
416 /**
417  * optee_disable_shm_cache() - Disables caching of some shared memory allocation
418  *                            in OP-TEE
419  * @optee:      main service struct
420  */
421 void optee_disable_shm_cache(struct optee *optee)
422 {
423         struct optee_call_waiter w;
424
425         /* We need to retry until secure world isn't busy. */
426         optee_cq_wait_init(&optee->call_queue, &w);
427         while (true) {
428                 union {
429                         struct arm_smccc_res smccc;
430                         struct optee_smc_disable_shm_cache_result result;
431                 } res;
432
433                 optee->invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE, 0, 0, 0, 0, 0, 0,
434                                  0, &res.smccc);
435                 if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)
436                         break; /* All shm's freed */
437                 if (res.result.status == OPTEE_SMC_RETURN_OK) {
438                         struct tee_shm *shm;
439
440                         shm = reg_pair_to_ptr(res.result.shm_upper32,
441                                               res.result.shm_lower32);
442                         tee_shm_free(shm);
443                 } else {
444                         optee_cq_wait_for_completion(&optee->call_queue, &w);
445                 }
446         }
447         optee_cq_wait_final(&optee->call_queue, &w);
448 }
449
450 #define PAGELIST_ENTRIES_PER_PAGE                               \
451         ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
452
453 /**
454  * optee_fill_pages_list() - write list of user pages to given shared
455  * buffer.
456  *
457  * @dst: page-aligned buffer where list of pages will be stored
458  * @pages: array of pages that represents shared buffer
459  * @num_pages: number of entries in @pages
460  * @page_offset: offset of user buffer from page start
461  *
462  * @dst should be big enough to hold list of user page addresses and
463  *      links to the next pages of buffer
464  */
465 void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,
466                            size_t page_offset)
467 {
468         int n = 0;
469         phys_addr_t optee_page;
470         /*
471          * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
472          * for details.
473          */
474         struct {
475                 u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];
476                 u64 next_page_data;
477         } *pages_data;
478
479         /*
480          * Currently OP-TEE uses 4k page size and it does not looks
481          * like this will change in the future.  On other hand, there are
482          * no know ARM architectures with page size < 4k.
483          * Thus the next built assert looks redundant. But the following
484          * code heavily relies on this assumption, so it is better be
485          * safe than sorry.
486          */
487         BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);
488
489         pages_data = (void *)dst;
490         /*
491          * If linux page is bigger than 4k, and user buffer offset is
492          * larger than 4k/8k/12k/etc this will skip first 4k pages,
493          * because they bear no value data for OP-TEE.
494          */
495         optee_page = page_to_phys(*pages) +
496                 round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
497
498         while (true) {
499                 pages_data->pages_list[n++] = optee_page;
500
501                 if (n == PAGELIST_ENTRIES_PER_PAGE) {
502                         pages_data->next_page_data =
503                                 virt_to_phys(pages_data + 1);
504                         pages_data++;
505                         n = 0;
506                 }
507
508                 optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;
509                 if (!(optee_page & ~PAGE_MASK)) {
510                         if (!--num_pages)
511                                 break;
512                         pages++;
513                         optee_page = page_to_phys(*pages);
514                 }
515         }
516 }
517
518 /*
519  * The final entry in each pagelist page is a pointer to the next
520  * pagelist page.
521  */
522 static size_t get_pages_list_size(size_t num_entries)
523 {
524         int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
525
526         return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
527 }
528
529 u64 *optee_allocate_pages_list(size_t num_entries)
530 {
531         return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL);
532 }
533
534 void optee_free_pages_list(void *list, size_t num_entries)
535 {
536         free_pages_exact(list, get_pages_list_size(num_entries));
537 }
538
539 static bool is_normal_memory(pgprot_t p)
540 {
541 #if defined(CONFIG_ARM)
542         return (((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEALLOC) ||
543                 ((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEBACK));
544 #elif defined(CONFIG_ARM64)
545         return (pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL);
546 #else
547 #error "Unuspported architecture"
548 #endif
549 }
550
551 static int __check_mem_type(struct vm_area_struct *vma, unsigned long end)
552 {
553         while (vma && is_normal_memory(vma->vm_page_prot)) {
554                 if (vma->vm_end >= end)
555                         return 0;
556                 vma = vma->vm_next;
557         }
558
559         return -EINVAL;
560 }
561
562 static int check_mem_type(unsigned long start, size_t num_pages)
563 {
564         struct mm_struct *mm = current->mm;
565         int rc;
566
567         /*
568          * Allow kernel address to register with OP-TEE as kernel
569          * pages are configured as normal memory only.
570          */
571         if (virt_addr_valid(start))
572                 return 0;
573
574         mmap_read_lock(mm);
575         rc = __check_mem_type(find_vma(mm, start),
576                               start + num_pages * PAGE_SIZE);
577         mmap_read_unlock(mm);
578
579         return rc;
580 }
581
582 int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
583                        struct page **pages, size_t num_pages,
584                        unsigned long start)
585 {
586         struct tee_shm *shm_arg = NULL;
587         struct optee_msg_arg *msg_arg;
588         u64 *pages_list;
589         phys_addr_t msg_parg;
590         int rc;
591
592         if (!num_pages)
593                 return -EINVAL;
594
595         rc = check_mem_type(start, num_pages);
596         if (rc)
597                 return rc;
598
599         pages_list = optee_allocate_pages_list(num_pages);
600         if (!pages_list)
601                 return -ENOMEM;
602
603         shm_arg = get_msg_arg(ctx, 1, &msg_arg, &msg_parg);
604         if (IS_ERR(shm_arg)) {
605                 rc = PTR_ERR(shm_arg);
606                 goto out;
607         }
608
609         optee_fill_pages_list(pages_list, pages, num_pages,
610                               tee_shm_get_page_offset(shm));
611
612         msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;
613         msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
614                                 OPTEE_MSG_ATTR_NONCONTIG;
615         msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;
616         msg_arg->params->u.tmem.size = tee_shm_get_size(shm);
617         /*
618          * In the least bits of msg_arg->params->u.tmem.buf_ptr we
619          * store buffer offset from 4k page, as described in OP-TEE ABI.
620          */
621         msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) |
622           (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
623
624         if (optee_do_call_with_arg(ctx, msg_parg) ||
625             msg_arg->ret != TEEC_SUCCESS)
626                 rc = -EINVAL;
627
628         tee_shm_free(shm_arg);
629 out:
630         optee_free_pages_list(pages_list, num_pages);
631         return rc;
632 }
633
634 int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
635 {
636         struct tee_shm *shm_arg;
637         struct optee_msg_arg *msg_arg;
638         phys_addr_t msg_parg;
639         int rc = 0;
640
641         shm_arg = get_msg_arg(ctx, 1, &msg_arg, &msg_parg);
642         if (IS_ERR(shm_arg))
643                 return PTR_ERR(shm_arg);
644
645         msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;
646
647         msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
648         msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
649
650         if (optee_do_call_with_arg(ctx, msg_parg) ||
651             msg_arg->ret != TEEC_SUCCESS)
652                 rc = -EINVAL;
653         tee_shm_free(shm_arg);
654         return rc;
655 }
656
657 int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,
658                             struct page **pages, size_t num_pages,
659                             unsigned long start)
660 {
661         /*
662          * We don't want to register supplicant memory in OP-TEE.
663          * Instead information about it will be passed in RPC code.
664          */
665         return check_mem_type(start, num_pages);
666 }
667
668 int optee_shm_unregister_supp(struct tee_context *ctx, struct tee_shm *shm)
669 {
670         return 0;
671 }