drivers/tee/tee_shm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2015-2017, 2019-2021 Linaro Limited
   4  */
   5 #include <linux/anon_inodes.h>
   6 #include <linux/device.h>
   7 #include <linux/idr.h>
   8 #include <linux/mm.h>
   9 #include <linux/sched.h>
  10 #include <linux/slab.h>
  11 #include <linux/tee_drv.h>
  12 #include <linux/uio.h>
  13 #include "tee_private.h"
  14
  15 static void shm_put_kernel_pages(struct page **pages, size_t page_count)
  16 {
  17         size_t n;
  18
  19         for (n = 0; n < page_count; n++)
  20                 put_page(pages[n]);
  21 }
  22
  23 static int shm_get_kernel_pages(unsigned long start, size_t page_count,
  24                                 struct page **pages)
  25 {
  26         size_t n;
  27         int rc;
  28
  29         if (is_vmalloc_addr((void *)start)) {
  30                 struct page *page;
  31
  32                 for (n = 0; n < page_count; n++) {
  33                         page = vmalloc_to_page((void *)(start + PAGE_SIZE * n));
  34                         if (!page)
  35                                 return -ENOMEM;
  36
  37                         get_page(page);
  38                         pages[n] = page;
  39                 }
  40                 rc = page_count;
  41         } else {
  42                 struct kvec *kiov;
  43
  44                 kiov = kcalloc(page_count, sizeof(*kiov), GFP_KERNEL);
  45                 if (!kiov)
  46                         return -ENOMEM;
  47
  48                 for (n = 0; n < page_count; n++) {
  49                         kiov[n].iov_base = (void *)(start + n * PAGE_SIZE);
  50                         kiov[n].iov_len = PAGE_SIZE;
  51                 }
  52
  53                 rc = get_kernel_pages(kiov, page_count, 0, pages);
  54                 kfree(kiov);
  55         }
  56
  57         return rc;
  58 }
  59
  60 static void release_registered_pages(struct tee_shm *shm)
  61 {
  62         if (shm->pages) {
  63                 if (shm->flags & TEE_SHM_USER_MAPPED)
  64                         unpin_user_pages(shm->pages, shm->num_pages);
  65                 else
  66                         shm_put_kernel_pages(shm->pages, shm->num_pages);
  67
  68                 kfree(shm->pages);
  69         }
  70 }
  71
  72 static void tee_shm_release(struct tee_device *teedev, struct tee_shm *shm)
  73 {
  74         if (shm->flags & TEE_SHM_POOL) {
  75                 teedev->pool->ops->free(teedev->pool, shm);
  76         } else if (shm->flags & TEE_SHM_DYNAMIC) {
  77                 int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);
  78
  79                 if (rc)
  80                         dev_err(teedev->dev.parent,
  81                                 "unregister shm %p failed: %d", shm, rc);
  82
  83                 release_registered_pages(shm);
  84         }
  85
  86         teedev_ctx_put(shm->ctx);
  87
  88         kfree(shm);
  89
  90         tee_device_put(teedev);
  91 }
  92
  93 static struct tee_shm *shm_alloc_helper(struct tee_context *ctx, size_t size,
  94                                         size_t align, u32 flags, int id)
  95 {
  96         struct tee_device *teedev = ctx->teedev;
  97         struct tee_shm *shm;
  98         void *ret;
  99         int rc;
 100
 101         if (!tee_device_get(teedev))
 102                 return ERR_PTR(-EINVAL);
 103
 104         if (!teedev->pool) {
 105                 /* teedev has been detached from driver */
 106                 ret = ERR_PTR(-EINVAL);
 107                 goto err_dev_put;
 108         }
 109
 110         shm = kzalloc(sizeof(*shm), GFP_KERNEL);
 111         if (!shm) {
 112                 ret = ERR_PTR(-ENOMEM);
 113                 goto err_dev_put;
 114         }
 115
 116         refcount_set(&shm->refcount, 1);
 117         shm->flags = flags;
 118         shm->id = id;
 119
 120         /*
 121          * We're assigning this as it is needed if the shm is to be
 122          * registered. If this function returns OK then the caller expected
 123          * to call teedev_ctx_get() or clear shm->ctx in case it's not
 124          * needed any longer.
 125          */
 126         shm->ctx = ctx;
 127
 128         rc = teedev->pool->ops->alloc(teedev->pool, shm, size, align);
 129         if (rc) {
 130                 ret = ERR_PTR(rc);
 131                 goto err_kfree;
 132         }
 133
 134         teedev_ctx_get(ctx);
 135         return shm;
 136 err_kfree:
 137         kfree(shm);
 138 err_dev_put:
 139         tee_device_put(teedev);
 140         return ret;
 141 }
 142
 143 /**
 144  * tee_shm_alloc_user_buf() - Allocate shared memory for user space
 145  * @ctx:        Context that allocates the shared memory
 146  * @size:       Requested size of shared memory
 147  *
 148  * Memory allocated as user space shared memory is automatically freed when
 149  * the TEE file pointer is closed. The primary usage of this function is
 150  * when the TEE driver doesn't support registering ordinary user space
 151  * memory.
 152  *
 153  * @returns a pointer to 'struct tee_shm'
 154  */
 155 struct tee_shm *tee_shm_alloc_user_buf(struct tee_context *ctx, size_t size)
 156 {
 157         u32 flags = TEE_SHM_DYNAMIC | TEE_SHM_POOL;
 158         struct tee_device *teedev = ctx->teedev;
 159         struct tee_shm *shm;
 160         void *ret;
 161         int id;
 162
 163         mutex_lock(&teedev->mutex);
 164         id = idr_alloc(&teedev->idr, NULL, 1, 0, GFP_KERNEL);
 165         mutex_unlock(&teedev->mutex);
 166         if (id < 0)
 167                 return ERR_PTR(id);
 168
 169         shm = shm_alloc_helper(ctx, size, PAGE_SIZE, flags, id);
 170         if (IS_ERR(shm)) {
 171                 mutex_lock(&teedev->mutex);
 172                 idr_remove(&teedev->idr, id);
 173                 mutex_unlock(&teedev->mutex);
 174                 return shm;
 175         }
 176
 177         mutex_lock(&teedev->mutex);
 178         ret = idr_replace(&teedev->idr, shm, id);
 179         mutex_unlock(&teedev->mutex);
 180         if (IS_ERR(ret)) {
 181                 tee_shm_free(shm);
 182                 return ret;
 183         }
 184
 185         return shm;
 186 }
 187
 188 /**
 189  * tee_shm_alloc_kernel_buf() - Allocate shared memory for kernel buffer
 190  * @ctx:        Context that allocates the shared memory
 191  * @size:       Requested size of shared memory
 192  *
 193  * The returned memory registered in secure world and is suitable to be
 194  * passed as a memory buffer in parameter argument to
 195  * tee_client_invoke_func(). The memory allocated is later freed with a
 196  * call to tee_shm_free().
 197  *
 198  * @returns a pointer to 'struct tee_shm'
 199  */
 200 struct tee_shm *tee_shm_alloc_kernel_buf(struct tee_context *ctx, size_t size)
 201 {
 202         u32 flags = TEE_SHM_DYNAMIC | TEE_SHM_POOL;
 203
 204         return shm_alloc_helper(ctx, size, PAGE_SIZE, flags, -1);
 205 }
 206 EXPORT_SYMBOL_GPL(tee_shm_alloc_kernel_buf);
 207
 208 /**
 209  * tee_shm_alloc_priv_buf() - Allocate shared memory for a privately shared
 210  *                            kernel buffer
 211  * @ctx:        Context that allocates the shared memory
 212  * @size:       Requested size of shared memory
 213  *
 214  * This function returns similar shared memory as
 215  * tee_shm_alloc_kernel_buf(), but with the difference that the memory
 216  * might not be registered in secure world in case the driver supports
 217  * passing memory not registered in advance.
 218  *
 219  * This function should normally only be used internally in the TEE
 220  * drivers.
 221  *
 222  * @returns a pointer to 'struct tee_shm'
 223  */
 224 struct tee_shm *tee_shm_alloc_priv_buf(struct tee_context *ctx, size_t size)
 225 {
 226         u32 flags = TEE_SHM_PRIV | TEE_SHM_POOL;
 227
 228         return shm_alloc_helper(ctx, size, sizeof(long) * 2, flags, -1);
 229 }
 230 EXPORT_SYMBOL_GPL(tee_shm_alloc_priv_buf);
 231
 232 static struct tee_shm *
 233 register_shm_helper(struct tee_context *ctx, unsigned long addr,
 234                     size_t length, u32 flags, int id)
 235 {
 236         struct tee_device *teedev = ctx->teedev;
 237         struct tee_shm *shm;
 238         unsigned long start;
 239         size_t num_pages;
 240         void *ret;
 241         int rc;
 242
 243         if (!tee_device_get(teedev))
 244                 return ERR_PTR(-EINVAL);
 245
 246         if (!teedev->desc->ops->shm_register ||
 247             !teedev->desc->ops->shm_unregister) {
 248                 ret = ERR_PTR(-ENOTSUPP);
 249                 goto err_dev_put;
 250         }
 251
 252         teedev_ctx_get(ctx);
 253
 254         shm = kzalloc(sizeof(*shm), GFP_KERNEL);
 255         if (!shm) {
 256                 ret = ERR_PTR(-ENOMEM);
 257                 goto err_ctx_put;
 258         }
 259
 260         refcount_set(&shm->refcount, 1);
 261         shm->flags = flags;
 262         shm->ctx = ctx;
 263         shm->id = id;
 264         addr = untagged_addr(addr);
 265         start = rounddown(addr, PAGE_SIZE);
 266         shm->offset = addr - start;
 267         shm->size = length;
 268         num_pages = (roundup(addr + length, PAGE_SIZE) - start) / PAGE_SIZE;
 269         shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);
 270         if (!shm->pages) {
 271                 ret = ERR_PTR(-ENOMEM);
 272                 goto err_free_shm;
 273         }
 274
 275         if (flags & TEE_SHM_USER_MAPPED)
 276                 rc = pin_user_pages_fast(start, num_pages, FOLL_WRITE,
 277                                          shm->pages);
 278         else
 279                 rc = shm_get_kernel_pages(start, num_pages, shm->pages);
 280         if (rc > 0)
 281                 shm->num_pages = rc;
 282         if (rc != num_pages) {
 283                 if (rc >= 0)
 284                         rc = -ENOMEM;
 285                 ret = ERR_PTR(rc);
 286                 goto err_put_shm_pages;
 287         }
 288
 289         rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,
 290                                              shm->num_pages, start);
 291         if (rc) {
 292                 ret = ERR_PTR(rc);
 293                 goto err_put_shm_pages;
 294         }
 295
 296         return shm;
 297 err_put_shm_pages:
 298         if (flags & TEE_SHM_USER_MAPPED)
 299                 unpin_user_pages(shm->pages, shm->num_pages);
 300         else
 301                 shm_put_kernel_pages(shm->pages, shm->num_pages);
 302         kfree(shm->pages);
 303 err_free_shm:
 304         kfree(shm);
 305 err_ctx_put:
 306         teedev_ctx_put(ctx);
 307 err_dev_put:
 308         tee_device_put(teedev);
 309         return ret;
 310 }
 311
 312 /**
 313  * tee_shm_register_user_buf() - Register a userspace shared memory buffer
 314  * @ctx:        Context that registers the shared memory
 315  * @addr:       The userspace address of the shared buffer
 316  * @length:     Length of the shared buffer
 317  *
 318  * @returns a pointer to 'struct tee_shm'
 319  */
 320 struct tee_shm *tee_shm_register_user_buf(struct tee_context *ctx,
 321                                           unsigned long addr, size_t length)
 322 {
 323         u32 flags = TEE_SHM_USER_MAPPED | TEE_SHM_DYNAMIC;
 324         struct tee_device *teedev = ctx->teedev;
 325         struct tee_shm *shm;
 326         void *ret;
 327         int id;
 328
 329         mutex_lock(&teedev->mutex);
 330         id = idr_alloc(&teedev->idr, NULL, 1, 0, GFP_KERNEL);
 331         mutex_unlock(&teedev->mutex);
 332         if (id < 0)
 333                 return ERR_PTR(id);
 334
 335         shm = register_shm_helper(ctx, addr, length, flags, id);
 336         if (IS_ERR(shm)) {
 337                 mutex_lock(&teedev->mutex);
 338                 idr_remove(&teedev->idr, id);
 339                 mutex_unlock(&teedev->mutex);
 340                 return shm;
 341         }
 342
 343         mutex_lock(&teedev->mutex);
 344         ret = idr_replace(&teedev->idr, shm, id);
 345         mutex_unlock(&teedev->mutex);
 346         if (IS_ERR(ret)) {
 347                 tee_shm_free(shm);
 348                 return ret;
 349         }
 350
 351         return shm;
 352 }
 353
 354 /**
 355  * tee_shm_register_kernel_buf() - Register kernel memory to be shared with
 356  *                                 secure world
 357  * @ctx:        Context that registers the shared memory
 358  * @addr:       The buffer
 359  * @length:     Length of the buffer
 360  *
 361  * @returns a pointer to 'struct tee_shm'
 362  */
 363
 364 struct tee_shm *tee_shm_register_kernel_buf(struct tee_context *ctx,
 365                                             void *addr, size_t length)
 366 {
 367         u32 flags = TEE_SHM_DYNAMIC;
 368
 369         return register_shm_helper(ctx, (unsigned long)addr, length, flags, -1);
 370 }
 371 EXPORT_SYMBOL_GPL(tee_shm_register_kernel_buf);
 372
 373 static int tee_shm_fop_release(struct inode *inode, struct file *filp)
 374 {
 375         tee_shm_put(filp->private_data);
 376         return 0;
 377 }
 378
 379 static int tee_shm_fop_mmap(struct file *filp, struct vm_area_struct *vma)
 380 {
 381         struct tee_shm *shm = filp->private_data;
 382         size_t size = vma->vm_end - vma->vm_start;
 383
 384         /* Refuse sharing shared memory provided by application */
 385         if (shm->flags & TEE_SHM_USER_MAPPED)
 386                 return -EINVAL;
 387
 388         /* check for overflowing the buffer's size */
 389         if (vma->vm_pgoff + vma_pages(vma) > shm->size >> PAGE_SHIFT)
 390                 return -EINVAL;
 391
 392         return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,
 393                                size, vma->vm_page_prot);
 394 }
 395
 396 static const struct file_operations tee_shm_fops = {
 397         .owner = THIS_MODULE,
 398         .release = tee_shm_fop_release,
 399         .mmap = tee_shm_fop_mmap,
 400 };
 401
 402 /**
 403  * tee_shm_get_fd() - Increase reference count and return file descriptor
 404  * @shm:        Shared memory handle
 405  * @returns user space file descriptor to shared memory
 406  */
 407 int tee_shm_get_fd(struct tee_shm *shm)
 408 {
 409         int fd;
 410
 411         if (shm->id < 0)
 412                 return -EINVAL;
 413
 414         /* matched by tee_shm_put() in tee_shm_op_release() */
 415         refcount_inc(&shm->refcount);
 416         fd = anon_inode_getfd("tee_shm", &tee_shm_fops, shm, O_RDWR);
 417         if (fd < 0)
 418                 tee_shm_put(shm);
 419         return fd;
 420 }
 421
 422 /**
 423  * tee_shm_free() - Free shared memory
 424  * @shm:        Handle to shared memory to free
 425  */
 426 void tee_shm_free(struct tee_shm *shm)
 427 {
 428         tee_shm_put(shm);
 429 }
 430 EXPORT_SYMBOL_GPL(tee_shm_free);
 431
 432 /**
 433  * tee_shm_get_va() - Get virtual address of a shared memory plus an offset
 434  * @shm:        Shared memory handle
 435  * @offs:       Offset from start of this shared memory
 436  * @returns virtual address of the shared memory + offs if offs is within
 437  *      the bounds of this shared memory, else an ERR_PTR
 438  */
 439 void *tee_shm_get_va(struct tee_shm *shm, size_t offs)
 440 {
 441         if (!shm->kaddr)
 442                 return ERR_PTR(-EINVAL);
 443         if (offs >= shm->size)
 444                 return ERR_PTR(-EINVAL);
 445         return (char *)shm->kaddr + offs;
 446 }
 447 EXPORT_SYMBOL_GPL(tee_shm_get_va);
 448
 449 /**
 450  * tee_shm_get_pa() - Get physical address of a shared memory plus an offset
 451  * @shm:        Shared memory handle
 452  * @offs:       Offset from start of this shared memory
 453  * @pa:         Physical address to return
 454  * @returns 0 if offs is within the bounds of this shared memory, else an
 455  *      error code.
 456  */
 457 int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)
 458 {
 459         if (offs >= shm->size)
 460                 return -EINVAL;
 461         if (pa)
 462                 *pa = shm->paddr + offs;
 463         return 0;
 464 }
 465 EXPORT_SYMBOL_GPL(tee_shm_get_pa);
 466
 467 /**
 468  * tee_shm_get_from_id() - Find shared memory object and increase reference
 469  * count
 470  * @ctx:        Context owning the shared memory
 471  * @id:         Id of shared memory object
 472  * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
 473  */
 474 struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id)
 475 {
 476         struct tee_device *teedev;
 477         struct tee_shm *shm;
 478
 479         if (!ctx)
 480                 return ERR_PTR(-EINVAL);
 481
 482         teedev = ctx->teedev;
 483         mutex_lock(&teedev->mutex);
 484         shm = idr_find(&teedev->idr, id);
 485         /*
 486          * If the tee_shm was found in the IDR it must have a refcount
 487          * larger than 0 due to the guarantee in tee_shm_put() below. So
 488          * it's safe to use refcount_inc().
 489          */
 490         if (!shm || shm->ctx != ctx)
 491                 shm = ERR_PTR(-EINVAL);
 492         else
 493                 refcount_inc(&shm->refcount);
 494         mutex_unlock(&teedev->mutex);
 495         return shm;
 496 }
 497 EXPORT_SYMBOL_GPL(tee_shm_get_from_id);
 498
 499 /**
 500  * tee_shm_put() - Decrease reference count on a shared memory handle
 501  * @shm:        Shared memory handle
 502  */
 503 void tee_shm_put(struct tee_shm *shm)
 504 {
 505         struct tee_device *teedev = shm->ctx->teedev;
 506         bool do_release = false;
 507
 508         mutex_lock(&teedev->mutex);
 509         if (refcount_dec_and_test(&shm->refcount)) {
 510                 /*
 511                  * refcount has reached 0, we must now remove it from the
 512                  * IDR before releasing the mutex. This will guarantee that
 513                  * the refcount_inc() in tee_shm_get_from_id() never starts
 514                  * from 0.
 515                  */
 516                 if (shm->id >= 0)
 517                         idr_remove(&teedev->idr, shm->id);
 518                 do_release = true;
 519         }
 520         mutex_unlock(&teedev->mutex);
 521
 522         if (do_release)
 523                 tee_shm_release(teedev, shm);
 524 }
 525 EXPORT_SYMBOL_GPL(tee_shm_put);