drivers/dma-buf/heaps/system_heap.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * DMABUF System heap exporter
   4  *
   5  * Copyright (C) 2011 Google, Inc.
   6  * Copyright (C) 2019, 2020 Linaro Ltd.
   7  *
   8  * Portions based off of Andrew Davis' SRAM heap:
   9  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
  10  *      Andrew F. Davis <afd@ti.com>
  11  */
  12
  13 #include <linux/dma-buf.h>
  14 #include <linux/dma-mapping.h>
  15 #include <linux/dma-heap.h>
  16 #include <linux/err.h>
  17 #include <linux/highmem.h>
  18 #include <linux/mm.h>
  19 #include <linux/module.h>
  20 #include <linux/scatterlist.h>
  21 #include <linux/slab.h>
  22 #include <linux/vmalloc.h>
  23
  24 static struct dma_heap *sys_heap;
  25
  26 struct system_heap_buffer {
  27         struct dma_heap *heap;
  28         struct list_head attachments;
  29         struct mutex lock;
  30         unsigned long len;
  31         struct sg_table sg_table;
  32         int vmap_cnt;
  33         void *vaddr;
  34 };
  35
  36 struct dma_heap_attachment {
  37         struct device *dev;
  38         struct sg_table *table;
  39         struct list_head list;
  40         bool mapped;
  41 };
  42
  43 #define HIGH_ORDER_GFP  (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \
  44                                 | __GFP_NORETRY) & ~__GFP_RECLAIM) \
  45                                 | __GFP_COMP)
  46 #define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP)
  47 static gfp_t order_flags[] = {HIGH_ORDER_GFP, LOW_ORDER_GFP, LOW_ORDER_GFP};
  48 /*
  49  * The selection of the orders used for allocation (1MB, 64K, 4K) is designed
  50  * to match with the sizes often found in IOMMUs. Using order 4 pages instead
  51  * of order 0 pages can significantly improve the performance of many IOMMUs
  52  * by reducing TLB pressure and time spent updating page tables.
  53  */
  54 static const unsigned int orders[] = {8, 4, 0};
  55 #define NUM_ORDERS ARRAY_SIZE(orders)
  56
  57 static struct sg_table *dup_sg_table(struct sg_table *table)
  58 {
  59         struct sg_table *new_table;
  60         int ret, i;
  61         struct scatterlist *sg, *new_sg;
  62
  63         new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
  64         if (!new_table)
  65                 return ERR_PTR(-ENOMEM);
  66
  67         ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
  68         if (ret) {
  69                 kfree(new_table);
  70                 return ERR_PTR(-ENOMEM);
  71         }
  72
  73         new_sg = new_table->sgl;
  74         for_each_sgtable_sg(table, sg, i) {
  75                 sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
  76                 new_sg = sg_next(new_sg);
  77         }
  78
  79         return new_table;
  80 }
  81
  82 static int system_heap_attach(struct dma_buf *dmabuf,
  83                               struct dma_buf_attachment *attachment)
  84 {
  85         struct system_heap_buffer *buffer = dmabuf->priv;
  86         struct dma_heap_attachment *a;
  87         struct sg_table *table;
  88
  89         a = kzalloc(sizeof(*a), GFP_KERNEL);
  90         if (!a)
  91                 return -ENOMEM;
  92
  93         table = dup_sg_table(&buffer->sg_table);
  94         if (IS_ERR(table)) {
  95                 kfree(a);
  96                 return -ENOMEM;
  97         }
  98
  99         a->table = table;
 100         a->dev = attachment->dev;
 101         INIT_LIST_HEAD(&a->list);
 102         a->mapped = false;
 103
 104         attachment->priv = a;
 105
 106         mutex_lock(&buffer->lock);
 107         list_add(&a->list, &buffer->attachments);
 108         mutex_unlock(&buffer->lock);
 109
 110         return 0;
 111 }
 112
 113 static void system_heap_detach(struct dma_buf *dmabuf,
 114                                struct dma_buf_attachment *attachment)
 115 {
 116         struct system_heap_buffer *buffer = dmabuf->priv;
 117         struct dma_heap_attachment *a = attachment->priv;
 118
 119         mutex_lock(&buffer->lock);
 120         list_del(&a->list);
 121         mutex_unlock(&buffer->lock);
 122
 123         sg_free_table(a->table);
 124         kfree(a->table);
 125         kfree(a);
 126 }
 127
 128 static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment,
 129                                                 enum dma_data_direction direction)
 130 {
 131         struct dma_heap_attachment *a = attachment->priv;
 132         struct sg_table *table = a->table;
 133         int ret;
 134
 135         ret = dma_map_sgtable(attachment->dev, table, direction, 0);
 136         if (ret)
 137                 return ERR_PTR(ret);
 138
 139         a->mapped = true;
 140         return table;
 141 }
 142
 143 static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
 144                                       struct sg_table *table,
 145                                       enum dma_data_direction direction)
 146 {
 147         struct dma_heap_attachment *a = attachment->priv;
 148
 149         a->mapped = false;
 150         dma_unmap_sgtable(attachment->dev, table, direction, 0);
 151 }
 152
 153 static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
 154                                                 enum dma_data_direction direction)
 155 {
 156         struct system_heap_buffer *buffer = dmabuf->priv;
 157         struct dma_heap_attachment *a;
 158
 159         mutex_lock(&buffer->lock);
 160
 161         if (buffer->vmap_cnt)
 162                 invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
 163
 164         list_for_each_entry(a, &buffer->attachments, list) {
 165                 if (!a->mapped)
 166                         continue;
 167                 dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
 168         }
 169         mutex_unlock(&buffer->lock);
 170
 171         return 0;
 172 }
 173
 174 static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
 175                                               enum dma_data_direction direction)
 176 {
 177         struct system_heap_buffer *buffer = dmabuf->priv;
 178         struct dma_heap_attachment *a;
 179
 180         mutex_lock(&buffer->lock);
 181
 182         if (buffer->vmap_cnt)
 183                 flush_kernel_vmap_range(buffer->vaddr, buffer->len);
 184
 185         list_for_each_entry(a, &buffer->attachments, list) {
 186                 if (!a->mapped)
 187                         continue;
 188                 dma_sync_sgtable_for_device(a->dev, a->table, direction);
 189         }
 190         mutex_unlock(&buffer->lock);
 191
 192         return 0;
 193 }
 194
 195 static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
 196 {
 197         struct system_heap_buffer *buffer = dmabuf->priv;
 198         struct sg_table *table = &buffer->sg_table;
 199         unsigned long addr = vma->vm_start;
 200         struct sg_page_iter piter;
 201         int ret;
 202
 203         for_each_sgtable_page(table, &piter, vma->vm_pgoff) {
 204                 struct page *page = sg_page_iter_page(&piter);
 205
 206                 ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
 207                                       vma->vm_page_prot);
 208                 if (ret)
 209                         return ret;
 210                 addr += PAGE_SIZE;
 211                 if (addr >= vma->vm_end)
 212                         return 0;
 213         }
 214         return 0;
 215 }
 216
 217 static void *system_heap_do_vmap(struct system_heap_buffer *buffer)
 218 {
 219         struct sg_table *table = &buffer->sg_table;
 220         int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
 221         struct page **pages = vmalloc(sizeof(struct page *) * npages);
 222         struct page **tmp = pages;
 223         struct sg_page_iter piter;
 224         void *vaddr;
 225
 226         if (!pages)
 227                 return ERR_PTR(-ENOMEM);
 228
 229         for_each_sgtable_page(table, &piter, 0) {
 230                 WARN_ON(tmp - pages >= npages);
 231                 *tmp++ = sg_page_iter_page(&piter);
 232         }
 233
 234         vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
 235         vfree(pages);
 236
 237         if (!vaddr)
 238                 return ERR_PTR(-ENOMEM);
 239
 240         return vaddr;
 241 }
 242
 243 static int system_heap_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
 244 {
 245         struct system_heap_buffer *buffer = dmabuf->priv;
 246         void *vaddr;
 247         int ret = 0;
 248
 249         mutex_lock(&buffer->lock);
 250         if (buffer->vmap_cnt) {
 251                 buffer->vmap_cnt++;
 252                 dma_buf_map_set_vaddr(map, buffer->vaddr);
 253                 goto out;
 254         }
 255
 256         vaddr = system_heap_do_vmap(buffer);
 257         if (IS_ERR(vaddr)) {
 258                 ret = PTR_ERR(vaddr);
 259                 goto out;
 260         }
 261
 262         buffer->vaddr = vaddr;
 263         buffer->vmap_cnt++;
 264         dma_buf_map_set_vaddr(map, buffer->vaddr);
 265 out:
 266         mutex_unlock(&buffer->lock);
 267
 268         return ret;
 269 }
 270
 271 static void system_heap_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
 272 {
 273         struct system_heap_buffer *buffer = dmabuf->priv;
 274
 275         mutex_lock(&buffer->lock);
 276         if (!--buffer->vmap_cnt) {
 277                 vunmap(buffer->vaddr);
 278                 buffer->vaddr = NULL;
 279         }
 280         mutex_unlock(&buffer->lock);
 281         dma_buf_map_clear(map);
 282 }
 283
 284 static void system_heap_dma_buf_release(struct dma_buf *dmabuf)
 285 {
 286         struct system_heap_buffer *buffer = dmabuf->priv;
 287         struct sg_table *table;
 288         struct scatterlist *sg;
 289         int i;
 290
 291         table = &buffer->sg_table;
 292         for_each_sg(table->sgl, sg, table->nents, i) {
 293                 struct page *page = sg_page(sg);
 294
 295                 __free_pages(page, compound_order(page));
 296         }
 297         sg_free_table(table);
 298         kfree(buffer);
 299 }
 300
 301 static const struct dma_buf_ops system_heap_buf_ops = {
 302         .attach = system_heap_attach,
 303         .detach = system_heap_detach,
 304         .map_dma_buf = system_heap_map_dma_buf,
 305         .unmap_dma_buf = system_heap_unmap_dma_buf,
 306         .begin_cpu_access = system_heap_dma_buf_begin_cpu_access,
 307         .end_cpu_access = system_heap_dma_buf_end_cpu_access,
 308         .mmap = system_heap_mmap,
 309         .vmap = system_heap_vmap,
 310         .vunmap = system_heap_vunmap,
 311         .release = system_heap_dma_buf_release,
 312 };
 313
 314 static struct page *alloc_largest_available(unsigned long size,
 315                                             unsigned int max_order)
 316 {
 317         struct page *page;
 318         int i;
 319
 320         for (i = 0; i < NUM_ORDERS; i++) {
 321                 if (size <  (PAGE_SIZE << orders[i]))
 322                         continue;
 323                 if (max_order < orders[i])
 324                         continue;
 325
 326                 page = alloc_pages(order_flags[i], orders[i]);
 327                 if (!page)
 328                         continue;
 329                 return page;
 330         }
 331         return NULL;
 332 }
 333
 334 static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
 335                                             unsigned long len,
 336                                             unsigned long fd_flags,
 337                                             unsigned long heap_flags)
 338 {
 339         struct system_heap_buffer *buffer;
 340         DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 341         unsigned long size_remaining = len;
 342         unsigned int max_order = orders[0];
 343         struct dma_buf *dmabuf;
 344         struct sg_table *table;
 345         struct scatterlist *sg;
 346         struct list_head pages;
 347         struct page *page, *tmp_page;
 348         int i, ret = -ENOMEM;
 349
 350         buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
 351         if (!buffer)
 352                 return ERR_PTR(-ENOMEM);
 353
 354         INIT_LIST_HEAD(&buffer->attachments);
 355         mutex_init(&buffer->lock);
 356         buffer->heap = heap;
 357         buffer->len = len;
 358
 359         INIT_LIST_HEAD(&pages);
 360         i = 0;
 361         while (size_remaining > 0) {
 362                 /*
 363                  * Avoid trying to allocate memory if the process
 364                  * has been killed by SIGKILL
 365                  */
 366                 if (fatal_signal_pending(current)) {
 367                         ret = -EINTR;
 368                         goto free_buffer;
 369                 }
 370
 371                 page = alloc_largest_available(size_remaining, max_order);
 372                 if (!page)
 373                         goto free_buffer;
 374
 375                 list_add_tail(&page->lru, &pages);
 376                 size_remaining -= page_size(page);
 377                 max_order = compound_order(page);
 378                 i++;
 379         }
 380
 381         table = &buffer->sg_table;
 382         if (sg_alloc_table(table, i, GFP_KERNEL))
 383                 goto free_buffer;
 384
 385         sg = table->sgl;
 386         list_for_each_entry_safe(page, tmp_page, &pages, lru) {
 387                 sg_set_page(sg, page, page_size(page), 0);
 388                 sg = sg_next(sg);
 389                 list_del(&page->lru);
 390         }
 391
 392         /* create the dmabuf */
 393         exp_info.exp_name = dma_heap_get_name(heap);
 394         exp_info.ops = &system_heap_buf_ops;
 395         exp_info.size = buffer->len;
 396         exp_info.flags = fd_flags;
 397         exp_info.priv = buffer;
 398         dmabuf = dma_buf_export(&exp_info);
 399         if (IS_ERR(dmabuf)) {
 400                 ret = PTR_ERR(dmabuf);
 401                 goto free_pages;
 402         }
 403         return dmabuf;
 404
 405 free_pages:
 406         for_each_sgtable_sg(table, sg, i) {
 407                 struct page *p = sg_page(sg);
 408
 409                 __free_pages(p, compound_order(p));
 410         }
 411         sg_free_table(table);
 412 free_buffer:
 413         list_for_each_entry_safe(page, tmp_page, &pages, lru)
 414                 __free_pages(page, compound_order(page));
 415         kfree(buffer);
 416
 417         return ERR_PTR(ret);
 418 }
 419
 420 static const struct dma_heap_ops system_heap_ops = {
 421         .allocate = system_heap_allocate,
 422 };
 423
 424 static int system_heap_create(void)
 425 {
 426         struct dma_heap_export_info exp_info;
 427
 428         exp_info.name = "system";
 429         exp_info.ops = &system_heap_ops;
 430         exp_info.priv = NULL;
 431
 432         sys_heap = dma_heap_add(&exp_info);
 433         if (IS_ERR(sys_heap))
 434                 return PTR_ERR(sys_heap);
 435
 436         return 0;
 437 }
 438 module_init(system_heap_create);
 439 MODULE_LICENSE("GPL v2");