kernel/dma/remap.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2012 ARM Ltd.
   4  * Copyright (c) 2014 The Linux Foundation
   5  */
   6 #include <linux/dma-direct.h>
   7 #include <linux/dma-noncoherent.h>
   8 #include <linux/dma-contiguous.h>
   9 #include <linux/init.h>
  10 #include <linux/genalloc.h>
  11 #include <linux/slab.h>
  12 #include <linux/vmalloc.h>
  13
  14 struct page **dma_common_find_pages(void *cpu_addr)
  15 {
  16         struct vm_struct *area = find_vm_area(cpu_addr);
  17
  18         if (!area || area->flags != VM_DMA_COHERENT)
  19                 return NULL;
  20         return area->pages;
  21 }
  22
  23 /*
  24  * Remaps an array of PAGE_SIZE pages into another vm_area.
  25  * Cannot be used in non-sleeping contexts
  26  */
  27 void *dma_common_pages_remap(struct page **pages, size_t size,
  28                          pgprot_t prot, const void *caller)
  29 {
  30         void *vaddr;
  31
  32         vaddr = vmap(pages, size >> PAGE_SHIFT, VM_DMA_COHERENT, prot);
  33         if (vaddr)
  34                 find_vm_area(vaddr)->pages = pages;
  35         return vaddr;
  36 }
  37
  38 /*
  39  * Remaps an allocated contiguous region into another vm_area.
  40  * Cannot be used in non-sleeping contexts
  41  */
  42 void *dma_common_contiguous_remap(struct page *page, size_t size,
  43                         pgprot_t prot, const void *caller)
  44 {
  45         int count = size >> PAGE_SHIFT;
  46         struct page **pages;
  47         void *vaddr;
  48         int i;
  49
  50         pages = kmalloc_array(count, sizeof(struct page *), GFP_KERNEL);
  51         if (!pages)
  52                 return NULL;
  53         for (i = 0; i < count; i++)
  54                 pages[i] = nth_page(page, i);
  55         vaddr = vmap(pages, count, VM_DMA_COHERENT, prot);
  56         kfree(pages);
  57
  58         return vaddr;
  59 }
  60
  61 /*
  62  * Unmaps a range previously mapped by dma_common_*_remap
  63  */
  64 void dma_common_free_remap(void *cpu_addr, size_t size)
  65 {
  66         struct vm_struct *area = find_vm_area(cpu_addr);
  67
  68         if (!area || area->flags != VM_DMA_COHERENT) {
  69                 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
  70                 return;
  71         }
  72
  73         unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
  74         vunmap(cpu_addr);
  75 }
  76
  77 #ifdef CONFIG_DMA_DIRECT_REMAP
  78 static struct gen_pool *atomic_pool __ro_after_init;
  79
  80 #define DEFAULT_DMA_COHERENT_POOL_SIZE  SZ_256K
  81 static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
  82
  83 static int __init early_coherent_pool(char *p)
  84 {
  85         atomic_pool_size = memparse(p, &p);
  86         return 0;
  87 }
  88 early_param("coherent_pool", early_coherent_pool);
  89
  90 static gfp_t dma_atomic_pool_gfp(void)
  91 {
  92         if (IS_ENABLED(CONFIG_ZONE_DMA))
  93                 return GFP_DMA;
  94         if (IS_ENABLED(CONFIG_ZONE_DMA32))
  95                 return GFP_DMA32;
  96         return GFP_KERNEL;
  97 }
  98
  99 static int __init dma_atomic_pool_init(void)
 100 {
 101         unsigned int pool_size_order = get_order(atomic_pool_size);
 102         unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
 103         struct page *page;
 104         void *addr;
 105         int ret;
 106
 107         if (dev_get_cma_area(NULL))
 108                 page = dma_alloc_from_contiguous(NULL, nr_pages,
 109                                                  pool_size_order, false);
 110         else
 111                 page = alloc_pages(dma_atomic_pool_gfp(), pool_size_order);
 112         if (!page)
 113                 goto out;
 114
 115         arch_dma_prep_coherent(page, atomic_pool_size);
 116
 117         atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
 118         if (!atomic_pool)
 119                 goto free_page;
 120
 121         addr = dma_common_contiguous_remap(page, atomic_pool_size,
 122                                            pgprot_dmacoherent(PAGE_KERNEL),
 123                                            __builtin_return_address(0));
 124         if (!addr)
 125                 goto destroy_genpool;
 126
 127         ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
 128                                 page_to_phys(page), atomic_pool_size, -1);
 129         if (ret)
 130                 goto remove_mapping;
 131         gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);
 132
 133         pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
 134                 atomic_pool_size / 1024);
 135         return 0;
 136
 137 remove_mapping:
 138         dma_common_free_remap(addr, atomic_pool_size);
 139 destroy_genpool:
 140         gen_pool_destroy(atomic_pool);
 141         atomic_pool = NULL;
 142 free_page:
 143         if (!dma_release_from_contiguous(NULL, page, nr_pages))
 144                 __free_pages(page, pool_size_order);
 145 out:
 146         pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
 147                 atomic_pool_size / 1024);
 148         return -ENOMEM;
 149 }
 150 postcore_initcall(dma_atomic_pool_init);
 151
 152 bool dma_in_atomic_pool(void *start, size_t size)
 153 {
 154         if (unlikely(!atomic_pool))
 155                 return false;
 156
 157         return gen_pool_has_addr(atomic_pool, (unsigned long)start, size);
 158 }
 159
 160 void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)
 161 {
 162         unsigned long val;
 163         void *ptr = NULL;
 164
 165         if (!atomic_pool) {
 166                 WARN(1, "coherent pool not initialised!\n");
 167                 return NULL;
 168         }
 169
 170         val = gen_pool_alloc(atomic_pool, size);
 171         if (val) {
 172                 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
 173
 174                 *ret_page = pfn_to_page(__phys_to_pfn(phys));
 175                 ptr = (void *)val;
 176                 memset(ptr, 0, size);
 177         }
 178
 179         return ptr;
 180 }
 181
 182 bool dma_free_from_pool(void *start, size_t size)
 183 {
 184         if (!dma_in_atomic_pool(start, size))
 185                 return false;
 186         gen_pool_free(atomic_pool, (unsigned long)start, size);
 187         return true;
 188 }
 189 #endif /* CONFIG_DMA_DIRECT_REMAP */