kernel/dma/direct.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2018 Christoph Hellwig.
   4  *
   5  * DMA operations that map physical memory directly without using an IOMMU.
   6  */
   7 #include <linux/bootmem.h> /* for max_pfn */
   8 #include <linux/export.h>
   9 #include <linux/mm.h>
  10 #include <linux/dma-direct.h>
  11 #include <linux/scatterlist.h>
  12 #include <linux/dma-contiguous.h>
  13 #include <linux/dma-noncoherent.h>
  14 #include <linux/pfn.h>
  15 #include <linux/set_memory.h>
  16
  17 #define DIRECT_MAPPING_ERROR            0
  18
  19 /*
  20  * Most architectures use ZONE_DMA for the first 16 Megabytes, but
  21  * some use it for entirely different regions:
  22  */
  23 #ifndef ARCH_ZONE_DMA_BITS
  24 #define ARCH_ZONE_DMA_BITS 24
  25 #endif
  26
  27 /*
  28  * For AMD SEV all DMA must be to unencrypted addresses.
  29  */
  30 static inline bool force_dma_unencrypted(void)
  31 {
  32         return sev_active();
  33 }
  34
  35 static bool
  36 check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
  37                 const char *caller)
  38 {
  39         if (unlikely(dev && !dma_capable(dev, dma_addr, size))) {
  40                 if (!dev->dma_mask) {
  41                         dev_err(dev,
  42                                 "%s: call on device without dma_mask\n",
  43                                 caller);
  44                         return false;
  45                 }
  46
  47                 if (*dev->dma_mask >= DMA_BIT_MASK(32) || dev->bus_dma_mask) {
  48                         dev_err(dev,
  49                                 "%s: overflow %pad+%zu of device mask %llx bus mask %llx\n",
  50                                 caller, &dma_addr, size,
  51                                 *dev->dma_mask, dev->bus_dma_mask);
  52                 }
  53                 return false;
  54         }
  55         return true;
  56 }
  57
  58 static inline dma_addr_t phys_to_dma_direct(struct device *dev,
  59                 phys_addr_t phys)
  60 {
  61         if (force_dma_unencrypted())
  62                 return __phys_to_dma(dev, phys);
  63         return phys_to_dma(dev, phys);
  64 }
  65
  66 u64 dma_direct_get_required_mask(struct device *dev)
  67 {
  68         u64 max_dma = phys_to_dma_direct(dev, (max_pfn - 1) << PAGE_SHIFT);
  69
  70         if (dev->bus_dma_mask && dev->bus_dma_mask < max_dma)
  71                 max_dma = dev->bus_dma_mask;
  72
  73         return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
  74 }
  75
  76 static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
  77                 u64 *phys_mask)
  78 {
  79         if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)
  80                 dma_mask = dev->bus_dma_mask;
  81
  82         if (force_dma_unencrypted())
  83                 *phys_mask = __dma_to_phys(dev, dma_mask);
  84         else
  85                 *phys_mask = dma_to_phys(dev, dma_mask);
  86
  87         /*
  88          * Optimistically try the zone that the physical address mask falls
  89          * into first.  If that returns memory that isn't actually addressable
  90          * we will fallback to the next lower zone and try again.
  91          *
  92          * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding
  93          * zones.
  94          */
  95         if (*phys_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
  96                 return GFP_DMA;
  97         if (*phys_mask <= DMA_BIT_MASK(32))
  98                 return GFP_DMA32;
  99         return 0;
 100 }
 101
 102 static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
 103 {
 104         return phys_to_dma_direct(dev, phys) + size - 1 <=
 105                         min_not_zero(dev->coherent_dma_mask, dev->bus_dma_mask);
 106 }
 107
 108 void *dma_direct_alloc_pages(struct device *dev, size_t size,
 109                 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 110 {
 111         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 112         int page_order = get_order(size);
 113         struct page *page = NULL;
 114         u64 phys_mask;
 115         void *ret;
 116
 117         /* we always manually zero the memory once we are done: */
 118         gfp &= ~__GFP_ZERO;
 119         gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
 120                         &phys_mask);
 121 again:
 122         /* CMA can be used only in the context which permits sleeping */
 123         if (gfpflags_allow_blocking(gfp)) {
 124                 page = dma_alloc_from_contiguous(dev, count, page_order,
 125                                                  gfp & __GFP_NOWARN);
 126                 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
 127                         dma_release_from_contiguous(dev, page, count);
 128                         page = NULL;
 129                 }
 130         }
 131         if (!page)
 132                 page = alloc_pages_node(dev_to_node(dev), gfp, page_order);
 133
 134         if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
 135                 __free_pages(page, page_order);
 136                 page = NULL;
 137
 138                 if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
 139                     phys_mask < DMA_BIT_MASK(64) &&
 140                     !(gfp & (GFP_DMA32 | GFP_DMA))) {
 141                         gfp |= GFP_DMA32;
 142                         goto again;
 143                 }
 144
 145                 if (IS_ENABLED(CONFIG_ZONE_DMA) &&
 146                     phys_mask < DMA_BIT_MASK(32) && !(gfp & GFP_DMA)) {
 147                         gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
 148                         goto again;
 149                 }
 150         }
 151
 152         if (!page)
 153                 return NULL;
 154         ret = page_address(page);
 155         if (force_dma_unencrypted()) {
 156                 set_memory_decrypted((unsigned long)ret, 1 << page_order);
 157                 *dma_handle = __phys_to_dma(dev, page_to_phys(page));
 158         } else {
 159                 *dma_handle = phys_to_dma(dev, page_to_phys(page));
 160         }
 161         memset(ret, 0, size);
 162         return ret;
 163 }
 164
 165 /*
 166  * NOTE: this function must never look at the dma_addr argument, because we want
 167  * to be able to use it as a helper for iommu implementations as well.
 168  */
 169 void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
 170                 dma_addr_t dma_addr, unsigned long attrs)
 171 {
 172         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 173         unsigned int page_order = get_order(size);
 174
 175         if (force_dma_unencrypted())
 176                 set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
 177         if (!dma_release_from_contiguous(dev, virt_to_page(cpu_addr), count))
 178                 free_pages((unsigned long)cpu_addr, page_order);
 179 }
 180
 181 void *dma_direct_alloc(struct device *dev, size_t size,
 182                 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 183 {
 184         if (!dev_is_dma_coherent(dev))
 185                 return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
 186         return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
 187 }
 188
 189 void dma_direct_free(struct device *dev, size_t size,
 190                 void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
 191 {
 192         if (!dev_is_dma_coherent(dev))
 193                 arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
 194         else
 195                 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
 196 }
 197
 198 static void dma_direct_sync_single_for_device(struct device *dev,
 199                 dma_addr_t addr, size_t size, enum dma_data_direction dir)
 200 {
 201         if (dev_is_dma_coherent(dev))
 202                 return;
 203         arch_sync_dma_for_device(dev, dma_to_phys(dev, addr), size, dir);
 204 }
 205
 206 static void dma_direct_sync_sg_for_device(struct device *dev,
 207                 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
 208 {
 209         struct scatterlist *sg;
 210         int i;
 211
 212         if (dev_is_dma_coherent(dev))
 213                 return;
 214
 215         for_each_sg(sgl, sg, nents, i)
 216                 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir);
 217 }
 218
 219 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
 220     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
 221 static void dma_direct_sync_single_for_cpu(struct device *dev,
 222                 dma_addr_t addr, size_t size, enum dma_data_direction dir)
 223 {
 224         if (dev_is_dma_coherent(dev))
 225                 return;
 226         arch_sync_dma_for_cpu(dev, dma_to_phys(dev, addr), size, dir);
 227         arch_sync_dma_for_cpu_all(dev);
 228 }
 229
 230 static void dma_direct_sync_sg_for_cpu(struct device *dev,
 231                 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
 232 {
 233         struct scatterlist *sg;
 234         int i;
 235
 236         if (dev_is_dma_coherent(dev))
 237                 return;
 238
 239         for_each_sg(sgl, sg, nents, i)
 240                 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
 241         arch_sync_dma_for_cpu_all(dev);
 242 }
 243
 244 static void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
 245                 size_t size, enum dma_data_direction dir, unsigned long attrs)
 246 {
 247         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 248                 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
 249 }
 250
 251 static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
 252                 int nents, enum dma_data_direction dir, unsigned long attrs)
 253 {
 254         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 255                 dma_direct_sync_sg_for_cpu(dev, sgl, nents, dir);
 256 }
 257 #endif
 258
 259 dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
 260                 unsigned long offset, size_t size, enum dma_data_direction dir,
 261                 unsigned long attrs)
 262 {
 263         phys_addr_t phys = page_to_phys(page) + offset;
 264         dma_addr_t dma_addr = phys_to_dma(dev, phys);
 265
 266         if (!check_addr(dev, dma_addr, size, __func__))
 267                 return DIRECT_MAPPING_ERROR;
 268
 269         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 270                 dma_direct_sync_single_for_device(dev, dma_addr, size, dir);
 271         return dma_addr;
 272 }
 273
 274 int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
 275                 enum dma_data_direction dir, unsigned long attrs)
 276 {
 277         int i;
 278         struct scatterlist *sg;
 279
 280         for_each_sg(sgl, sg, nents, i) {
 281                 BUG_ON(!sg_page(sg));
 282
 283                 sg_dma_address(sg) = phys_to_dma(dev, sg_phys(sg));
 284                 if (!check_addr(dev, sg_dma_address(sg), sg->length, __func__))
 285                         return 0;
 286                 sg_dma_len(sg) = sg->length;
 287         }
 288
 289         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 290                 dma_direct_sync_sg_for_device(dev, sgl, nents, dir);
 291         return nents;
 292 }
 293
 294 /*
 295  * Because 32-bit DMA masks are so common we expect every architecture to be
 296  * able to satisfy them - either by not supporting more physical memory, or by
 297  * providing a ZONE_DMA32.  If neither is the case, the architecture needs to
 298  * use an IOMMU instead of the direct mapping.
 299  */
 300 int dma_direct_supported(struct device *dev, u64 mask)
 301 {
 302         u64 min_mask;
 303
 304         if (IS_ENABLED(CONFIG_ZONE_DMA))
 305                 min_mask = DMA_BIT_MASK(ARCH_ZONE_DMA_BITS);
 306         else
 307                 min_mask = DMA_BIT_MASK(32);
 308
 309         min_mask = min_t(u64, min_mask, (max_pfn - 1) << PAGE_SHIFT);
 310
 311         return mask >= phys_to_dma(dev, min_mask);
 312 }
 313
 314 int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr)
 315 {
 316         return dma_addr == DIRECT_MAPPING_ERROR;
 317 }
 318
 319 const struct dma_map_ops dma_direct_ops = {
 320         .alloc                  = dma_direct_alloc,
 321         .free                   = dma_direct_free,
 322         .map_page               = dma_direct_map_page,
 323         .map_sg                 = dma_direct_map_sg,
 324 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE)
 325         .sync_single_for_device = dma_direct_sync_single_for_device,
 326         .sync_sg_for_device     = dma_direct_sync_sg_for_device,
 327 #endif
 328 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
 329     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
 330         .sync_single_for_cpu    = dma_direct_sync_single_for_cpu,
 331         .sync_sg_for_cpu        = dma_direct_sync_sg_for_cpu,
 332         .unmap_page             = dma_direct_unmap_page,
 333         .unmap_sg               = dma_direct_unmap_sg,
 334 #endif
 335         .get_required_mask      = dma_direct_get_required_mask,
 336         .dma_supported          = dma_direct_supported,
 337         .mapping_error          = dma_direct_mapping_error,
 338         .cache_sync             = arch_dma_cache_sync,
 339 };
 340 EXPORT_SYMBOL(dma_direct_ops);