Merge tag 'dma-mapping-5.8' of git://git.infradead.org/users/hch/dma-mapping
authorLinus Torvalds <torvalds@linux-foundation.org>
Sat, 6 Jun 2020 18:43:23 +0000 (11:43 -0700)
committerLinus Torvalds <torvalds@linux-foundation.org>
Sat, 6 Jun 2020 18:43:23 +0000 (11:43 -0700)
Pull dma-mapping updates from Christoph Hellwig:

 - enhance the dma pool to allow atomic allocation on x86 with AMD SEV
   (David Rientjes)

 - two small cleanups (Jason Yan and Peter Collingbourne)

* tag 'dma-mapping-5.8' of git://git.infradead.org/users/hch/dma-mapping:
  dma-contiguous: fix comment for dma_release_from_contiguous
  dma-pool: scale the default DMA coherent pool size with memory capacity
  x86/mm: unencrypted non-blocking DMA allocations use coherent pools
  dma-pool: add pool sizes to debugfs
  dma-direct: atomic allocations must come from atomic coherent pools
  dma-pool: dynamically expanding atomic pools
  dma-pool: add additional coherent pools to map to gfp mask
  dma-remap: separate DMA atomic pools from direct remap code
  dma-debug: make __dma_entry_alloc_check_leak() static

1  2 
arch/x86/Kconfig
kernel/dma/remap.c

diff --combined arch/x86/Kconfig
@@@ -59,9 -59,7 +59,9 @@@ config X8
        select ARCH_CLOCKSOURCE_INIT
        select ARCH_HAS_ACPI_TABLE_UPGRADE      if ACPI
        select ARCH_HAS_DEBUG_VIRTUAL
 +      select ARCH_HAS_DEBUG_VM_PGTABLE        if !X86_PAE
        select ARCH_HAS_DEVMEM_IS_ALLOWED
 +      select ARCH_HAS_EARLY_DEBUG             if KGDB
        select ARCH_HAS_ELF_RANDOMIZE
        select ARCH_HAS_FAST_MULTIPLIER
        select ARCH_HAS_FILTER_PGPROT
@@@ -70,7 -68,6 +70,7 @@@
        select ARCH_HAS_KCOV                    if X86_64
        select ARCH_HAS_MEM_ENCRYPT
        select ARCH_HAS_MEMBARRIER_SYNC_CORE
 +      select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
        select ARCH_HAS_PMEM_API                if X86_64
        select ARCH_HAS_PTE_DEVMAP              if X86_64
        select ARCH_HAS_PTE_SPECIAL
@@@ -83,7 -80,6 +83,7 @@@
        select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE
        select ARCH_HAS_SYSCALL_WRAPPER
        select ARCH_HAS_UBSAN_SANITIZE_ALL
 +      select ARCH_HAS_DEBUG_WX
        select ARCH_HAVE_NMI_SAFE_CMPXCHG
        select ARCH_MIGHT_HAVE_ACPI_PDC         if ACPI
        select ARCH_MIGHT_HAVE_PC_PARPORT
@@@ -95,7 -91,6 +95,7 @@@
        select ARCH_USE_BUILTIN_BSWAP
        select ARCH_USE_QUEUED_RWLOCKS
        select ARCH_USE_QUEUED_SPINLOCKS
 +      select ARCH_USE_SYM_ANNOTATIONS
        select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
        select ARCH_WANT_DEFAULT_BPF_JIT        if X86_64
        select ARCH_WANTS_DYNAMIC_TASK_STRUCT
        select HAVE_ARCH_TRACEHOOK
        select HAVE_ARCH_TRANSPARENT_HUGEPAGE
        select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD if X86_64
 -      select HAVE_ARCH_USERFAULTFD_WP         if USERFAULTFD
 +      select HAVE_ARCH_USERFAULTFD_WP         if X86_64 && USERFAULTFD
        select HAVE_ARCH_VMAP_STACK             if X86_64
        select HAVE_ARCH_WITHIN_STACK_FRAMES
        select HAVE_ASM_MODVERSIONS
        select HAVE_KRETPROBES
        select HAVE_KVM
        select HAVE_LIVEPATCH                   if X86_64
 -      select HAVE_MEMBLOCK_NODE_MAP
        select HAVE_MIXED_BREAKPOINTS_REGS
        select HAVE_MOD_ARCH_SPECIFIC
        select HAVE_MOVE_PMD
@@@ -599,7 -595,7 +599,7 @@@ config X86_INTEL_MI
        select I2C
        select DW_APB_TIMER
        select APB_TIMER
 -      select INTEL_SCU_IPC
 +      select INTEL_SCU_PCI
        select MFD_INTEL_MSIC
        ---help---
          Select to build a kernel capable of supporting Intel MID (Mobile
@@@ -1524,6 -1520,7 +1524,7 @@@ config X86_CPA_STATISTIC
  config AMD_MEM_ENCRYPT
        bool "AMD Secure Memory Encryption (SME) support"
        depends on X86_64 && CPU_SUP_AMD
+       select DMA_COHERENT_POOL
        select DYNAMIC_PHYSICAL_MASK
        select ARCH_USE_MEMREMAP_PROT
        select ARCH_HAS_FORCE_DMA_UNENCRYPTED
@@@ -1586,6 -1583,15 +1587,6 @@@ config X86_64_ACPI_NUM
        ---help---
          Enable ACPI SRAT based node topology detection.
  
 -# Some NUMA nodes have memory ranges that span
 -# other nodes.  Even though a pfn is valid and
 -# between a node's start and end pfns, it may not
 -# reside on that node.  See memmap_init_zone()
 -# for details.
 -config NODES_SPAN_OTHER_NODES
 -      def_bool y
 -      depends on X86_64_ACPI_NUMA
 -
  config NUMA_EMU
        bool "NUMA emulation"
        depends on NUMA
@@@ -1605,10 -1611,19 +1606,10 @@@ config NODES_SHIF
          Specify the maximum number of NUMA Nodes available on the target
          system.  Increases memory reserved to accommodate various tables.
  
 -config ARCH_HAVE_MEMORY_PRESENT
 -      def_bool y
 -      depends on X86_32 && DISCONTIGMEM
 -
  config ARCH_FLATMEM_ENABLE
        def_bool y
        depends on X86_32 && !NUMA
  
 -config ARCH_DISCONTIGMEM_ENABLE
 -      def_bool n
 -      depends on NUMA && X86_32
 -      depends on BROKEN
 -
  config ARCH_SPARSEMEM_ENABLE
        def_bool y
        depends on X86_64 || NUMA || X86_32 || X86_32_NON_STANDARD
@@@ -1873,10 -1888,10 +1874,10 @@@ config X86_UMI
          results are dummy.
  
  config X86_INTEL_MEMORY_PROTECTION_KEYS
 -      prompt "Intel Memory Protection Keys"
 +      prompt "Memory Protection Keys"
        def_bool y
        # Note: only available in 64-bit mode
 -      depends on CPU_SUP_INTEL && X86_64
 +      depends on X86_64 && (CPU_SUP_INTEL || CPU_SUP_AMD)
        select ARCH_USES_HIGH_VMA_FLAGS
        select ARCH_HAS_PKEYS
        ---help---
diff --combined kernel/dma/remap.c
@@@ -1,13 -1,8 +1,8 @@@
  // SPDX-License-Identifier: GPL-2.0
  /*
-  * Copyright (C) 2012 ARM Ltd.
   * Copyright (c) 2014 The Linux Foundation
   */
- #include <linux/dma-direct.h>
- #include <linux/dma-noncoherent.h>
- #include <linux/dma-contiguous.h>
- #include <linux/init.h>
- #include <linux/genalloc.h>
+ #include <linux/dma-mapping.h>
  #include <linux/slab.h>
  #include <linux/vmalloc.h>
  
@@@ -20,6 -15,23 +15,6 @@@ struct page **dma_common_find_pages(voi
        return area->pages;
  }
  
 -static struct vm_struct *__dma_common_pages_remap(struct page **pages,
 -                      size_t size, pgprot_t prot, const void *caller)
 -{
 -      struct vm_struct *area;
 -
 -      area = get_vm_area_caller(size, VM_DMA_COHERENT, caller);
 -      if (!area)
 -              return NULL;
 -
 -      if (map_vm_area(area, prot, pages)) {
 -              vunmap(area->addr);
 -              return NULL;
 -      }
 -
 -      return area;
 -}
 -
  /*
   * Remaps an array of PAGE_SIZE pages into another vm_area.
   * Cannot be used in non-sleeping contexts
  void *dma_common_pages_remap(struct page **pages, size_t size,
                         pgprot_t prot, const void *caller)
  {
 -      struct vm_struct *area;
 +      void *vaddr;
  
 -      area = __dma_common_pages_remap(pages, size, prot, caller);
 -      if (!area)
 -              return NULL;
 -
 -      area->pages = pages;
 -
 -      return area->addr;
 +      vaddr = vmap(pages, size >> PAGE_SHIFT, VM_DMA_COHERENT, prot);
 +      if (vaddr)
 +              find_vm_area(vaddr)->pages = pages;
 +      return vaddr;
  }
  
  /*
  void *dma_common_contiguous_remap(struct page *page, size_t size,
                        pgprot_t prot, const void *caller)
  {
 -      int i;
 +      int count = size >> PAGE_SHIFT;
        struct page **pages;
 -      struct vm_struct *area;
 +      void *vaddr;
 +      int i;
  
 -      pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL);
 +      pages = kmalloc_array(count, sizeof(struct page *), GFP_KERNEL);
        if (!pages)
                return NULL;
 -
 -      for (i = 0; i < (size >> PAGE_SHIFT); i++)
 +      for (i = 0; i < count; i++)
                pages[i] = nth_page(page, i);
 -
 -      area = __dma_common_pages_remap(pages, size, prot, caller);
 -
 +      vaddr = vmap(pages, count, VM_DMA_COHERENT, prot);
        kfree(pages);
  
 -      if (!area)
 -              return NULL;
 -      return area->addr;
 +      return vaddr;
  }
  
  /*
@@@ -73,117 -92,3 +68,3 @@@ void dma_common_free_remap(void *cpu_ad
        unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
        vunmap(cpu_addr);
  }
- #ifdef CONFIG_DMA_DIRECT_REMAP
- static struct gen_pool *atomic_pool __ro_after_init;
- #define DEFAULT_DMA_COHERENT_POOL_SIZE  SZ_256K
- static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
- static int __init early_coherent_pool(char *p)
- {
-       atomic_pool_size = memparse(p, &p);
-       return 0;
- }
- early_param("coherent_pool", early_coherent_pool);
- static gfp_t dma_atomic_pool_gfp(void)
- {
-       if (IS_ENABLED(CONFIG_ZONE_DMA))
-               return GFP_DMA;
-       if (IS_ENABLED(CONFIG_ZONE_DMA32))
-               return GFP_DMA32;
-       return GFP_KERNEL;
- }
- static int __init dma_atomic_pool_init(void)
- {
-       unsigned int pool_size_order = get_order(atomic_pool_size);
-       unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
-       struct page *page;
-       void *addr;
-       int ret;
-       if (dev_get_cma_area(NULL))
-               page = dma_alloc_from_contiguous(NULL, nr_pages,
-                                                pool_size_order, false);
-       else
-               page = alloc_pages(dma_atomic_pool_gfp(), pool_size_order);
-       if (!page)
-               goto out;
-       arch_dma_prep_coherent(page, atomic_pool_size);
-       atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
-       if (!atomic_pool)
-               goto free_page;
-       addr = dma_common_contiguous_remap(page, atomic_pool_size,
-                                          pgprot_dmacoherent(PAGE_KERNEL),
-                                          __builtin_return_address(0));
-       if (!addr)
-               goto destroy_genpool;
-       ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
-                               page_to_phys(page), atomic_pool_size, -1);
-       if (ret)
-               goto remove_mapping;
-       gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);
-       pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
-               atomic_pool_size / 1024);
-       return 0;
- remove_mapping:
-       dma_common_free_remap(addr, atomic_pool_size);
- destroy_genpool:
-       gen_pool_destroy(atomic_pool);
-       atomic_pool = NULL;
- free_page:
-       if (!dma_release_from_contiguous(NULL, page, nr_pages))
-               __free_pages(page, pool_size_order);
- out:
-       pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
-               atomic_pool_size / 1024);
-       return -ENOMEM;
- }
- postcore_initcall(dma_atomic_pool_init);
- bool dma_in_atomic_pool(void *start, size_t size)
- {
-       if (unlikely(!atomic_pool))
-               return false;
-       return gen_pool_has_addr(atomic_pool, (unsigned long)start, size);
- }
- void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)
- {
-       unsigned long val;
-       void *ptr = NULL;
-       if (!atomic_pool) {
-               WARN(1, "coherent pool not initialised!\n");
-               return NULL;
-       }
-       val = gen_pool_alloc(atomic_pool, size);
-       if (val) {
-               phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
-               *ret_page = pfn_to_page(__phys_to_pfn(phys));
-               ptr = (void *)val;
-               memset(ptr, 0, size);
-       }
-       return ptr;
- }
- bool dma_free_from_pool(void *start, size_t size)
- {
-       if (!dma_in_atomic_pool(start, size))
-               return false;
-       gen_pool_free(atomic_pool, (unsigned long)start, size);
-       return true;
- }
- #endif /* CONFIG_DMA_DIRECT_REMAP */