Merge branch 'for-next/iommu/iova' into for-next/iommu/core

IOVA allocator updates for 5.11, including removal of unused symbols and
functions as well as some optimisations to improve allocation behaviour
in the face of fragmentation.

* for-next/iommu/iova:
  iommu: Stop exporting free_iova_mem()
  iommu: Stop exporting alloc_iova_mem()
  iommu: Delete split_and_remove_iova()
  iommu: avoid taking iova_rbtree_lock twice
  iommu/iova: Free global iova rcache on iova alloc failure
  iommu/iova: Retry from last rb tree node if iova search fails

diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 30d969a..4bb3293 100644 (file)
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -25,6 +25,7 @@ static void init_iova_rcaches(struct iova_domain *iovad);
 static void free_iova_rcaches(struct iova_domain *iovad);
 static void fq_destroy_all_entries(struct iova_domain *iovad);
 static void fq_flush_timeout(struct timer_list *t);
+static void free_global_cached_iovas(struct iova_domain *iovad);
 
 void
 init_iova_domain(struct iova_domain *iovad, unsigned long granule,
@@ -184,8 +185,9 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
        struct rb_node *curr, *prev;
        struct iova *curr_iova;
        unsigned long flags;
-       unsigned long new_pfn;
+       unsigned long new_pfn, retry_pfn;
        unsigned long align_mask = ~0UL;
+       unsigned long high_pfn = limit_pfn, low_pfn = iovad->start_pfn;
 
        if (size_aligned)
                align_mask <<= fls_long(size - 1);
@@ -198,15 +200,25 @@ static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
 
        curr = __get_cached_rbnode(iovad, limit_pfn);
        curr_iova = rb_entry(curr, struct iova, node);
+       retry_pfn = curr_iova->pfn_hi + 1;
+
+retry:
        do {
-               limit_pfn = min(limit_pfn, curr_iova->pfn_lo);
-               new_pfn = (limit_pfn - size) & align_mask;
+               high_pfn = min(high_pfn, curr_iova->pfn_lo);
+               new_pfn = (high_pfn - size) & align_mask;
                prev = curr;
                curr = rb_prev(curr);
                curr_iova = rb_entry(curr, struct iova, node);
-       } while (curr && new_pfn <= curr_iova->pfn_hi);
-
-       if (limit_pfn < size || new_pfn < iovad->start_pfn) {
+       } while (curr && new_pfn <= curr_iova->pfn_hi && new_pfn >= low_pfn);
+
+       if (high_pfn < size || new_pfn < low_pfn) {
+               if (low_pfn == iovad->start_pfn && retry_pfn < limit_pfn) {
+                       high_pfn = limit_pfn;
+                       low_pfn = retry_pfn;
+                       curr = &iovad->anchor.node;
+                       curr_iova = rb_entry(curr, struct iova, node);
+                       goto retry;
+               }
                iovad->max32_alloc_size = size;
                goto iova32_full;
        }
@@ -231,18 +243,16 @@ static struct kmem_cache *iova_cache;
 static unsigned int iova_cache_users;
 static DEFINE_MUTEX(iova_cache_mutex);
 
-struct iova *alloc_iova_mem(void)
+static struct iova *alloc_iova_mem(void)
 {
        return kmem_cache_zalloc(iova_cache, GFP_ATOMIC | __GFP_NOWARN);
 }
-EXPORT_SYMBOL(alloc_iova_mem);
 
-void free_iova_mem(struct iova *iova)
+static void free_iova_mem(struct iova *iova)
 {
        if (iova->pfn_lo != IOVA_ANCHOR)
                kmem_cache_free(iova_cache, iova);
 }
-EXPORT_SYMBOL(free_iova_mem);
 
 int iova_cache_get(void)
 {
@@ -390,10 +400,14 @@ EXPORT_SYMBOL_GPL(__free_iova);
 void
 free_iova(struct iova_domain *iovad, unsigned long pfn)
 {
-       struct iova *iova = find_iova(iovad, pfn);
+       unsigned long flags;
+       struct iova *iova;
 
+       spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
+       iova = private_find_iova(iovad, pfn);
        if (iova)
-               __free_iova(iovad, iova);
+               private_free_iova(iovad, iova);
+       spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
 
 }
 EXPORT_SYMBOL_GPL(free_iova);
@@ -431,6 +445,7 @@ retry:
                flush_rcache = false;
                for_each_online_cpu(cpu)
                        free_cpu_cached_iovas(cpu, iovad);
+               free_global_cached_iovas(iovad);
                goto retry;
        }
 
@@ -725,47 +740,6 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to)
 }
 EXPORT_SYMBOL_GPL(copy_reserved_iova);
 
-struct iova *
-split_and_remove_iova(struct iova_domain *iovad, struct iova *iova,
-                     unsigned long pfn_lo, unsigned long pfn_hi)
-{
-       unsigned long flags;
-       struct iova *prev = NULL, *next = NULL;
-
-       spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
-       if (iova->pfn_lo < pfn_lo) {
-               prev = alloc_and_init_iova(iova->pfn_lo, pfn_lo - 1);
-               if (prev == NULL)
-                       goto error;
-       }
-       if (iova->pfn_hi > pfn_hi) {
-               next = alloc_and_init_iova(pfn_hi + 1, iova->pfn_hi);
-               if (next == NULL)
-                       goto error;
-       }
-
-       __cached_rbnode_delete_update(iovad, iova);
-       rb_erase(&iova->node, &iovad->rbroot);
-
-       if (prev) {
-               iova_insert_rbtree(&iovad->rbroot, prev, NULL);
-               iova->pfn_lo = pfn_lo;
-       }
-       if (next) {
-               iova_insert_rbtree(&iovad->rbroot, next, NULL);
-               iova->pfn_hi = pfn_hi;
-       }
-       spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
-
-       return iova;
-
-error:
-       spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
-       if (prev)
-               free_iova_mem(prev);
-       return NULL;
-}
-
 /*
  * Magazine caches for IOVA ranges.  For an introduction to magazines,
  * see the USENIX 2001 paper "Magazines and Vmem: Extending the Slab
@@ -1046,5 +1020,25 @@ void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad)
        }
 }
 
+/*
+ * free all the IOVA ranges of global cache
+ */
+static void free_global_cached_iovas(struct iova_domain *iovad)
+{
+       struct iova_rcache *rcache;
+       unsigned long flags;
+       int i, j;
+
+       for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
+               rcache = &iovad->rcaches[i];
+               spin_lock_irqsave(&rcache->lock, flags);
+               for (j = 0; j < rcache->depot_size; ++j) {
+                       iova_magazine_free_pfns(rcache->depot[j], iovad);
+                       iova_magazine_free(rcache->depot[j]);
+               }
+               rcache->depot_size = 0;
+               spin_unlock_irqrestore(&rcache->lock, flags);
+       }
+}
 MODULE_AUTHOR("Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>");
 MODULE_LICENSE("GPL");

diff --git a/include/linux/iova.h b/include/linux/iova.h
index a0637ab..76e16ae 100644 (file)
--- a/include/linux/iova.h
+++ b/include/linux/iova.h
@@ -136,8 +136,6 @@ static inline unsigned long iova_pfn(struct iova_domain *iovad, dma_addr_t iova)
 int iova_cache_get(void);
 void iova_cache_put(void);
 
-struct iova *alloc_iova_mem(void);
-void free_iova_mem(struct iova *iova);
 void free_iova(struct iova_domain *iovad, unsigned long pfn);
 void __free_iova(struct iova_domain *iovad, struct iova *iova);
 struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
@@ -160,8 +158,6 @@ int init_iova_flush_queue(struct iova_domain *iovad,
                          iova_flush_cb flush_cb, iova_entry_dtor entry_dtor);
 struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
 void put_iova_domain(struct iova_domain *iovad);
-struct iova *split_and_remove_iova(struct iova_domain *iovad,
-       struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi);
 void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
 #else
 static inline int iova_cache_get(void)
@@ -173,15 +169,6 @@ static inline void iova_cache_put(void)
 {
 }
 
-static inline struct iova *alloc_iova_mem(void)
-{
-       return NULL;
-}
-
-static inline void free_iova_mem(struct iova *iova)
-{
-}
-
 static inline void free_iova(struct iova_domain *iovad, unsigned long pfn)
 {
 }
@@ -258,14 +245,6 @@ static inline void put_iova_domain(struct iova_domain *iovad)
 {
 }
 
-static inline struct iova *split_and_remove_iova(struct iova_domain *iovad,
-                                                struct iova *iova,
-                                                unsigned long pfn_lo,
-                                                unsigned long pfn_hi)
-{
-       return NULL;
-}
-
 static inline void free_cpu_cached_iovas(unsigned int cpu,
                                         struct iova_domain *iovad)
 {