Merge tag '9p-for-5.13-rc1' of git://github.com/martinetd/linux

[linux-2.6-microblaze.git] / net / core / page_pool.c

diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index ad8b070..9ec1aa9 100644 (file)
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -180,40 +180,10 @@ static void page_pool_dma_sync_for_device(struct page_pool *pool,
                                         pool->p.dma_dir);
 }
 
-/* slow path */
-noinline
-static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
-                                                gfp_t _gfp)
+static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
 {
-       struct page *page;
-       gfp_t gfp = _gfp;
        dma_addr_t dma;
 
-       /* We could always set __GFP_COMP, and avoid this branch, as
-        * prep_new_page() can handle order-0 with __GFP_COMP.
-        */
-       if (pool->p.order)
-               gfp |= __GFP_COMP;
-
-       /* FUTURE development:
-        *
-        * Current slow-path essentially falls back to single page
-        * allocations, which doesn't improve performance.  This code
-        * need bulk allocation support from the page allocator code.
-        */
-
-       /* Cache was empty, do real allocation */
-#ifdef CONFIG_NUMA
-       page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
-#else
-       page = alloc_pages(gfp, pool->p.order);
-#endif
-       if (!page)
-               return NULL;
-
-       if (!(pool->p.flags & PP_FLAG_DMA_MAP))
-               goto skip_dma_map;
-
        /* Setup DMA mapping: use 'struct page' area for storing DMA-addr
         * since dma_addr_t can be either 32 or 64 bits and does not always fit
         * into page private data (i.e 32bit cpu with 64bit DMA caps)
@@ -222,20 +192,87 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
        dma = dma_map_page_attrs(pool->p.dev, page, 0,
                                 (PAGE_SIZE << pool->p.order),
                                 pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
-       if (dma_mapping_error(pool->p.dev, dma)) {
-               put_page(page);
-               return NULL;
-       }
+       if (dma_mapping_error(pool->p.dev, dma))
+               return false;
+
        page->dma_addr = dma;
 
        if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
                page_pool_dma_sync_for_device(pool, page, pool->p.max_len);
 
-skip_dma_map:
+       return true;
+}
+
+static struct page *__page_pool_alloc_page_order(struct page_pool *pool,
+                                                gfp_t gfp)
+{
+       struct page *page;
+
+       gfp |= __GFP_COMP;
+       page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
+       if (unlikely(!page))
+               return NULL;
+
+       if ((pool->p.flags & PP_FLAG_DMA_MAP) &&
+           unlikely(!page_pool_dma_map(pool, page))) {
+               put_page(page);
+               return NULL;
+       }
+
        /* Track how many pages are held 'in-flight' */
        pool->pages_state_hold_cnt++;
-
        trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt);
+       return page;
+}
+
+/* slow path */
+noinline
+static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
+                                                gfp_t gfp)
+{
+       const int bulk = PP_ALLOC_CACHE_REFILL;
+       unsigned int pp_flags = pool->p.flags;
+       unsigned int pp_order = pool->p.order;
+       struct page *page;
+       int i, nr_pages;
+
+       /* Don't support bulk alloc for high-order pages */
+       if (unlikely(pp_order))
+               return __page_pool_alloc_page_order(pool, gfp);
+
+       /* Unnecessary as alloc cache is empty, but guarantees zero count */
+       if (unlikely(pool->alloc.count > 0))
+               return pool->alloc.cache[--pool->alloc.count];
+
+       /* Mark empty alloc.cache slots "empty" for alloc_pages_bulk_array */
+       memset(&pool->alloc.cache, 0, sizeof(void *) * bulk);
+
+       nr_pages = alloc_pages_bulk_array(gfp, bulk, pool->alloc.cache);
+       if (unlikely(!nr_pages))
+               return NULL;
+
+       /* Pages have been filled into alloc.cache array, but count is zero and
+        * page element have not been (possibly) DMA mapped.
+        */
+       for (i = 0; i < nr_pages; i++) {
+               page = pool->alloc.cache[i];
+               if ((pp_flags & PP_FLAG_DMA_MAP) &&
+                   unlikely(!page_pool_dma_map(pool, page))) {
+                       put_page(page);
+                       continue;
+               }
+               pool->alloc.cache[pool->alloc.count++] = page;
+               /* Track how many pages are held 'in-flight' */
+               pool->pages_state_hold_cnt++;
+               trace_page_pool_state_hold(pool, page,
+                                          pool->pages_state_hold_cnt);
+       }
+
+       /* Return last page */
+       if (likely(pool->alloc.count > 0))
+               page = pool->alloc.cache[--pool->alloc.count];
+       else
+               page = NULL;
 
        /* When page just alloc'ed is should/must have refcnt 1. */
        return page;