Merge tag 'ovl-update-5.15' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/vfs

[linux-2.6-microblaze.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 3b97e17..eeb3a9c 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -840,21 +840,24 @@ void init_mem_debugging_and_hardening(void)
        }
 #endif
 
-       if (_init_on_alloc_enabled_early) {
-               if (page_poisoning_requested)
-                       pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
-                               "will take precedence over init_on_alloc\n");
-               else
-                       static_branch_enable(&init_on_alloc);
-       }
-       if (_init_on_free_enabled_early) {
-               if (page_poisoning_requested)
-                       pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
-                               "will take precedence over init_on_free\n");
-               else
-                       static_branch_enable(&init_on_free);
+       if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&
+           page_poisoning_requested) {
+               pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
+                       "will take precedence over init_on_alloc and init_on_free\n");
+               _init_on_alloc_enabled_early = false;
+               _init_on_free_enabled_early = false;
        }
 
+       if (_init_on_alloc_enabled_early)
+               static_branch_enable(&init_on_alloc);
+       else
+               static_branch_disable(&init_on_alloc);
+
+       if (_init_on_free_enabled_early)
+               static_branch_enable(&init_on_free);
+       else
+               static_branch_disable(&init_on_free);
+
 #ifdef CONFIG_DEBUG_PAGEALLOC
        if (!debug_pagealloc_enabled())
                return;
@@ -3450,19 +3453,10 @@ void free_unref_page_list(struct list_head *list)
                 * comment in free_unref_page.
                 */
                migratetype = get_pcppage_migratetype(page);
-               if (unlikely(migratetype >= MIGRATE_PCPTYPES)) {
-                       if (unlikely(is_migrate_isolate(migratetype))) {
-                               list_del(&page->lru);
-                               free_one_page(page_zone(page), page, pfn, 0,
-                                                       migratetype, FPI_NONE);
-                               continue;
-                       }
-
-                       /*
-                        * Non-isolated types over MIGRATE_PCPTYPES get added
-                        * to the MIGRATE_MOVABLE pcp list.
-                        */
-                       set_pcppage_migratetype(page, MIGRATE_MOVABLE);
+               if (unlikely(is_migrate_isolate(migratetype))) {
+                       list_del(&page->lru);
+                       free_one_page(page_zone(page), page, pfn, 0, migratetype, FPI_NONE);
+                       continue;
                }
 
                set_page_private(page, pfn);
@@ -3472,7 +3466,15 @@ void free_unref_page_list(struct list_head *list)
        list_for_each_entry_safe(page, next, list, lru) {
                pfn = page_private(page);
                set_page_private(page, 0);
+
+               /*
+                * Non-isolated types over MIGRATE_PCPTYPES get added
+                * to the MIGRATE_MOVABLE pcp list.
+                */
                migratetype = get_pcppage_migratetype(page);
+               if (unlikely(migratetype >= MIGRATE_PCPTYPES))
+                       migratetype = MIGRATE_MOVABLE;
+
                trace_mm_page_free_batched(page);
                free_unref_page_commit(page, pfn, migratetype, 0);
 
@@ -3820,7 +3822,7 @@ static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
 
 #endif /* CONFIG_FAIL_PAGE_ALLOC */
 
-static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
+noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
 {
        return __should_fail_alloc_page(gfp_mask, order);
 }
@@ -5221,9 +5223,6 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
        unsigned int alloc_flags = ALLOC_WMARK_LOW;
        int nr_populated = 0, nr_account = 0;
 
-       if (unlikely(nr_pages <= 0))
-               return 0;
-
        /*
         * Skip populated array elements to determine if any pages need
         * to be allocated before disabling IRQs.
@@ -5231,19 +5230,35 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
        while (page_array && nr_populated < nr_pages && page_array[nr_populated])
                nr_populated++;
 
+       /* No pages requested? */
+       if (unlikely(nr_pages <= 0))
+               goto out;
+
        /* Already populated array? */
        if (unlikely(page_array && nr_pages - nr_populated == 0))
-               return nr_populated;
+               goto out;
 
        /* Use the single page allocator for one page. */
        if (nr_pages - nr_populated == 1)
                goto failed;
 
+#ifdef CONFIG_PAGE_OWNER
+       /*
+        * PAGE_OWNER may recurse into the allocator to allocate space to
+        * save the stack with pagesets.lock held. Releasing/reacquiring
+        * removes much of the performance benefit of bulk allocation so
+        * force the caller to allocate one page at a time as it'll have
+        * similar performance to added complexity to the bulk allocator.
+        */
+       if (static_branch_unlikely(&page_owner_inited))
+               goto failed;
+#endif
+
        /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */
        gfp &= gfp_allowed_mask;
        alloc_gfp = gfp;
        if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags))
-               return 0;
+               goto out;
        gfp = alloc_gfp;
 
        /* Find an allowed local zone that meets the low watermark. */
@@ -5311,6 +5326,7 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
        __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account);
        zone_statistics(ac.preferred_zoneref->zone, zone, nr_account);
 
+out:
        return nr_populated;
 
 failed_irq:
@@ -5326,7 +5342,7 @@ failed:
                nr_populated++;
        }
 
-       return nr_populated;
+       goto out;
 }
 EXPORT_SYMBOL_GPL(__alloc_pages_bulk);