Merge tag 'kvmarm-5.15' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmar...

[linux-2.6-microblaze.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 3b97e17..856b175 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -840,21 +840,24 @@ void init_mem_debugging_and_hardening(void)
        }
 #endif
 
-       if (_init_on_alloc_enabled_early) {
-               if (page_poisoning_requested)
-                       pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
-                               "will take precedence over init_on_alloc\n");
-               else
-                       static_branch_enable(&init_on_alloc);
-       }
-       if (_init_on_free_enabled_early) {
-               if (page_poisoning_requested)
-                       pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
-                               "will take precedence over init_on_free\n");
-               else
-                       static_branch_enable(&init_on_free);
+       if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&
+           page_poisoning_requested) {
+               pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
+                       "will take precedence over init_on_alloc and init_on_free\n");
+               _init_on_alloc_enabled_early = false;
+               _init_on_free_enabled_early = false;
        }
 
+       if (_init_on_alloc_enabled_early)
+               static_branch_enable(&init_on_alloc);
+       else
+               static_branch_disable(&init_on_alloc);
+
+       if (_init_on_free_enabled_early)
+               static_branch_enable(&init_on_free);
+       else
+               static_branch_disable(&init_on_free);
+
 #ifdef CONFIG_DEBUG_PAGEALLOC
        if (!debug_pagealloc_enabled())
                return;
@@ -3820,7 +3823,7 @@ static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
 
 #endif /* CONFIG_FAIL_PAGE_ALLOC */
 
-static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
+noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
 {
        return __should_fail_alloc_page(gfp_mask, order);
 }
@@ -5221,9 +5224,6 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
        unsigned int alloc_flags = ALLOC_WMARK_LOW;
        int nr_populated = 0, nr_account = 0;
 
-       if (unlikely(nr_pages <= 0))
-               return 0;
-
        /*
         * Skip populated array elements to determine if any pages need
         * to be allocated before disabling IRQs.
@@ -5231,19 +5231,35 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
        while (page_array && nr_populated < nr_pages && page_array[nr_populated])
                nr_populated++;
 
+       /* No pages requested? */
+       if (unlikely(nr_pages <= 0))
+               goto out;
+
        /* Already populated array? */
        if (unlikely(page_array && nr_pages - nr_populated == 0))
-               return nr_populated;
+               goto out;
 
        /* Use the single page allocator for one page. */
        if (nr_pages - nr_populated == 1)
                goto failed;
 
+#ifdef CONFIG_PAGE_OWNER
+       /*
+        * PAGE_OWNER may recurse into the allocator to allocate space to
+        * save the stack with pagesets.lock held. Releasing/reacquiring
+        * removes much of the performance benefit of bulk allocation so
+        * force the caller to allocate one page at a time as it'll have
+        * similar performance to added complexity to the bulk allocator.
+        */
+       if (static_branch_unlikely(&page_owner_inited))
+               goto failed;
+#endif
+
        /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */
        gfp &= gfp_allowed_mask;
        alloc_gfp = gfp;
        if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags))
-               return 0;
+               goto out;
        gfp = alloc_gfp;
 
        /* Find an allowed local zone that meets the low watermark. */
@@ -5311,6 +5327,7 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
        __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account);
        zone_statistics(ac.preferred_zoneref->zone, zone, nr_account);
 
+out:
        return nr_populated;
 
 failed_irq:
@@ -5326,7 +5343,7 @@ failed:
                nr_populated++;
        }
 
-       return nr_populated;
+       goto out;
 }
 EXPORT_SYMBOL_GPL(__alloc_pages_bulk);