doc/vm/page_owner.rst: remove content related to -c option

[linux-2.6-microblaze.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 94a220d..2db9578 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1108,6 +1108,9 @@ continue_merging:
 
                buddy_pfn = __find_buddy_pfn(pfn, order);
                buddy = page + (buddy_pfn - pfn);
+
+               if (!page_is_buddy(page, buddy, order))
+                       goto done_merging;
                buddy_mt = get_pageblock_migratetype(buddy);
 
                if (migratetype != buddy_mt
@@ -2343,9 +2346,43 @@ static inline bool check_new_pcp(struct page *page, unsigned int order)
 }
 #endif /* CONFIG_DEBUG_VM */
 
+static inline bool should_skip_kasan_unpoison(gfp_t flags, bool init_tags)
+{
+       /* Don't skip if a software KASAN mode is enabled. */
+       if (IS_ENABLED(CONFIG_KASAN_GENERIC) ||
+           IS_ENABLED(CONFIG_KASAN_SW_TAGS))
+               return false;
+
+       /* Skip, if hardware tag-based KASAN is not enabled. */
+       if (!kasan_hw_tags_enabled())
+               return true;
+
+       /*
+        * With hardware tag-based KASAN enabled, skip if either:
+        *
+        * 1. Memory tags have already been cleared via tag_clear_highpage().
+        * 2. Skipping has been requested via __GFP_SKIP_KASAN_UNPOISON.
+        */
+       return init_tags || (flags & __GFP_SKIP_KASAN_UNPOISON);
+}
+
+static inline bool should_skip_init(gfp_t flags)
+{
+       /* Don't skip, if hardware tag-based KASAN is not enabled. */
+       if (!kasan_hw_tags_enabled())
+               return false;
+
+       /* For hardware tag-based KASAN, skip if requested. */
+       return (flags & __GFP_SKIP_ZERO);
+}
+
 inline void post_alloc_hook(struct page *page, unsigned int order,
                                gfp_t gfp_flags)
 {
+       bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) &&
+                       !should_skip_init(gfp_flags);
+       bool init_tags = init && (gfp_flags & __GFP_ZEROTAGS);
+
        set_page_private(page, 0);
        set_page_refcounted(page);
 
@@ -2361,27 +2398,38 @@ inline void post_alloc_hook(struct page *page, unsigned int order,
 
        /*
         * As memory initialization might be integrated into KASAN,
-        * kasan_alloc_pages and kernel_init_free_pages must be
+        * KASAN unpoisoning and memory initializion code must be
         * kept together to avoid discrepancies in behavior.
         */
-       if (kasan_has_integrated_init()) {
-               kasan_alloc_pages(page, order, gfp_flags);
-       } else {
-               bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags);
 
-               kasan_unpoison_pages(page, order, init);
+       /*
+        * If memory tags should be zeroed (which happens only when memory
+        * should be initialized as well).
+        */
+       if (init_tags) {
+               int i;
 
-               if (init) {
-                       if (gfp_flags & __GFP_ZEROTAGS) {
-                               int i;
+               /* Initialize both memory and tags. */
+               for (i = 0; i != 1 << order; ++i)
+                       tag_clear_highpage(page + i);
 
-                               for (i = 0; i < 1 << order; i++)
-                                       tag_clear_highpage(page + i);
-                       } else {
-                               kernel_init_free_pages(page, 1 << order);
-                       }
-               }
+               /* Note that memory is already initialized by the loop above. */
+               init = false;
+       }
+       if (!should_skip_kasan_unpoison(gfp_flags, init_tags)) {
+               /* Unpoison shadow memory or set memory tags. */
+               kasan_unpoison_pages(page, order, init);
+
+               /* Note that memory is already initialized by KASAN. */
+               if (kasan_has_integrated_init())
+                       init = false;
        }
+       /* If memory is still not initialized, do it now. */
+       if (init)
+               kernel_init_free_pages(page, 1 << order);
+       /* Propagate __GFP_SKIP_KASAN_POISON to page flags. */
+       if (kasan_hw_tags_enabled() && (gfp_flags & __GFP_SKIP_KASAN_POISON))
+               SetPageSkipKASanPoison(page);
 
        set_page_owner(page, order, gfp_flags);
        page_table_check_alloc(page, order);
@@ -8319,6 +8367,7 @@ static int page_alloc_cpu_dead(unsigned int cpu)
        struct zone *zone;
 
        lru_add_drain_cpu(cpu);
+       mlock_page_drain_remote(cpu);
        drain_pages(cpu);
 
        /*