mm, compaction: capture a page under direct compaction

[linux-2.6-microblaze.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 2e132b9..09bf2c5 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -789,6 +789,57 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
        return 0;
 }
 
+#ifdef CONFIG_COMPACTION
+static inline struct capture_control *task_capc(struct zone *zone)
+{
+       struct capture_control *capc = current->capture_control;
+
+       return capc &&
+               !(current->flags & PF_KTHREAD) &&
+               !capc->page &&
+               capc->cc->zone == zone &&
+               capc->cc->direct_compaction ? capc : NULL;
+}
+
+static inline bool
+compaction_capture(struct capture_control *capc, struct page *page,
+                  int order, int migratetype)
+{
+       if (!capc || order != capc->cc->order)
+               return false;
+
+       /* Do not accidentally pollute CMA or isolated regions*/
+       if (is_migrate_cma(migratetype) ||
+           is_migrate_isolate(migratetype))
+               return false;
+
+       /*
+        * Do not let lower order allocations polluate a movable pageblock.
+        * This might let an unmovable request use a reclaimable pageblock
+        * and vice-versa but no more than normal fallback logic which can
+        * have trouble finding a high-order free page.
+        */
+       if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
+               return false;
+
+       capc->page = page;
+       return true;
+}
+
+#else
+static inline struct capture_control *task_capc(struct zone *zone)
+{
+       return NULL;
+}
+
+static inline bool
+compaction_capture(struct capture_control *capc, struct page *page,
+                  int order, int migratetype)
+{
+       return false;
+}
+#endif /* CONFIG_COMPACTION */
+
 /*
  * Freeing function for a buddy system allocator.
  *
@@ -822,6 +873,7 @@ static inline void __free_one_page(struct page *page,
        unsigned long uninitialized_var(buddy_pfn);
        struct page *buddy;
        unsigned int max_order;
+       struct capture_control *capc = task_capc(zone);
 
        max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
 
@@ -837,6 +889,11 @@ static inline void __free_one_page(struct page *page,
 
 continue_merging:
        while (order < max_order - 1) {
+               if (compaction_capture(capc, page, order, migratetype)) {
+                       __mod_zone_freepage_state(zone, -(1 << order),
+                                                               migratetype);
+                       return;
+               }
                buddy_pfn = __find_buddy_pfn(pfn, order);
                buddy = page + (buddy_pfn - pfn);
 
@@ -3710,7 +3767,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
                unsigned int alloc_flags, const struct alloc_context *ac,
                enum compact_priority prio, enum compact_result *compact_result)
 {
-       struct page *page;
+       struct page *page = NULL;
        unsigned long pflags;
        unsigned int noreclaim_flag;
 
@@ -3721,13 +3778,15 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
        noreclaim_flag = memalloc_noreclaim_save();
 
        *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
-                                                                       prio);
+                                                               prio, &page);
 
        memalloc_noreclaim_restore(noreclaim_flag);
        psi_memstall_leave(&pflags);
 
-       if (*compact_result <= COMPACT_INACTIVE)
+       if (*compact_result <= COMPACT_INACTIVE) {
+               WARN_ON_ONCE(page);
                return NULL;
+       }
 
        /*
         * At least in one zone compaction wasn't deferred or skipped, so let's
@@ -3735,7 +3794,13 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
         */
        count_vm_event(COMPACTSTALL);
 
-       page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
+       /* Prep a captured page if available */
+       if (page)
+               prep_new_page(page, order, gfp_mask, alloc_flags);
+
+       /* Try get a page from the freelist if available */
+       if (!page)
+               page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
 
        if (page) {
                struct zone *zone = page_zone(page);