mm, slub: move slub_debug static key enabling outside slab_mutex

[linux-2.6-microblaze.git] / mm / compaction.c

diff --git a/mm/compaction.c b/mm/compaction.c
index c4d8007..84fde27 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -788,7 +788,7 @@ static bool too_many_isolated(pg_data_t *pgdat)
  * Isolate all pages that can be migrated from the range specified by
  * [low_pfn, end_pfn). The range is expected to be within same pageblock.
  * Returns errno, like -EAGAIN or -EINTR in case e.g signal pending or congestion,
- * or 0.
+ * -ENOMEM in case we could not allocate a page, or 0.
  * cc->migrate_pfn will contain the next pfn to scan.
  *
  * The pages are isolated on cc->migratepages list (not required to be empty),
@@ -906,6 +906,38 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                        valid_page = page;
                }
 
+               if (PageHuge(page) && cc->alloc_contig) {
+                       ret = isolate_or_dissolve_huge_page(page, &cc->migratepages);
+
+                       /*
+                        * Fail isolation in case isolate_or_dissolve_huge_page()
+                        * reports an error. In case of -ENOMEM, abort right away.
+                        */
+                       if (ret < 0) {
+                                /* Do not report -EBUSY down the chain */
+                               if (ret == -EBUSY)
+                                       ret = 0;
+                               low_pfn += (1UL << compound_order(page)) - 1;
+                               goto isolate_fail;
+                       }
+
+                       if (PageHuge(page)) {
+                               /*
+                                * Hugepage was successfully isolated and placed
+                                * on the cc->migratepages list.
+                                */
+                               low_pfn += compound_nr(page) - 1;
+                               goto isolate_success_no_list;
+                       }
+
+                       /*
+                        * Ok, the hugepage was dissolved. Now these pages are
+                        * Buddy and cannot be re-allocated because they are
+                        * isolated. Fall-through as the check below handles
+                        * Buddy pages.
+                        */
+               }
+
                /*
                 * Skip if free. We read page order here without zone lock
                 * which is generally unsafe, but the race window is small and
@@ -1039,6 +1071,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 
 isolate_success:
                list_add(&page->lru, &cc->migratepages);
+isolate_success_no_list:
                cc->nr_migratepages += compound_nr(page);
                nr_isolated += compound_nr(page);
 
@@ -1065,7 +1098,7 @@ isolate_fail_put:
                put_page(page);
 
 isolate_fail:
-               if (!skip_on_failure)
+               if (!skip_on_failure && ret != -ENOMEM)
                        continue;
 
                /*
@@ -1091,6 +1124,9 @@ isolate_fail:
                         */
                        next_skip_pfn += 1UL << cc->order;
                }
+
+               if (ret == -ENOMEM)
+                       break;
        }
 
        /*
@@ -1143,7 +1179,8 @@ fatal_pending:
  * @start_pfn: The first PFN to start isolating.
  * @end_pfn:   The one-past-last PFN.
  *
- * Returns -EAGAIN when contented, -EINTR in case of a signal pending or 0.
+ * Returns -EAGAIN when contented, -EINTR in case of a signal pending, -ENOMEM
+ * in case we could not allocate a page, or 0.
  */
 int
 isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
@@ -1975,8 +2012,8 @@ static unsigned int fragmentation_score_wmark(pg_data_t *pgdat, bool low)
        unsigned int wmark_low;
 
        /*
-        * Cap the low watermak to avoid excessive compaction
-        * activity in case a user sets the proactivess tunable
+        * Cap the low watermark to avoid excessive compaction
+        * activity in case a user sets the proactiveness tunable
         * close to 100 (maximum).
         */
        wmark_low = max(100U - sysctl_compaction_proactiveness, 5U);
@@ -2317,7 +2354,8 @@ compact_zone(struct compact_control *cc, struct capture_control *capc)
        trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
                                cc->free_pfn, end_pfn, sync);
 
-       migrate_prep_local();
+       /* lru_add_drain_all could be expensive with involving other CPUs */
+       lru_add_drain();
 
        while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
                int err;
@@ -2492,6 +2530,14 @@ static enum compact_result compact_zone_order(struct zone *zone, int order,
         */
        WRITE_ONCE(current->capture_control, NULL);
        *capture = READ_ONCE(capc.page);
+       /*
+        * Technically, it is also possible that compaction is skipped but
+        * the page is still captured out of luck(IRQ came and freed the page).
+        * Returning COMPACT_SUCCESS in such cases helps in properly accounting
+        * the COMPACT[STALL|FAIL] when compaction is skipped.
+        */
+       if (*capture)
+               ret = COMPACT_SUCCESS;
 
        return ret;
 }
@@ -2655,9 +2701,6 @@ static void compact_nodes(void)
                compact_node(nid);
 }
 
-/* The written value is actually unused, all memory is compacted */
-int sysctl_compact_memory;
-
 /*
  * Tunable for proactive compaction. It determines how
  * aggressively the kernel should compact memory in the
@@ -2842,7 +2885,7 @@ void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx)
  */
 static int kcompactd(void *p)
 {
-       pg_data_t *pgdat = (pg_data_t*)p;
+       pg_data_t *pgdat = (pg_data_t *)p;
        struct task_struct *tsk = current;
        unsigned int proactive_defer = 0;