mm/damon: adaptively adjust regions

[linux-2.6-microblaze.git] / mm / vmscan.c

diff --git a/mm/vmscan.c b/mm/vmscan.c
index d7c3cb8..eeae2f6 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -100,9 +100,12 @@ struct scan_control {
        unsigned int may_swap:1;
 
        /*
-        * Cgroups are not reclaimed below their configured memory.low,
-        * unless we threaten to OOM. If any cgroups are skipped due to
-        * memory.low and nothing was reclaimed, go back for memory.low.
+        * Cgroup memory below memory.low is protected as long as we
+        * don't threaten to OOM. If any cgroup is reclaimed at
+        * reduced force or passed over entirely due to its memory.low
+        * setting (memcg_low_skipped), and nothing is reclaimed as a
+        * result, then go back for one more cycle that reclaims the protected
+        * memory (memcg_low_reclaim) to avert OOM.
         */
        unsigned int memcg_low_reclaim:1;
        unsigned int memcg_low_skipped:1;
@@ -1499,7 +1502,8 @@ static unsigned int shrink_page_list(struct list_head *page_list,
                        if (unlikely(PageTransHuge(page)))
                                flags |= TTU_SPLIT_HUGE_PMD;
 
-                       if (!try_to_unmap(page, flags)) {
+                       try_to_unmap(page, flags);
+                       if (page_mapped(page)) {
                                stat->nr_unmap_fail += nr_pages;
                                if (!was_swapbacked && PageSwapBacked(page))
                                        stat->nr_lazyfree_fail += nr_pages;
@@ -1701,6 +1705,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
        unsigned int nr_reclaimed;
        struct page *page, *next;
        LIST_HEAD(clean_pages);
+       unsigned int noreclaim_flag;
 
        list_for_each_entry_safe(page, next, page_list, lru) {
                if (!PageHuge(page) && page_is_file_lru(page) &&
@@ -1711,8 +1716,17 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
                }
        }
 
+       /*
+        * We should be safe here since we are only dealing with file pages and
+        * we are not kswapd and therefore cannot write dirty file pages. But
+        * call memalloc_noreclaim_save() anyway, just in case these conditions
+        * change in the future.
+        */
+       noreclaim_flag = memalloc_noreclaim_save();
        nr_reclaimed = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
                                        &stat, true);
+       memalloc_noreclaim_restore(noreclaim_flag);
+
        list_splice(&clean_pages, page_list);
        mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
                            -(long)nr_reclaimed);
@@ -1810,7 +1824,7 @@ static __always_inline void update_lru_sizes(struct lruvec *lruvec,
 
 }
 
-/**
+/*
  * Isolating page from the lruvec to fill in @dst list by nr_to_scan times.
  *
  * lruvec->lru_lock is heavily contended.  Some of the functions that
@@ -2306,6 +2320,7 @@ unsigned long reclaim_pages(struct list_head *page_list)
        LIST_HEAD(node_page_list);
        struct reclaim_stat dummy_stat;
        struct page *page;
+       unsigned int noreclaim_flag;
        struct scan_control sc = {
                .gfp_mask = GFP_KERNEL,
                .priority = DEF_PRIORITY,
@@ -2314,6 +2329,8 @@ unsigned long reclaim_pages(struct list_head *page_list)
                .may_swap = 1,
        };
 
+       noreclaim_flag = memalloc_noreclaim_save();
+
        while (!list_empty(page_list)) {
                page = lru_to_page(page_list);
                if (nid == NUMA_NO_NODE) {
@@ -2350,6 +2367,8 @@ unsigned long reclaim_pages(struct list_head *page_list)
                }
        }
 
+       memalloc_noreclaim_restore(noreclaim_flag);
+
        return nr_reclaimed;
 }
 
@@ -2521,15 +2540,14 @@ out:
        for_each_evictable_lru(lru) {
                int file = is_file_lru(lru);
                unsigned long lruvec_size;
+               unsigned long low, min;
                unsigned long scan;
-               unsigned long protection;
 
                lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
-               protection = mem_cgroup_protection(sc->target_mem_cgroup,
-                                                  memcg,
-                                                  sc->memcg_low_reclaim);
+               mem_cgroup_protection(sc->target_mem_cgroup, memcg,
+                                     &min, &low);
 
-               if (protection) {
+               if (min || low) {
                        /*
                         * Scale a cgroup's reclaim pressure by proportioning
                         * its current usage to its memory.low or memory.min
@@ -2560,6 +2578,15 @@ out:
                         * hard protection.
                         */
                        unsigned long cgroup_size = mem_cgroup_size(memcg);
+                       unsigned long protection;
+
+                       /* memory.low scaling, make sure we retry before OOM */
+                       if (!sc->memcg_low_reclaim && low > min) {
+                               protection = low;
+                               sc->memcg_low_skipped = 1;
+                       } else {
+                               protection = min;
+                       }
 
                        /* Avoid TOCTOU with earlier protection check */
                        cgroup_size = max(cgroup_size, protection);
@@ -4397,11 +4424,13 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
                .may_swap = 1,
                .reclaim_idx = gfp_zone(gfp_mask),
        };
+       unsigned long pflags;
 
        trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,
                                           sc.gfp_mask);
 
        cond_resched();
+       psi_memstall_enter(&pflags);
        fs_reclaim_acquire(sc.gfp_mask);
        /*
         * We need to be able to allocate from the reserves for RECLAIM_UNMAP
@@ -4426,6 +4455,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
        current->flags &= ~PF_SWAPWRITE;
        memalloc_noreclaim_restore(noreclaim_flag);
        fs_reclaim_release(sc.gfp_mask);
+       psi_memstall_leave(&pflags);
 
        trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed);