mm: workingset: age nonresident information alongside anonymous pages

[linux-2.6-microblaze.git] / mm / workingset.c

diff --git a/mm/workingset.c b/mm/workingset.c
index d481ea4..50b7937 100644 (file)
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -156,8 +156,8 @@
  *
  *             Implementation
  *
- * For each node's file LRU lists, a counter for inactive evictions
- * and activations is maintained (node->inactive_age).
+ * For each node's LRU lists, a counter for inactive evictions and
+ * activations is maintained (node->nonresident_age).
  *
  * On eviction, a snapshot of this counter (along with some bits to
  * identify the node) is stored in the now empty page cache
@@ -213,7 +213,17 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat,
        *workingsetp = workingset;
 }
 
-static void advance_inactive_age(struct mem_cgroup *memcg, pg_data_t *pgdat)
+/**
+ * workingset_age_nonresident - age non-resident entries as LRU ages
+ * @memcg: the lruvec that was aged
+ * @nr_pages: the number of pages to count
+ *
+ * As in-memory pages are aged, non-resident pages need to be aged as
+ * well, in order for the refault distances later on to be comparable
+ * to the in-memory dimensions. This function allows reclaim and LRU
+ * operations to drive the non-resident aging along in parallel.
+ */
+void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages)
 {
        /*
         * Reclaiming a cgroup means reclaiming all its children in a
@@ -227,11 +237,8 @@ static void advance_inactive_age(struct mem_cgroup *memcg, pg_data_t *pgdat)
         * the root cgroup's, age as well.
         */
        do {
-               struct lruvec *lruvec;
-
-               lruvec = mem_cgroup_lruvec(memcg, pgdat);
-               atomic_long_inc(&lruvec->inactive_age);
-       } while (memcg && (memcg = parent_mem_cgroup(memcg)));
+               atomic_long_add(nr_pages, &lruvec->nonresident_age);
+       } while ((lruvec = parent_lruvec(lruvec)));
 }
 
 /**
@@ -254,12 +261,11 @@ void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg)
        VM_BUG_ON_PAGE(page_count(page), page);
        VM_BUG_ON_PAGE(!PageLocked(page), page);
 
-       advance_inactive_age(page_memcg(page), pgdat);
-
        lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
+       workingset_age_nonresident(lruvec, hpage_nr_pages(page));
        /* XXX: target_memcg can be NULL, go through lruvec */
        memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
-       eviction = atomic_long_read(&lruvec->inactive_age);
+       eviction = atomic_long_read(&lruvec->nonresident_age);
        return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page));
 }
 
@@ -309,20 +315,20 @@ void workingset_refault(struct page *page, void *shadow)
        if (!mem_cgroup_disabled() && !eviction_memcg)
                goto out;
        eviction_lruvec = mem_cgroup_lruvec(eviction_memcg, pgdat);
-       refault = atomic_long_read(&eviction_lruvec->inactive_age);
+       refault = atomic_long_read(&eviction_lruvec->nonresident_age);
 
        /*
         * Calculate the refault distance
         *
         * The unsigned subtraction here gives an accurate distance
-        * across inactive_age overflows in most cases. There is a
+        * across nonresident_age overflows in most cases. There is a
         * special case: usually, shadow entries have a short lifetime
         * and are either refaulted or reclaimed along with the inode
         * before they get too old.  But it is not impossible for the
-        * inactive_age to lap a shadow entry in the field, which can
-        * then result in a false small refault distance, leading to a
-        * false activation should this old entry actually refault
-        * again.  However, earlier kernels used to deactivate
+        * nonresident_age to lap a shadow entry in the field, which
+        * can then result in a false small refault distance, leading
+        * to a false activation should this old entry actually
+        * refault again.  However, earlier kernels used to deactivate
         * unconditionally with *every* reclaim invocation for the
         * longest time, so the occasional inappropriate activation
         * leading to pressure on the active list is not a problem.
@@ -359,7 +365,7 @@ void workingset_refault(struct page *page, void *shadow)
                goto out;
 
        SetPageActive(page);
-       advance_inactive_age(memcg, pgdat);
+       workingset_age_nonresident(lruvec, hpage_nr_pages(page));
        inc_lruvec_state(lruvec, WORKINGSET_ACTIVATE);
 
        /* Page was active prior to eviction */
@@ -382,6 +388,7 @@ out:
 void workingset_activation(struct page *page)
 {
        struct mem_cgroup *memcg;
+       struct lruvec *lruvec;
 
        rcu_read_lock();
        /*
@@ -394,7 +401,8 @@ void workingset_activation(struct page *page)
        memcg = page_memcg_rcu(page);
        if (!mem_cgroup_disabled() && !memcg)
                goto out;
-       advance_inactive_age(memcg, page_pgdat(page));
+       lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
+       workingset_age_nonresident(lruvec, hpage_nr_pages(page));
 out:
        rcu_read_unlock();
 }