memcg: switch lruvec stats to rstat

[linux-2.6-microblaze.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 81e15a6..400d210 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -660,23 +660,11 @@ static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx)
        return x;
 }
 
-static struct mem_cgroup_per_node *
-parent_nodeinfo(struct mem_cgroup_per_node *pn, int nid)
-{
-       struct mem_cgroup *parent;
-
-       parent = parent_mem_cgroup(pn->memcg);
-       if (!parent)
-               return NULL;
-       return parent->nodeinfo[nid];
-}
-
 void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
                              int val)
 {
        struct mem_cgroup_per_node *pn;
        struct mem_cgroup *memcg;
-       long x, threshold = MEMCG_CHARGE_BATCH;
 
        pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
        memcg = pn->memcg;
@@ -685,21 +673,7 @@ void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
        __mod_memcg_state(memcg, idx, val);
 
        /* Update lruvec */
-       __this_cpu_add(pn->lruvec_stat_local->count[idx], val);
-
-       if (vmstat_item_in_bytes(idx))
-               threshold <<= PAGE_SHIFT;
-
-       x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
-       if (unlikely(abs(x) > threshold)) {
-               pg_data_t *pgdat = lruvec_pgdat(lruvec);
-               struct mem_cgroup_per_node *pi;
-
-               for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
-                       atomic_long_add(x, &pi->lruvec_stat[idx]);
-               x = 0;
-       }
-       __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
+       __this_cpu_add(pn->lruvec_stats_percpu->state[idx], val);
 }
 
 /**
@@ -2278,40 +2252,13 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
        mutex_unlock(&percpu_charge_mutex);
 }
 
-static void memcg_flush_lruvec_page_state(struct mem_cgroup *memcg, int cpu)
-{
-       int nid;
-
-       for_each_node(nid) {
-               struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
-               unsigned long stat[NR_VM_NODE_STAT_ITEMS];
-               struct batched_lruvec_stat *lstatc;
-               int i;
-
-               lstatc = per_cpu_ptr(pn->lruvec_stat_cpu, cpu);
-               for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
-                       stat[i] = lstatc->count[i];
-                       lstatc->count[i] = 0;
-               }
-
-               do {
-                       for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
-                               atomic_long_add(stat[i], &pn->lruvec_stat[i]);
-               } while ((pn = parent_nodeinfo(pn, nid)));
-       }
-}
-
 static int memcg_hotplug_cpu_dead(unsigned int cpu)
 {
        struct memcg_stock_pcp *stock;
-       struct mem_cgroup *memcg;
 
        stock = &per_cpu(memcg_stock, cpu);
        drain_stock(stock);
 
-       for_each_mem_cgroup(memcg)
-               memcg_flush_lruvec_page_state(memcg, cpu);
-
        return 0;
 }
 
@@ -5118,17 +5065,9 @@ static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
        if (!pn)
                return 1;
 
-       pn->lruvec_stat_local = alloc_percpu_gfp(struct lruvec_stat,
-                                                GFP_KERNEL_ACCOUNT);
-       if (!pn->lruvec_stat_local) {
-               kfree(pn);
-               return 1;
-       }
-
-       pn->lruvec_stat_cpu = alloc_percpu_gfp(struct batched_lruvec_stat,
-                                              GFP_KERNEL_ACCOUNT);
-       if (!pn->lruvec_stat_cpu) {
-               free_percpu(pn->lruvec_stat_local);
+       pn->lruvec_stats_percpu = alloc_percpu_gfp(struct lruvec_stats_percpu,
+                                                  GFP_KERNEL_ACCOUNT);
+       if (!pn->lruvec_stats_percpu) {
                kfree(pn);
                return 1;
        }
@@ -5149,8 +5088,7 @@ static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
        if (!pn)
                return;
 
-       free_percpu(pn->lruvec_stat_cpu);
-       free_percpu(pn->lruvec_stat_local);
+       free_percpu(pn->lruvec_stats_percpu);
        kfree(pn);
 }
 
@@ -5166,15 +5104,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg)
 
 static void mem_cgroup_free(struct mem_cgroup *memcg)
 {
-       int cpu;
-
        memcg_wb_domain_exit(memcg);
-       /*
-        * Flush percpu lruvec stats to guarantee the value
-        * correctness on parent's and all ancestor levels.
-        */
-       for_each_online_cpu(cpu)
-               memcg_flush_lruvec_page_state(memcg, cpu);
        __mem_cgroup_free(memcg);
 }
 
@@ -5407,7 +5337,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
        struct mem_cgroup *parent = parent_mem_cgroup(memcg);
        struct memcg_vmstats_percpu *statc;
        long delta, v;
-       int i;
+       int i, nid;
 
        statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
 
@@ -5455,6 +5385,36 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
                if (parent)
                        parent->vmstats.events_pending[i] += delta;
        }
+
+       for_each_node_state(nid, N_MEMORY) {
+               struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
+               struct mem_cgroup_per_node *ppn = NULL;
+               struct lruvec_stats_percpu *lstatc;
+
+               if (parent)
+                       ppn = parent->nodeinfo[nid];
+
+               lstatc = per_cpu_ptr(pn->lruvec_stats_percpu, cpu);
+
+               for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
+                       delta = pn->lruvec_stats.state_pending[i];
+                       if (delta)
+                               pn->lruvec_stats.state_pending[i] = 0;
+
+                       v = READ_ONCE(lstatc->state[i]);
+                       if (v != lstatc->state_prev[i]) {
+                               delta += v - lstatc->state_prev[i];
+                               lstatc->state_prev[i] = v;
+                       }
+
+                       if (!delta)
+                               continue;
+
+                       pn->lruvec_stats.state[i] += delta;
+                       if (ppn)
+                               ppn->lruvec_stats.state_pending[i] += delta;
+               }
+       }
 }
 
 #ifdef CONFIG_MMU
@@ -6388,6 +6348,8 @@ static int memory_numa_stat_show(struct seq_file *m, void *v)
        int i;
        struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
 
+       cgroup_rstat_flush(memcg->css.cgroup);
+
        for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
                int nid;