mm: memcg: link page counters to root if use_hierarchy is false

[linux-2.6-microblaze.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 7f74a15..3dcbf24 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -73,6 +73,9 @@ EXPORT_SYMBOL(memory_cgrp_subsys);
 
 struct mem_cgroup *root_mem_cgroup __read_mostly;
 
+/* Active memory cgroup to use from an interrupt context */
+DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
+
 /* Socket memory accounting disabled? */
 static bool cgroup_memory_nosocket;
 
@@ -1061,23 +1064,56 @@ struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
 }
 EXPORT_SYMBOL(get_mem_cgroup_from_page);
 
-/**
- * If current->active_memcg is non-NULL, do not fallback to current->mm->memcg.
- */
-static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
+static __always_inline struct mem_cgroup *active_memcg(void)
 {
-       if (unlikely(current->active_memcg)) {
-               struct mem_cgroup *memcg;
+       if (in_interrupt())
+               return this_cpu_read(int_active_memcg);
+       else
+               return current->active_memcg;
+}
 
-               rcu_read_lock();
+static __always_inline struct mem_cgroup *get_active_memcg(void)
+{
+       struct mem_cgroup *memcg;
+
+       rcu_read_lock();
+       memcg = active_memcg();
+       if (memcg) {
                /* current->active_memcg must hold a ref. */
-               if (WARN_ON_ONCE(!css_tryget(&current->active_memcg->css)))
+               if (WARN_ON_ONCE(!css_tryget(&memcg->css)))
                        memcg = root_mem_cgroup;
                else
                        memcg = current->active_memcg;
-               rcu_read_unlock();
-               return memcg;
        }
+       rcu_read_unlock();
+
+       return memcg;
+}
+
+static __always_inline bool memcg_kmem_bypass(void)
+{
+       /* Allow remote memcg charging from any context. */
+       if (unlikely(active_memcg()))
+               return false;
+
+       /* Memcg to charge can't be determined. */
+       if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
+               return true;
+
+       return false;
+}
+
+/**
+ * If active memcg is set, do not fallback to current->mm->memcg.
+ */
+static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
+{
+       if (memcg_kmem_bypass())
+               return NULL;
+
+       if (unlikely(active_memcg()))
+               return get_active_memcg();
+
        return get_mem_cgroup_from_mm(current->mm);
 }
 
@@ -2933,12 +2969,12 @@ __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
        struct obj_cgroup *objcg = NULL;
        struct mem_cgroup *memcg;
 
-       if (unlikely(!current->mm && !current->active_memcg))
+       if (memcg_kmem_bypass())
                return NULL;
 
        rcu_read_lock();
-       if (unlikely(current->active_memcg))
-               memcg = rcu_dereference(current->active_memcg);
+       if (unlikely(active_memcg()))
+               memcg = active_memcg();
        else
                memcg = mem_cgroup_from_task(current);
 
@@ -3059,19 +3095,16 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
        struct mem_cgroup *memcg;
        int ret = 0;
 
-       if (memcg_kmem_bypass())
-               return 0;
-
        memcg = get_mem_cgroup_from_current();
-       if (!mem_cgroup_is_root(memcg)) {
+       if (memcg && !mem_cgroup_is_root(memcg)) {
                ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
                if (!ret) {
                        page->mem_cgroup = memcg;
                        __SetPageKmemcg(page);
                        return 0;
                }
+               css_put(&memcg->css);
        }
-       css_put(&memcg->css);
        return ret;
 }
 
@@ -4077,11 +4110,17 @@ static int memcg_stat_show(struct seq_file *m, void *v)
                           (u64)memsw * PAGE_SIZE);
 
        for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
+               unsigned long nr;
+
                if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
                        continue;
+               nr = memcg_page_state(memcg, memcg1_stats[i]);
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+               if (memcg1_stats[i] == NR_ANON_THPS)
+                       nr *= HPAGE_PMD_NR;
+#endif
                seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
-                          (u64)memcg_page_state(memcg, memcg1_stats[i]) *
-                          PAGE_SIZE);
+                                               (u64)nr * PAGE_SIZE);
        }
 
        for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
@@ -5290,12 +5329,12 @@ static struct cgroup_subsys_state * __ref
 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
-       struct mem_cgroup *memcg;
+       struct mem_cgroup *memcg, *old_memcg;
        long error = -ENOMEM;
 
-       memalloc_use_memcg(parent);
+       old_memcg = set_active_memcg(parent);
        memcg = mem_cgroup_alloc();
-       memalloc_unuse_memcg();
+       set_active_memcg(old_memcg);
        if (IS_ERR(memcg))
                return ERR_CAST(memcg);
 
@@ -5306,17 +5345,22 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
                memcg->swappiness = mem_cgroup_swappiness(parent);
                memcg->oom_kill_disable = parent->oom_kill_disable;
        }
-       if (parent && parent->use_hierarchy) {
+       if (!parent) {
+               page_counter_init(&memcg->memory, NULL);
+               page_counter_init(&memcg->swap, NULL);
+               page_counter_init(&memcg->kmem, NULL);
+               page_counter_init(&memcg->tcpmem, NULL);
+       } else if (parent->use_hierarchy) {
                memcg->use_hierarchy = true;
                page_counter_init(&memcg->memory, &parent->memory);
                page_counter_init(&memcg->swap, &parent->swap);
                page_counter_init(&memcg->kmem, &parent->kmem);
                page_counter_init(&memcg->tcpmem, &parent->tcpmem);
        } else {
-               page_counter_init(&memcg->memory, NULL);
-               page_counter_init(&memcg->swap, NULL);
-               page_counter_init(&memcg->kmem, NULL);
-               page_counter_init(&memcg->tcpmem, NULL);
+               page_counter_init(&memcg->memory, &root_mem_cgroup->memory);
+               page_counter_init(&memcg->swap, &root_mem_cgroup->swap);
+               page_counter_init(&memcg->kmem, &root_mem_cgroup->kmem);
+               page_counter_init(&memcg->tcpmem, &root_mem_cgroup->tcpmem);
                /*
                 * Deeper hierachy with use_hierarchy == false doesn't make
                 * much sense so let cgroup subsystem know about this