Merge tag 'io_uring-5.10-2020-10-24' of git://git.kernel.dk/linux-block

[linux-2.6-microblaze.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 7f74a15..3a24e3b 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -73,6 +73,9 @@ EXPORT_SYMBOL(memory_cgrp_subsys);
 
 struct mem_cgroup *root_mem_cgroup __read_mostly;
 
+/* Active memory cgroup to use from an interrupt context */
+DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
+
 /* Socket memory accounting disabled? */
 static bool cgroup_memory_nosocket;
 
@@ -1061,23 +1064,56 @@ struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
 }
 EXPORT_SYMBOL(get_mem_cgroup_from_page);
 
-/**
- * If current->active_memcg is non-NULL, do not fallback to current->mm->memcg.
- */
-static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
+static __always_inline struct mem_cgroup *active_memcg(void)
 {
-       if (unlikely(current->active_memcg)) {
-               struct mem_cgroup *memcg;
+       if (in_interrupt())
+               return this_cpu_read(int_active_memcg);
+       else
+               return current->active_memcg;
+}
 
-               rcu_read_lock();
+static __always_inline struct mem_cgroup *get_active_memcg(void)
+{
+       struct mem_cgroup *memcg;
+
+       rcu_read_lock();
+       memcg = active_memcg();
+       if (memcg) {
                /* current->active_memcg must hold a ref. */
-               if (WARN_ON_ONCE(!css_tryget(&current->active_memcg->css)))
+               if (WARN_ON_ONCE(!css_tryget(&memcg->css)))
                        memcg = root_mem_cgroup;
                else
                        memcg = current->active_memcg;
-               rcu_read_unlock();
-               return memcg;
        }
+       rcu_read_unlock();
+
+       return memcg;
+}
+
+static __always_inline bool memcg_kmem_bypass(void)
+{
+       /* Allow remote memcg charging from any context. */
+       if (unlikely(active_memcg()))
+               return false;
+
+       /* Memcg to charge can't be determined. */
+       if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
+               return true;
+
+       return false;
+}
+
+/**
+ * If active memcg is set, do not fallback to current->mm->memcg.
+ */
+static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
+{
+       if (memcg_kmem_bypass())
+               return NULL;
+
+       if (unlikely(active_memcg()))
+               return get_active_memcg();
+
        return get_mem_cgroup_from_mm(current->mm);
 }
 
@@ -2933,12 +2969,12 @@ __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
        struct obj_cgroup *objcg = NULL;
        struct mem_cgroup *memcg;
 
-       if (unlikely(!current->mm && !current->active_memcg))
+       if (memcg_kmem_bypass())
                return NULL;
 
        rcu_read_lock();
-       if (unlikely(current->active_memcg))
-               memcg = rcu_dereference(current->active_memcg);
+       if (unlikely(active_memcg()))
+               memcg = active_memcg();
        else
                memcg = mem_cgroup_from_task(current);
 
@@ -3059,19 +3095,16 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
        struct mem_cgroup *memcg;
        int ret = 0;
 
-       if (memcg_kmem_bypass())
-               return 0;
-
        memcg = get_mem_cgroup_from_current();
-       if (!mem_cgroup_is_root(memcg)) {
+       if (memcg && !mem_cgroup_is_root(memcg)) {
                ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
                if (!ret) {
                        page->mem_cgroup = memcg;
                        __SetPageKmemcg(page);
                        return 0;
                }
+               css_put(&memcg->css);
        }
-       css_put(&memcg->css);
        return ret;
 }
 
@@ -5290,12 +5323,12 @@ static struct cgroup_subsys_state * __ref
 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 {
        struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
-       struct mem_cgroup *memcg;
+       struct mem_cgroup *memcg, *old_memcg;
        long error = -ENOMEM;
 
-       memalloc_use_memcg(parent);
+       old_memcg = set_active_memcg(parent);
        memcg = mem_cgroup_alloc();
-       memalloc_unuse_memcg();
+       set_active_memcg(old_memcg);
        if (IS_ERR(memcg))
                return ERR_CAST(memcg);