io_uring: end waiting before task cancel attempts

[linux-2.6-microblaze.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 9c5b14f..e3c7ca7 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -20,6 +20,9 @@
  * Lockless page tracking & accounting
  * Unified hierarchy configuration model
  * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
+ *
+ * Per memcg lru locking
+ * Copyright (C) 2020 Alibaba, Inc, Alex Shi
  */
 
 #include <linux/page_counter.h>
@@ -1322,6 +1325,23 @@ int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
        return ret;
 }
 
+#ifdef CONFIG_DEBUG_VM
+void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
+{
+       struct mem_cgroup *memcg;
+
+       if (mem_cgroup_disabled())
+               return;
+
+       memcg = page_memcg(page);
+
+       if (!memcg)
+               VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != root_mem_cgroup, page);
+       else
+               VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != memcg, page);
+}
+#endif
+
 /**
  * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
  * @page: the page
@@ -1362,6 +1382,60 @@ out:
        return lruvec;
 }
 
+/**
+ * lock_page_lruvec - lock and return lruvec for a given page.
+ * @page: the page
+ *
+ * This series functions should be used in either conditions:
+ * PageLRU is cleared or unset
+ * or page->_refcount is zero
+ * or page is locked.
+ */
+struct lruvec *lock_page_lruvec(struct page *page)
+{
+       struct lruvec *lruvec;
+       struct pglist_data *pgdat = page_pgdat(page);
+
+       rcu_read_lock();
+       lruvec = mem_cgroup_page_lruvec(page, pgdat);
+       spin_lock(&lruvec->lru_lock);
+       rcu_read_unlock();
+
+       lruvec_memcg_debug(lruvec, page);
+
+       return lruvec;
+}
+
+struct lruvec *lock_page_lruvec_irq(struct page *page)
+{
+       struct lruvec *lruvec;
+       struct pglist_data *pgdat = page_pgdat(page);
+
+       rcu_read_lock();
+       lruvec = mem_cgroup_page_lruvec(page, pgdat);
+       spin_lock_irq(&lruvec->lru_lock);
+       rcu_read_unlock();
+
+       lruvec_memcg_debug(lruvec, page);
+
+       return lruvec;
+}
+
+struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags)
+{
+       struct lruvec *lruvec;
+       struct pglist_data *pgdat = page_pgdat(page);
+
+       rcu_read_lock();
+       lruvec = mem_cgroup_page_lruvec(page, pgdat);
+       spin_lock_irqsave(&lruvec->lru_lock, *flags);
+       rcu_read_unlock();
+
+       lruvec_memcg_debug(lruvec, page);
+
+       return lruvec;
+}
+
 /**
  * mem_cgroup_update_lru_size - account for adding or removing an lru page
  * @lruvec: mem_cgroup per zone lru vector
@@ -2142,6 +2216,12 @@ again:
        if (unlikely(!memcg))
                return NULL;
 
+#ifdef CONFIG_PROVE_LOCKING
+       local_irq_save(flags);
+       might_lock(&memcg->move_lock);
+       local_irq_restore(flags);
+#endif
+
        if (atomic_read(&memcg->moving_account) <= 0)
                return memcg;
 
@@ -3263,10 +3343,8 @@ void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
 #endif /* CONFIG_MEMCG_KMEM */
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-
 /*
- * Because tail pages are not marked as "used", set it. We're under
- * pgdat->lru_lock and migration entries setup in all page mappings.
+ * Because page_memcg(head) is not set on compound tails, set it now.
  */
 void mem_cgroup_split_huge_fixup(struct page *head)
 {