mm: kmem: add lockdep assertion to obj_cgroup_memcg

obj_cgroup_memcg() is supposed to safe to prevent the returned memory
cgroup from being freed only when the caller is holding the rcu read lock
or objcg_lock or cgroup_mutex.  It is very easy to ignore thoes conditions
when users call some upper APIs which call obj_cgroup_memcg() internally
like mem_cgroup_from_slab_obj() (See the link below).  So it is better to
add lockdep assertion to obj_cgroup_memcg() to find those issues ASAP.

Because there is no user of obj_cgroup_memcg() holding objcg_lock to make
the returned memory cgroup safe, do not add objcg_lock assertion (We
should export objcg_lock if we really want to do).  Additionally, this is
some internal implementation detail of memcg and should not be accessible
outside memcg code.

Some users like __mem_cgroup_uncharge() do not care the lifetime of the
returned memory cgroup, which just want to know if the folio is charged to
a memory cgroup, therefore, they do not need to hold the needed locks.  In
which case, introduce a new helper folio_memcg_charged() to do this.
Compare it to folio_memcg(), it could eliminate a memory access of
objcg->memcg for kmem, actually, a really small gain.

[songmuchun@bytedance.com: fix split_page_memcg()]
Link: https://lkml.kernel.org/r/20240819080415.44964-1-songmuchun@bytedance.com
Link: https://lore.kernel.org/all/20240718083607.42068-1-songmuchun@bytedance.com/
Link: https://lkml.kernel.org/r/20240814093415.17634-1-songmuchun@bytedance.com
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Acked-by: Roman Gushchin <roman.gushchin@linux.dev>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 1b79760..07eadf7 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -366,11 +366,11 @@ static inline bool folio_memcg_kmem(struct folio *folio);
  * After the initialization objcg->memcg is always pointing at
  * a valid memcg, but can be atomically swapped to the parent memcg.
  *
- * The caller must ensure that the returned memcg won't be released:
- * e.g. acquire the rcu_read_lock or css_set_lock.
+ * The caller must ensure that the returned memcg won't be released.
  */
 static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg)
 {
+       lockdep_assert_once(rcu_read_lock_held() || lockdep_is_held(&cgroup_mutex));
        return READ_ONCE(objcg->memcg);
 }
 
@@ -444,6 +444,19 @@ static inline struct mem_cgroup *folio_memcg(struct folio *folio)
        return __folio_memcg(folio);
 }
 
+/*
+ * folio_memcg_charged - If a folio is charged to a memory cgroup.
+ * @folio: Pointer to the folio.
+ *
+ * Returns true if folio is charged to a memory cgroup, otherwise returns false.
+ */
+static inline bool folio_memcg_charged(struct folio *folio)
+{
+       if (folio_memcg_kmem(folio))
+               return __folio_objcg(folio) != NULL;
+       return __folio_memcg(folio) != NULL;
+}
+
 /**
  * folio_memcg_rcu - Locklessly get the memory cgroup associated with a folio.
  * @folio: Pointer to the folio.
@@ -460,7 +473,6 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
        unsigned long memcg_data = READ_ONCE(folio->memcg_data);
 
        VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
-       WARN_ON_ONCE(!rcu_read_lock_held());
 
        if (memcg_data & MEMCG_DATA_KMEM) {
                struct obj_cgroup *objcg;
@@ -469,6 +481,8 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
                return obj_cgroup_memcg(objcg);
        }
 
+       WARN_ON_ONCE(!rcu_read_lock_held());
+
        return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
 }
 

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index cb2996d..96f85fd 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2374,7 +2374,7 @@ void mem_cgroup_cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
 
 static void commit_charge(struct folio *folio, struct mem_cgroup *memcg)
 {
-       VM_BUG_ON_FOLIO(folio_memcg(folio), folio);
+       VM_BUG_ON_FOLIO(folio_memcg_charged(folio), folio);
        /*
         * Any of the following ensures page's memcg stability:
         *
@@ -3035,12 +3035,11 @@ void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
 void split_page_memcg(struct page *head, int old_order, int new_order)
 {
        struct folio *folio = page_folio(head);
-       struct mem_cgroup *memcg = folio_memcg(folio);
        int i;
        unsigned int old_nr = 1 << old_order;
        unsigned int new_nr = 1 << new_order;
 
-       if (mem_cgroup_disabled() || !memcg)
+       if (mem_cgroup_disabled() || !folio_memcg_charged(folio))
                return;
 
        for (i = new_nr; i < old_nr; i += new_nr)
@@ -3049,7 +3048,7 @@ void split_page_memcg(struct page *head, int old_order, int new_order)
        if (folio_memcg_kmem(folio))
                obj_cgroup_get_many(__folio_objcg(folio), old_nr / new_nr - 1);
        else
-               css_get_many(&memcg->css, old_nr / new_nr - 1);
+               css_get_many(&folio_memcg(folio)->css, old_nr / new_nr - 1);
 }
 
 unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
@@ -4707,7 +4706,7 @@ void __mem_cgroup_uncharge(struct folio *folio)
        struct uncharge_gather ug;
 
        /* Don't touch folio->lru of any random page, pre-check: */
-       if (!folio_memcg(folio))
+       if (!folio_memcg_charged(folio))
                return;
 
        uncharge_gather_clear(&ug);
@@ -4752,7 +4751,7 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)
                return;
 
        /* Page cache replacement: new folio already charged? */
-       if (folio_memcg(new))
+       if (folio_memcg_charged(new))
                return;
 
        memcg = folio_memcg(old);