blk-throttle: Fix that bps of child could exceed bps limited in parent

[linux-2.6-microblaze.git] / mm / workingset.c

diff --git a/mm/workingset.c b/mm/workingset.c
index a5e8486..ae7e984 100644 (file)
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -187,7 +187,6 @@ static unsigned int bucket_order __read_mostly;
 static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction,
                         bool workingset)
 {
-       eviction >>= bucket_order;
        eviction &= EVICTION_MASK;
        eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid;
        eviction = (eviction << NODES_SHIFT) | pgdat->node_id;
@@ -212,10 +211,107 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat,
 
        *memcgidp = memcgid;
        *pgdat = NODE_DATA(nid);
-       *evictionp = entry << bucket_order;
+       *evictionp = entry;
        *workingsetp = workingset;
 }
 
+#ifdef CONFIG_LRU_GEN
+
+static void *lru_gen_eviction(struct folio *folio)
+{
+       int hist;
+       unsigned long token;
+       unsigned long min_seq;
+       struct lruvec *lruvec;
+       struct lru_gen_struct *lrugen;
+       int type = folio_is_file_lru(folio);
+       int delta = folio_nr_pages(folio);
+       int refs = folio_lru_refs(folio);
+       int tier = lru_tier_from_refs(refs);
+       struct mem_cgroup *memcg = folio_memcg(folio);
+       struct pglist_data *pgdat = folio_pgdat(folio);
+
+       BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT);
+
+       lruvec = mem_cgroup_lruvec(memcg, pgdat);
+       lrugen = &lruvec->lrugen;
+       min_seq = READ_ONCE(lrugen->min_seq[type]);
+       token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0);
+
+       hist = lru_hist_from_seq(min_seq);
+       atomic_long_add(delta, &lrugen->evicted[hist][type][tier]);
+
+       return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs);
+}
+
+static void lru_gen_refault(struct folio *folio, void *shadow)
+{
+       int hist, tier, refs;
+       int memcg_id;
+       bool workingset;
+       unsigned long token;
+       unsigned long min_seq;
+       struct lruvec *lruvec;
+       struct lru_gen_struct *lrugen;
+       struct mem_cgroup *memcg;
+       struct pglist_data *pgdat;
+       int type = folio_is_file_lru(folio);
+       int delta = folio_nr_pages(folio);
+
+       unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset);
+
+       if (pgdat != folio_pgdat(folio))
+               return;
+
+       rcu_read_lock();
+
+       memcg = folio_memcg_rcu(folio);
+       if (memcg_id != mem_cgroup_id(memcg))
+               goto unlock;
+
+       lruvec = mem_cgroup_lruvec(memcg, pgdat);
+       lrugen = &lruvec->lrugen;
+
+       min_seq = READ_ONCE(lrugen->min_seq[type]);
+       if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH)))
+               goto unlock;
+
+       hist = lru_hist_from_seq(min_seq);
+       /* see the comment in folio_lru_refs() */
+       refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset;
+       tier = lru_tier_from_refs(refs);
+
+       atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]);
+       mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta);
+
+       /*
+        * Count the following two cases as stalls:
+        * 1. For pages accessed through page tables, hotter pages pushed out
+        *    hot pages which refaulted immediately.
+        * 2. For pages accessed multiple times through file descriptors,
+        *    numbers of accesses might have been out of the range.
+        */
+       if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) {
+               folio_set_workingset(folio);
+               mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta);
+       }
+unlock:
+       rcu_read_unlock();
+}
+
+#else /* !CONFIG_LRU_GEN */
+
+static void *lru_gen_eviction(struct folio *folio)
+{
+       return NULL;
+}
+
+static void lru_gen_refault(struct folio *folio, void *shadow)
+{
+}
+
+#endif /* CONFIG_LRU_GEN */
+
 /**
  * workingset_age_nonresident - age non-resident entries as LRU ages
  * @lruvec: the lruvec that was aged
@@ -264,10 +360,14 @@ void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg)
        VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
        VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
 
+       if (lru_gen_enabled())
+               return lru_gen_eviction(folio);
+
        lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
        /* XXX: target_memcg can be NULL, go through lruvec */
        memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
        eviction = atomic_long_read(&lruvec->nonresident_age);
+       eviction >>= bucket_order;
        workingset_age_nonresident(lruvec, folio_nr_pages(folio));
        return pack_shadow(memcgid, pgdat, eviction,
                                folio_test_workingset(folio));
@@ -298,7 +398,13 @@ void workingset_refault(struct folio *folio, void *shadow)
        int memcgid;
        long nr;
 
+       if (lru_gen_enabled()) {
+               lru_gen_refault(folio, shadow);
+               return;
+       }
+
        unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset);
+       eviction <<= bucket_order;
 
        rcu_read_lock();
        /*