mm/mglru: rework refault detection

With anon and file min_seq being able to move independently, rework
workingset protection as well so that the comparison of refaults between
anon and file is always on an equal footing.

Specifically, make lru_gen_test_recent() return true for refaults
happening within the distance of MAX_NR_GENS.  For example, if min_seq of
a type is max_seq-MIN_NR_GENS, refaults from min_seq-1, i.e.,
max_seq-MIN_NR_GENS-1, are also considered recent, since the distance
max_seq-(max_seq-MIN_NR_GENS-1), i.e., MIN_NR_GENS+1 is less than
MAX_NR_GENS.

As an intermediate step to the final optimization, this change by itself
should not have userspace-visiable effects beyond performance.

Link: https://lkml.kernel.org/r/20241231043538.4075764-6-yuzhao@google.com
Signed-off-by: Yu Zhao <yuzhao@google.com>
Reported-by: Kairui Song <kasong@tencent.com>
Closes: https://lore.kernel.org/CAOUHufahuWcKf5f1Sg3emnqX+cODuR=2TQo7T4Gr-QYLujn4RA@mail.gmail.com/
Tested-by: Kalesh Singh <kaleshsingh@google.com>
Cc: Barry Song <v-songbaohua@oppo.com>
Cc: Bharata B Rao <bharata@amd.com>
Cc: David Stevens <stevensd@chromium.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/workingset.c b/mm/workingset.c
index ad181d1b8cf153f436828bb99c97811a2da9e464..2c310c29f51e6aacb20d888e12d0bebfd7ba1e5b 100644 (file)
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -260,11 +260,11 @@ static void *lru_gen_eviction(struct folio *folio)
  * Tests if the shadow entry is for a folio that was recently evicted.
  * Fills in @lruvec, @token, @workingset with the values unpacked from shadow.
  */
-static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec,
+static bool lru_gen_test_recent(void *shadow, struct lruvec **lruvec,
                                unsigned long *token, bool *workingset)
 {
        int memcg_id;
-       unsigned long min_seq;
+       unsigned long max_seq;
        struct mem_cgroup *memcg;
        struct pglist_data *pgdat;
 
@@ -273,8 +273,10 @@ static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec,
        memcg = mem_cgroup_from_id(memcg_id);
        *lruvec = mem_cgroup_lruvec(memcg, pgdat);
 
-       min_seq = READ_ONCE((*lruvec)->lrugen.min_seq[file]);
-       return (*token >> LRU_REFS_WIDTH) == (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH));
+       max_seq = READ_ONCE((*lruvec)->lrugen.max_seq);
+       max_seq &= EVICTION_MASK >> LRU_REFS_WIDTH;
+
+       return abs_diff(max_seq, *token >> LRU_REFS_WIDTH) < MAX_NR_GENS;
 }
 
 static void lru_gen_refault(struct folio *folio, void *shadow)
@@ -290,7 +292,7 @@ static void lru_gen_refault(struct folio *folio, void *shadow)
 
        rcu_read_lock();
 
-       recent = lru_gen_test_recent(shadow, type, &lruvec, &token, &workingset);
+       recent = lru_gen_test_recent(shadow, &lruvec, &token, &workingset);
        if (lruvec != folio_lruvec(folio))
                goto unlock;
 
@@ -331,7 +333,7 @@ static void *lru_gen_eviction(struct folio *folio)
        return NULL;
 }
 
-static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec,
+static bool lru_gen_test_recent(void *shadow, struct lruvec **lruvec,
                                unsigned long *token, bool *workingset)
 {
        return false;
@@ -432,8 +434,7 @@ bool workingset_test_recent(void *shadow, bool file, bool *workingset,
                bool recent;
 
                rcu_read_lock();
-               recent = lru_gen_test_recent(shadow, file, &eviction_lruvec,
-                                            &eviction, workingset);
+               recent = lru_gen_test_recent(shadow, &eviction_lruvec, &eviction, workingset);
                rcu_read_unlock();
                return recent;
        }