mm: vmscan: split proactive reclaim statistics from direct reclaim statistics

Patch series "Adding Proactive Memory Reclaim Statistics".

These two patches are related to proactive memory reclaim.

Patch 1 Split proactive reclaim statistics from direct reclaim counters
and introduces new counters: pgsteal_proactive, pgdemote_proactive,
and pgscan_proactive.

Patch 2 Adds pswpin and pswpout items to the cgroup-v2 documentation.

This patch (of 2):

In proactive memory reclaim scenarios, it is necessary to accurately track
proactive reclaim statistics to dynamically adjust the frequency and
amount of memory being reclaimed proactively.  Currently, proactive
reclaim is included in direct reclaim statistics, which can make these
direct reclaim statistics misleading.

Therefore, separate proactive reclaim memory from the direct reclaim
counters by introducing new counters: pgsteal_proactive,
pgdemote_proactive, and pgscan_proactive, to avoid confusion with direct
reclaim.

Link: https://lkml.kernel.org/r/20250318075833.90615-1-jiahao.kernel@gmail.com
Link: https://lkml.kernel.org/r/20250318075833.90615-2-jiahao.kernel@gmail.com
Signed-off-by: Hao Jia <jiahao1@lixiang.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Michal Koutný <mkoutny@suse.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index f8a894a163072326951ef2a3bbe37a89c1401d78..d7624e5006107cb3d1a9dcd764b1ac7300ff7098 100644 (file)
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -1576,6 +1576,9 @@ The following nested keys are defined.
          pgscan_khugepaged (npn)
                Amount of scanned pages by khugepaged  (in an inactive LRU list)
 
+         pgscan_proactive (npn)
+               Amount of scanned pages proactively (in an inactive LRU list)
+
          pgsteal_kswapd (npn)
                Amount of reclaimed pages by kswapd
 
@@ -1585,6 +1588,9 @@ The following nested keys are defined.
          pgsteal_khugepaged (npn)
                Amount of reclaimed pages by khugepaged
 
+         pgsteal_proactive (npn)
+               Amount of reclaimed pages proactively
+
          pgfault (npn)
                Total number of page faults incurred
 
@@ -1662,6 +1668,9 @@ The following nested keys are defined.
          pgdemote_khugepaged
                Number of pages demoted by khugepaged.
 
+         pgdemote_proactive
+               Number of pages demoted by proactively.
+
          hugetlb
                Amount of memory used by hugetlb pages. This metric only shows
                up if hugetlb usage is accounted for in memory.current (i.e.

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index a9db0fbd2b941a527eb977e9e42b827194166261..f7fe1126dc75608b429ef03aca8418d147e0efa6 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -221,6 +221,7 @@ enum node_stat_item {
        PGDEMOTE_KSWAPD,
        PGDEMOTE_DIRECT,
        PGDEMOTE_KHUGEPAGED,
+       PGDEMOTE_PROACTIVE,
 #ifdef CONFIG_HUGETLB_PAGE
        NR_HUGETLB,
 #endif

diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index f70d0958095cd95b729f4835ee768e690b93e042..f11b6fa9c5b34875ce57d44e778c071a6813a0a8 100644 (file)
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -41,9 +41,11 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
                PGSTEAL_KSWAPD,
                PGSTEAL_DIRECT,
                PGSTEAL_KHUGEPAGED,
+               PGSTEAL_PROACTIVE,
                PGSCAN_KSWAPD,
                PGSCAN_DIRECT,
                PGSCAN_KHUGEPAGED,
+               PGSCAN_PROACTIVE,
                PGSCAN_DIRECT_THROTTLE,
                PGSCAN_ANON,
                PGSCAN_FILE,

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 57cf5a6c279c5e04ee2b9d58dd78b940d9d0ff2c..40c07b8699ae3a541b6f4928065575ee88d420e5 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -315,6 +315,7 @@ static const unsigned int memcg_node_stat_items[] = {
        PGDEMOTE_KSWAPD,
        PGDEMOTE_DIRECT,
        PGDEMOTE_KHUGEPAGED,
+       PGDEMOTE_PROACTIVE,
 #ifdef CONFIG_HUGETLB_PAGE
        NR_HUGETLB,
 #endif
@@ -431,9 +432,11 @@ static const unsigned int memcg_vm_event_stat[] = {
        PGSCAN_KSWAPD,
        PGSCAN_DIRECT,
        PGSCAN_KHUGEPAGED,
+       PGSCAN_PROACTIVE,
        PGSTEAL_KSWAPD,
        PGSTEAL_DIRECT,
        PGSTEAL_KHUGEPAGED,
+       PGSTEAL_PROACTIVE,
        PGFAULT,
        PGMAJFAULT,
        PGREFILL,
@@ -1394,6 +1397,7 @@ static const struct memory_stat memory_stats[] = {
        { "pgdemote_kswapd",            PGDEMOTE_KSWAPD         },
        { "pgdemote_direct",            PGDEMOTE_DIRECT         },
        { "pgdemote_khugepaged",        PGDEMOTE_KHUGEPAGED     },
+       { "pgdemote_proactive",         PGDEMOTE_PROACTIVE      },
 #ifdef CONFIG_NUMA_BALANCING
        { "pgpromote_success",          PGPROMOTE_SUCCESS       },
 #endif
@@ -1436,6 +1440,7 @@ static int memcg_page_state_output_unit(int item)
        case PGDEMOTE_KSWAPD:
        case PGDEMOTE_DIRECT:
        case PGDEMOTE_KHUGEPAGED:
+       case PGDEMOTE_PROACTIVE:
 #ifdef CONFIG_NUMA_BALANCING
        case PGPROMOTE_SUCCESS:
 #endif
@@ -1509,10 +1514,12 @@ static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
        seq_buf_printf(s, "pgscan %lu\n",
                       memcg_events(memcg, PGSCAN_KSWAPD) +
                       memcg_events(memcg, PGSCAN_DIRECT) +
+                      memcg_events(memcg, PGSCAN_PROACTIVE) +
                       memcg_events(memcg, PGSCAN_KHUGEPAGED));
        seq_buf_printf(s, "pgsteal %lu\n",
                       memcg_events(memcg, PGSTEAL_KSWAPD) +
                       memcg_events(memcg, PGSTEAL_DIRECT) +
+                      memcg_events(memcg, PGSTEAL_PROACTIVE) +
                       memcg_events(memcg, PGSTEAL_KHUGEPAGED));
 
        for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) {

diff --git a/mm/vmscan.c b/mm/vmscan.c
index b5c7dfc2b18999c8a0b02c0811af3089d54e79e8..98e6ac82e428615536fb13b179bc055f0699a121 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -456,21 +456,26 @@ void drop_slab(void)
        } while ((freed >> shift++) > 1);
 }
 
-static int reclaimer_offset(void)
+#define CHECK_RECLAIMER_OFFSET(type)                                   \
+       do {                                                            \
+               BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD !=         \
+                            PGDEMOTE_##type - PGDEMOTE_KSWAPD);        \
+               BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD !=         \
+                            PGSCAN_##type - PGSCAN_KSWAPD);            \
+       } while (0)
+
+static int reclaimer_offset(struct scan_control *sc)
 {
-       BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=
-                       PGDEMOTE_DIRECT - PGDEMOTE_KSWAPD);
-       BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD !=
-                       PGDEMOTE_KHUGEPAGED - PGDEMOTE_KSWAPD);
-       BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=
-                       PGSCAN_DIRECT - PGSCAN_KSWAPD);
-       BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD !=
-                       PGSCAN_KHUGEPAGED - PGSCAN_KSWAPD);
+       CHECK_RECLAIMER_OFFSET(DIRECT);
+       CHECK_RECLAIMER_OFFSET(KHUGEPAGED);
+       CHECK_RECLAIMER_OFFSET(PROACTIVE);
 
        if (current_is_kswapd())
                return 0;
        if (current_is_khugepaged())
                return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD;
+       if (sc->proactive)
+               return PGSTEAL_PROACTIVE - PGSTEAL_KSWAPD;
        return PGSTEAL_DIRECT - PGSTEAL_KSWAPD;
 }
 
@@ -2008,7 +2013,7 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
                                     &nr_scanned, sc, lru);
 
        __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
-       item = PGSCAN_KSWAPD + reclaimer_offset();
+       item = PGSCAN_KSWAPD + reclaimer_offset(sc);
        if (!cgroup_reclaim(sc))
                __count_vm_events(item, nr_scanned);
        __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
@@ -2024,10 +2029,10 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
        spin_lock_irq(&lruvec->lru_lock);
        move_folios_to_lru(lruvec, &folio_list);
 
-       __mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(),
+       __mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
                                        stat.nr_demoted);
        __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
-       item = PGSTEAL_KSWAPD + reclaimer_offset();
+       item = PGSTEAL_KSWAPD + reclaimer_offset(sc);
        if (!cgroup_reclaim(sc))
                __count_vm_events(item, nr_reclaimed);
        __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
@@ -4571,7 +4576,7 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
                        break;
        }
 
-       item = PGSCAN_KSWAPD + reclaimer_offset();
+       item = PGSCAN_KSWAPD + reclaimer_offset(sc);
        if (!cgroup_reclaim(sc)) {
                __count_vm_events(item, isolated);
                __count_vm_events(PGREFILL, sorted);
@@ -4721,10 +4726,10 @@ retry:
                reset_batch_size(walk);
        }
 
-       __mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(),
+       __mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
                                        stat.nr_demoted);
 
-       item = PGSTEAL_KSWAPD + reclaimer_offset();
+       item = PGSTEAL_KSWAPD + reclaimer_offset(sc);
        if (!cgroup_reclaim(sc))
                __count_vm_events(item, reclaimed);
        __count_memcg_events(memcg, item, reclaimed);

diff --git a/mm/vmstat.c b/mm/vmstat.c
index ae0e4259ac236da9260f91ee455f0867ff929236..ab5c840941f3ad829d965163f58f6a331ca9cadf 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1274,6 +1274,7 @@ const char * const vmstat_text[] = {
        "pgdemote_kswapd",
        "pgdemote_direct",
        "pgdemote_khugepaged",
+       "pgdemote_proactive",
 #ifdef CONFIG_HUGETLB_PAGE
        "nr_hugetlb",
 #endif
@@ -1309,9 +1310,11 @@ const char * const vmstat_text[] = {
        "pgsteal_kswapd",
        "pgsteal_direct",
        "pgsteal_khugepaged",
+       "pgsteal_proactive",
        "pgscan_kswapd",
        "pgscan_direct",
        "pgscan_khugepaged",
+       "pgscan_proactive",
        "pgscan_direct_throttle",
        "pgscan_anon",
        "pgscan_file",