mm/list_lru: don't pass unnecessary key parameters

Patch series "Split list_lru lock into per-cgroup scope", v2.

Currently, every list_lru has a per-node lock that protects adding,
deletion, isolation, and reparenting of all list_lru_one instances
belonging to this list_lru on this node.  This lock contention is heavy
when multiple cgroups modify the same list_lru.

This can be alleviated by splitting the lock into per-cgroup scope.

To achieve this, this series reworked and optimized the reparenting
process step by step, making it possible to have a stable list_lru_one,
and making it possible to pin the list_lru_one.  Then split the lock into
per-cgroup scope.

The result is ~15% performance gain for simple multi-cgroup tar test of
small files, and reduced LOC.  See PATCH 5/6 for test details.

This patch (of 6):

When LOCKDEP is not enabled, lock_class_key is an empty struct that is
never used.  But the list_lru initialization function still takes a
placeholder pointer as parameter, and the compiler cannot optimize it
because the function is not static and exported.

Remove this parameter and move it inside the list_lru struct.  Only use it
when LOCKDEP is enabled.  Kernel builds with LOCKDEP will be slightly
larger, while !LOCKDEP builds without it will be slightly smaller (the
common case).

Link: https://lkml.kernel.org/r/20240925171020.32142-1-ryncsn@gmail.com
Link: https://lkml.kernel.org/r/20240925171020.32142-2-ryncsn@gmail.com
Signed-off-by: Kairui Song <kasong@tencent.com>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Chengming Zhou <zhouchengming@bytedance.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Waiman Long <longman@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h
index 5099a8ccd5f4c70ae85bc146b44bfd2e9911bd08..eba93f6511f316ded625ab6d6e441a08f32705d9 100644 (file)
--- a/include/linux/list_lru.h
+++ b/include/linux/list_lru.h
@@ -56,16 +56,28 @@ struct list_lru {
        bool                    memcg_aware;
        struct xarray           xa;
 #endif
+#ifdef CONFIG_LOCKDEP
+       struct lock_class_key   *key;
+#endif
 };
 
 void list_lru_destroy(struct list_lru *lru);
 int __list_lru_init(struct list_lru *lru, bool memcg_aware,
-                   struct lock_class_key *key, struct shrinker *shrinker);
+                   struct shrinker *shrinker);
 
 #define list_lru_init(lru)                             \
-       __list_lru_init((lru), false, NULL, NULL)
+       __list_lru_init((lru), false, NULL)
 #define list_lru_init_memcg(lru, shrinker)             \
-       __list_lru_init((lru), true, NULL, shrinker)
+       __list_lru_init((lru), true, shrinker)
+
+static inline int list_lru_init_memcg_key(struct list_lru *lru, struct shrinker *shrinker,
+                                         struct lock_class_key *key)
+{
+#ifdef CONFIG_LOCKDEP
+       lru->key = key;
+#endif
+       return list_lru_init_memcg(lru, shrinker);
+}
 
 int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru,
                         gfp_t gfp);

diff --git a/mm/list_lru.c b/mm/list_lru.c
index 9b7ff06e9d326bb0da9f486da6c05a56c77d786c..ea7dc9fa4d05a24c59b555d2911cc586d04b01df 100644 (file)
--- a/mm/list_lru.c
+++ b/mm/list_lru.c
@@ -562,8 +562,7 @@ static void memcg_destroy_list_lru(struct list_lru *lru)
 }
 #endif /* CONFIG_MEMCG */
 
-int __list_lru_init(struct list_lru *lru, bool memcg_aware,
-                   struct lock_class_key *key, struct shrinker *shrinker)
+int __list_lru_init(struct list_lru *lru, bool memcg_aware, struct shrinker *shrinker)
 {
        int i;
 
@@ -583,8 +582,10 @@ int __list_lru_init(struct list_lru *lru, bool memcg_aware,
 
        for_each_node(i) {
                spin_lock_init(&lru->node[i].lock);
-               if (key)
-                       lockdep_set_class(&lru->node[i].lock, key);
+#ifdef CONFIG_LOCKDEP
+               if (lru->key)
+                       lockdep_set_class(&lru->node[i].lock, lru->key);
+#endif
                init_one_lru(&lru->node[i].lru);
        }
 

diff --git a/mm/workingset.c b/mm/workingset.c
index a2b28e356e68ed4b525d052acac0999afbb10cbd..df3937c5eedc1da6a0ffa8ab138914d80437b168 100644 (file)
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -823,8 +823,8 @@ static int __init workingset_init(void)
        if (!workingset_shadow_shrinker)
                goto err;
 
-       ret = __list_lru_init(&shadow_nodes, true, &shadow_nodes_key,
-                             workingset_shadow_shrinker);
+       ret = list_lru_init_memcg_key(&shadow_nodes, workingset_shadow_shrinker,
+                                     &shadow_nodes_key);
        if (ret)
                goto err_list_lru;