mm: memcg: use READ_ONCE()/WRITE_ONCE() to access stock->cached

A memcg pointer in the percpu stock can be accessed by drain_all_stock()
from another cpu in a lockless way.  In theory it might lead to an issue,
similar to the one which has been discovered with stock->cached_objcg,
where the pointer was zeroed between the check for being NULL and
dereferencing.  In this case the issue is unlikely a real problem, but to
make it bulletproof and similar to stock->cached_objcg, let's annotate all
accesses to stock->cached with READ_ONCE()/WTRITE_ONCE().

Link: https://lkml.kernel.org/r/20230502160839.361544-2-roman.gushchin@linux.dev
Signed-off-by: Roman Gushchin <roman.gushchin@linux.dev>
Acked-by: Shakeel Butt <shakeelb@google.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index c823c35c2ed464e0e0ba78e68c62e363c66faa08..1e364ad495a3a325e2dcb13b2059089b376f5183 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2275,7 +2275,7 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
        local_lock_irqsave(&memcg_stock.stock_lock, flags);
 
        stock = this_cpu_ptr(&memcg_stock);
-       if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
+       if (memcg == READ_ONCE(stock->cached) && stock->nr_pages >= nr_pages) {
                stock->nr_pages -= nr_pages;
                ret = true;
        }
@@ -2290,7 +2290,7 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
  */
 static void drain_stock(struct memcg_stock_pcp *stock)
 {
-       struct mem_cgroup *old = stock->cached;
+       struct mem_cgroup *old = READ_ONCE(stock->cached);
 
        if (!old)
                return;
@@ -2303,7 +2303,7 @@ static void drain_stock(struct memcg_stock_pcp *stock)
        }
 
        css_put(&old->css);
-       stock->cached = NULL;
+       WRITE_ONCE(stock->cached, NULL);
 }
 
 static void drain_local_stock(struct work_struct *dummy)
@@ -2338,10 +2338,10 @@ static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
        struct memcg_stock_pcp *stock;
 
        stock = this_cpu_ptr(&memcg_stock);
-       if (stock->cached != memcg) { /* reset if necessary */
+       if (READ_ONCE(stock->cached) != memcg) { /* reset if necessary */
                drain_stock(stock);
                css_get(&memcg->css);
-               stock->cached = memcg;
+               WRITE_ONCE(stock->cached, memcg);
        }
        stock->nr_pages += nr_pages;
 
@@ -2383,7 +2383,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
                bool flush = false;
 
                rcu_read_lock();
-               memcg = stock->cached;
+               memcg = READ_ONCE(stock->cached);
                if (memcg && stock->nr_pages &&
                    mem_cgroup_is_descendant(memcg, root_memcg))
                        flush = true;