mm, slub: validate slab from partial list or page allocator before making it cpu...

When we obtain a new slab page from node partial list or page allocator,
we assign it to kmem_cache_cpu, perform some checks, and if they fail, we
undo the assignment.

In order to allow doing the checks without irq disabled, restructure the
code so that the checks are done first, and kmem_cache_cpu.page assignment
only after they pass.

Link: https://lkml.kernel.org/r/20210805152000.12817-17-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au>

diff --git a/mm/slub.c b/mm/slub.c
index 8b240eb10a1d242bb72c63d326735f1cf389b415..fddfb0629b8ee1a4f992e5816d832e8de56eb076 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2788,10 +2788,8 @@ new_objects:
        lockdep_assert_irqs_disabled();
 
        freelist = get_partial(s, gfpflags, node, &page);
-       if (freelist) {
-               c->page = page;
+       if (freelist)
                goto check_new_page;
-       }
 
        local_irq_restore(flags);
        put_cpu_ptr(s->cpu_slab);
@@ -2804,9 +2802,6 @@ new_objects:
        }
 
        local_irq_save(flags);
-       if (c->page)
-               flush_slab(s, c);
-
        /*
         * No other reference to the page yet so we can
         * muck around with it freely without cmpxchg
@@ -2815,14 +2810,12 @@ new_objects:
        page->freelist = NULL;
 
        stat(s, ALLOC_SLAB);
-       c->page = page;
 
 check_new_page:
 
        if (kmem_cache_debug(s)) {
                if (!alloc_debug_processing(s, page, freelist, addr)) {
                        /* Slab failed checks. Next slab needed */
-                       c->page = NULL;
                        local_irq_restore(flags);
                        goto new_slab;
                } else {
@@ -2841,10 +2834,18 @@ check_new_page:
                 */
                goto return_single;
 
+       if (unlikely(c->page))
+               flush_slab(s, c);
+       c->page = page;
+
        goto load_freelist;
 
 return_single:
 
+       if (unlikely(c->page))
+               flush_slab(s, c);
+       c->page = page;
+
        deactivate_slab(s, page, get_freepointer(s, freelist), c);
        local_irq_restore(flags);
        return freelist;