mm, slub: use migrate_disable() on PREEMPT_RT

We currently use preempt_disable() (directly or via get_cpu_ptr()) to stabilize
the pointer to kmem_cache_cpu. On PREEMPT_RT this would be incompatible with
the list_lock spinlock. We can use migrate_disable() instead, but that
increases overhead on !PREEMPT_RT as it's an unconditional function call.

In order to get the best available mechanism on both PREEMPT_RT and
!PREEMPT_RT, introduce private slub_get_cpu_ptr() and slub_put_cpu_ptr()
wrappers and use them.

Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/mm/slub.c b/mm/slub.c
index f4b33d2fddc14b573f1629de03a629e4bff6a26b..38d4cc51e880c5a19688560dcc5daa9bb47891b1 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -118,6 +118,26 @@
  *                     the fast path and disables lockless freelists.
  */
 
+/*
+ * We could simply use migrate_disable()/enable() but as long as it's a
+ * function call even on !PREEMPT_RT, use inline preempt_disable() there.
+ */
+#ifndef CONFIG_PREEMPT_RT
+#define slub_get_cpu_ptr(var)  get_cpu_ptr(var)
+#define slub_put_cpu_ptr(var)  put_cpu_ptr(var)
+#else
+#define slub_get_cpu_ptr(var)          \
+({                                     \
+       migrate_disable();              \
+       this_cpu_ptr(var);              \
+})
+#define slub_put_cpu_ptr(var)          \
+do {                                   \
+       (void)(var);                    \
+       migrate_enable();               \
+} while (0)
+#endif
+
 #ifdef CONFIG_SLUB_DEBUG
 #ifdef CONFIG_SLUB_DEBUG_ON
 DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
@@ -2852,7 +2872,7 @@ redo:
        if (unlikely(!pfmemalloc_match_unsafe(page, gfpflags)))
                goto deactivate_slab;
 
-       /* must check again c->page in case IRQ handler changed it */
+       /* must check again c->page in case we got preempted and it changed */
        local_irq_save(flags);
        if (unlikely(page != c->page)) {
                local_irq_restore(flags);
@@ -2911,7 +2931,8 @@ new_slab:
                }
                if (unlikely(!slub_percpu_partial(c))) {
                        local_irq_restore(flags);
-                       goto new_objects; /* stolen by an IRQ handler */
+                       /* we were preempted and partial list got empty */
+                       goto new_objects;
                }
 
                page = c->page = slub_percpu_partial(c);
@@ -2927,9 +2948,9 @@ new_objects:
        if (freelist)
                goto check_new_page;
 
-       put_cpu_ptr(s->cpu_slab);
+       slub_put_cpu_ptr(s->cpu_slab);
        page = new_slab(s, gfpflags, node);
-       c = get_cpu_ptr(s->cpu_slab);
+       c = slub_get_cpu_ptr(s->cpu_slab);
 
        if (unlikely(!page)) {
                slab_out_of_memory(s, gfpflags, node);
@@ -3012,12 +3033,12 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
         * cpu before disabling preemption. Need to reload cpu area
         * pointer.
         */
-       c = get_cpu_ptr(s->cpu_slab);
+       c = slub_get_cpu_ptr(s->cpu_slab);
 #endif
 
        p = ___slab_alloc(s, gfpflags, node, addr, c);
 #ifdef CONFIG_PREEMPT_COUNT
-       put_cpu_ptr(s->cpu_slab);
+       slub_put_cpu_ptr(s->cpu_slab);
 #endif
        return p;
 }
@@ -3546,7 +3567,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
         * IRQs, which protects against PREEMPT and interrupts
         * handlers invoking normal fastpath.
         */
-       c = get_cpu_ptr(s->cpu_slab);
+       c = slub_get_cpu_ptr(s->cpu_slab);
        local_irq_disable();
 
        for (i = 0; i < size; i++) {
@@ -3592,7 +3613,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
        }
        c->tid = next_tid(c->tid);
        local_irq_enable();
-       put_cpu_ptr(s->cpu_slab);
+       slub_put_cpu_ptr(s->cpu_slab);
 
        /*
         * memcg and kmem_cache debug support and memory initialization.
@@ -3602,7 +3623,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
                                slab_want_init_on_alloc(flags, s));
        return i;
 error:
-       put_cpu_ptr(s->cpu_slab);
+       slub_put_cpu_ptr(s->cpu_slab);
        slab_post_alloc_hook(s, objcg, flags, i, p, false);
        __kmem_cache_free_bulk(s, i, p);
        return 0;