slab: avoid running debug SLAB code with IRQs disabled for alloc_bulk

Move the call to cache_alloc_debugcheck_after() outside the IRQ disabled
section in kmem_cache_alloc_bulk().

When CONFIG_DEBUG_SLAB is disabled the compiler should remove this code.

Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Vladimir Davydov <vdavydov@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/slab.c b/mm/slab.c
index 0c2e2cafbc83cd87553feb1f162e69c0372b3c17..207fa92502defbfd2b0bf063b715f9d27e3b0b23 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3391,6 +3391,16 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
 }
 EXPORT_SYMBOL(kmem_cache_free_bulk);
 
+static __always_inline void
+cache_alloc_debugcheck_after_bulk(struct kmem_cache *s, gfp_t flags,
+                                 size_t size, void **p, unsigned long caller)
+{
+       size_t i;
+
+       for (i = 0; i < size; i++)
+               p[i] = cache_alloc_debugcheck_after(s, flags, p[i], caller);
+}
+
 int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
                          void **p)
 {
@@ -3406,15 +3416,14 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
        for (i = 0; i < size; i++) {
                void *objp = __do_cache_alloc(s, flags);
 
-               /* this call could be done outside IRQ disabled section */
-               objp = cache_alloc_debugcheck_after(s, flags, objp, _RET_IP_);
-
                if (unlikely(!objp))
                        goto error;
                p[i] = objp;
        }
        local_irq_enable();
 
+       cache_alloc_debugcheck_after_bulk(s, flags, size, p, _RET_IP_);
+
        /* Clear memory outside IRQ disabled section */
        if (unlikely(flags & __GFP_ZERO))
                for (i = 0; i < size; i++)
@@ -3425,6 +3434,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
        return size;
 error:
        local_irq_enable();
+       cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_);
        slab_post_alloc_hook(s, flags, i, p);
        __kmem_cache_free_bulk(s, i, p);
        return 0;