slub: relax CMPXCHG consistency restrictions

When debug options are enabled, cmpxchg on the page is disabled.  This
is because the page must be locked to ensure there are no false
positives when performing consistency checks.  Some debug options such
as poisoning and red zoning only act on the object itself.  There is no
need to protect other CPUs from modification on only the object.  Allow
cmpxchg to happen with poisoning and red zoning are set on a slab.

Credit to Mathias Krause for the original work which inspired this
series

Signed-off-by: Laura Abbott <labbott@fedoraproject.org>
Acked-by: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Mathias Krause <minipli@googlemail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/slub.c b/mm/slub.c
index 9cde663bbb1096fe544fb710ed3287dc33666e68..c2a227d8a4ee32e00a23d43091f8394dec98382c 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -163,6 +163,14 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
 #define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
                                SLAB_POISON | SLAB_STORE_USER)
 
+/*
+ * These debug flags cannot use CMPXCHG because there might be consistency
+ * issues when checking or reading debug information
+ */
+#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
+                               SLAB_TRACE)
+
+
 /*
  * Debugging flags that require metadata to be stored in the slab.  These get
  * disabled when slub_debug=O is used and a cache's min order increases with
@@ -3338,7 +3346,7 @@ static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
 
 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
     defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
-       if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
+       if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
                /* Enable fast mode */
                s->flags |= __CMPXCHG_DOUBLE;
 #endif
@@ -4846,7 +4854,6 @@ static ssize_t red_zone_store(struct kmem_cache *s,
 
        s->flags &= ~SLAB_RED_ZONE;
        if (buf[0] == '1') {
-               s->flags &= ~__CMPXCHG_DOUBLE;
                s->flags |= SLAB_RED_ZONE;
        }
        calculate_sizes(s, -1);
@@ -4867,7 +4874,6 @@ static ssize_t poison_store(struct kmem_cache *s,
 
        s->flags &= ~SLAB_POISON;
        if (buf[0] == '1') {
-               s->flags &= ~__CMPXCHG_DOUBLE;
                s->flags |= SLAB_POISON;
        }
        calculate_sizes(s, -1);