mm: memcg/slab: fix obj_cgroup_charge() return value handling

Commit 10befea91b61 ("mm: memcg/slab: use a single set of kmem_caches
for all allocations") introduced a regression into the handling of the
obj_cgroup_charge() return value.  If a non-zero value is returned
(indicating of exceeding one of memory.max limits), the allocation
should fail, instead of falling back to non-accounted mode.

To make the code more readable, move memcg_slab_pre_alloc_hook() and
memcg_slab_post_alloc_hook() calling conditions into bodies of these
hooks.

Fixes: 10befea91b61 ("mm: memcg/slab: use a single set of kmem_caches for all allocations")
Signed-off-by: Roman Gushchin <guro@fb.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Reviewed-by: Shakeel Butt <shakeelb@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: <stable@vger.kernel.org>
Link: https://lkml.kernel.org/r/20201127161828.GD840171@carbon.dhcp.thefacebook.com
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/slab.h b/mm/slab.h
index 6d7c6a5056baac906432f3b65daf6f7297aedc99..f9977d6613d6189a204daf040a40e126194fb727 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -274,22 +274,32 @@ static inline size_t obj_full_size(struct kmem_cache *s)
        return s->size + sizeof(struct obj_cgroup *);
 }
 
-static inline struct obj_cgroup *memcg_slab_pre_alloc_hook(struct kmem_cache *s,
-                                                          size_t objects,
-                                                          gfp_t flags)
+/*
+ * Returns false if the allocation should fail.
+ */
+static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
+                                            struct obj_cgroup **objcgp,
+                                            size_t objects, gfp_t flags)
 {
        struct obj_cgroup *objcg;
 
+       if (!memcg_kmem_enabled())
+               return true;
+
+       if (!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))
+               return true;
+
        objcg = get_obj_cgroup_from_current();
        if (!objcg)
-               return NULL;
+               return true;
 
        if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s))) {
                obj_cgroup_put(objcg);
-               return NULL;
+               return false;
        }
 
-       return objcg;
+       *objcgp = objcg;
+       return true;
 }
 
 static inline void mod_objcg_state(struct obj_cgroup *objcg,
@@ -315,7 +325,7 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
        unsigned long off;
        size_t i;
 
-       if (!objcg)
+       if (!memcg_kmem_enabled() || !objcg)
                return;
 
        flags &= ~__GFP_ACCOUNT;
@@ -400,11 +410,11 @@ static inline void memcg_free_page_obj_cgroups(struct page *page)
 {
 }
 
-static inline struct obj_cgroup *memcg_slab_pre_alloc_hook(struct kmem_cache *s,
-                                                          size_t objects,
-                                                          gfp_t flags)
+static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
+                                            struct obj_cgroup **objcgp,
+                                            size_t objects, gfp_t flags)
 {
-       return NULL;
+       return true;
 }
 
 static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
@@ -508,9 +518,8 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
        if (should_failslab(s, flags))
                return NULL;
 
-       if (memcg_kmem_enabled() &&
-           ((flags & __GFP_ACCOUNT) || (s->flags & SLAB_ACCOUNT)))
-               *objcgp = memcg_slab_pre_alloc_hook(s, size, flags);
+       if (!memcg_slab_pre_alloc_hook(s, objcgp, size, flags))
+               return NULL;
 
        return s;
 }
@@ -529,8 +538,7 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s,
                                         s->flags, flags);
        }
 
-       if (memcg_kmem_enabled())
-               memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
+       memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
 }
 
 #ifndef CONFIG_SLOB