slab: switch percpu sheaves locking to localtry_lock

Instead of local_lock_irqsave(), use localtry_trylock() when potential
callers include irq context, and localtry_lock() otherwise (such as when
we already know the gfp flags allow blocking).

This should reduce the locking (due to irq disabling/enabling) overhead.
Failing to use percpu sheaves in an irq due to preempting an already
locked user of sheaves should be rare so it's a favorable tradeoff.

Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>

diff --git a/mm/slub.c b/mm/slub.c
index db8e71ce2bf554ab308f5b0f5875130b1cc9a982..7c15d790f17a5ee9f59208aaf1815ad48f4acf42 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -450,7 +450,7 @@ struct slab_sheaf {
 };
 
 struct slub_percpu_sheaves {
-       local_lock_t lock;
+       localtry_lock_t lock;
        struct slab_sheaf *main; /* never NULL when unlocked */
        struct slab_sheaf *spare; /* empty or full, may be NULL */
        struct slab_sheaf *rcu_free;
@@ -2542,16 +2542,19 @@ static struct slab_sheaf *alloc_full_sheaf(struct kmem_cache *s, gfp_t gfp)
 
 static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
 
-static void sheaf_flush_main(struct kmem_cache *s)
+/* returns true if at least partially flushed */
+static bool sheaf_flush_main(struct kmem_cache *s)
 {
        struct slub_percpu_sheaves *pcs;
        unsigned int batch, remaining;
        void *objects[PCS_BATCH_MAX];
        struct slab_sheaf *sheaf;
-       unsigned long flags;
+       bool ret = false;
 
 next_batch:
-       local_lock_irqsave(&s->cpu_sheaves->lock, flags);
+       if (!localtry_trylock(&s->cpu_sheaves->lock))
+               return ret;
+
        pcs = this_cpu_ptr(s->cpu_sheaves);
        sheaf = pcs->main;
 
@@ -2562,14 +2565,18 @@ next_batch:
 
        remaining = sheaf->size;
 
-       local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+       localtry_unlock(&s->cpu_sheaves->lock);
 
        __kmem_cache_free_bulk(s, batch, &objects[0]);
 
        stat_add(s, SHEAF_FLUSH_MAIN, batch);
 
+       ret = true;
+
        if (remaining)
                goto next_batch;
+
+       return ret;
 }
 
 static void sheaf_flush(struct kmem_cache *s, struct slab_sheaf *sheaf)
@@ -2606,6 +2613,8 @@ static void rcu_free_sheaf_nobarn(struct rcu_head *head)
  * Caller needs to make sure migration is disabled in order to fully flush
  * single cpu's sheaves
  *
+ * must not be called from an irq
+ *
  * flushing operations are rare so let's keep it simple and flush to slabs
  * directly, skipping the barn
  */
@@ -2613,9 +2622,8 @@ static void pcs_flush_all(struct kmem_cache *s)
 {
        struct slub_percpu_sheaves *pcs;
        struct slab_sheaf *spare, *rcu_free;
-       unsigned long flags;
 
-       local_lock_irqsave(&s->cpu_sheaves->lock, flags);
+       localtry_lock(&s->cpu_sheaves->lock);
        pcs = this_cpu_ptr(s->cpu_sheaves);
 
        spare = pcs->spare;
@@ -2624,7 +2632,7 @@ static void pcs_flush_all(struct kmem_cache *s)
        rcu_free = pcs->rcu_free;
        pcs->rcu_free = NULL;
 
-       local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+       localtry_unlock(&s->cpu_sheaves->lock);
 
        if (spare) {
                sheaf_flush(s, spare);
@@ -4567,10 +4575,11 @@ static __fastpath_inline
 void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp)
 {
        struct slub_percpu_sheaves *pcs;
-       unsigned long flags;
        void *object;
 
-       local_lock_irqsave(&s->cpu_sheaves->lock, flags);
+       if (!localtry_trylock(&s->cpu_sheaves->lock))
+               return NULL;
+
        pcs = this_cpu_ptr(s->cpu_sheaves);
 
        if (unlikely(pcs->main->size == 0)) {
@@ -4603,7 +4612,7 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp)
                        }
                }
 
-               local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+               localtry_unlock(&s->cpu_sheaves->lock);
 
                if (!can_alloc)
                        return NULL;
@@ -4625,7 +4634,11 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp)
                if (!full)
                        return NULL;
 
-               local_lock_irqsave(&s->cpu_sheaves->lock, flags);
+               /*
+                * we can reach here only when gfpflags_allow_blocking
+                * so this must not be an irq
+                */
+               localtry_lock(&s->cpu_sheaves->lock);
                pcs = this_cpu_ptr(s->cpu_sheaves);
 
                /*
@@ -4659,7 +4672,7 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp)
 do_alloc:
        object = pcs->main->objects[--pcs->main->size];
 
-       local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+       localtry_unlock(&s->cpu_sheaves->lock);
 
        stat(s, ALLOC_PCS);
 
@@ -4671,12 +4684,13 @@ unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
 {
        struct slub_percpu_sheaves *pcs;
        struct slab_sheaf *main;
-       unsigned long flags;
        unsigned int allocated = 0;
        unsigned int batch;
 
 next_batch:
-       local_lock_irqsave(&s->cpu_sheaves->lock, flags);
+       if (!localtry_trylock(&s->cpu_sheaves->lock))
+               return allocated;
+
        pcs = this_cpu_ptr(s->cpu_sheaves);
 
        if (unlikely(pcs->main->size == 0)) {
@@ -4696,7 +4710,7 @@ next_batch:
                        goto do_alloc;
                }
 
-               local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+               localtry_unlock(&s->cpu_sheaves->lock);
 
                /*
                 * Once full sheaves in barn are depleted, let the bulk
@@ -4714,7 +4728,7 @@ do_alloc:
        main->size -= batch;
        memcpy(p, main->objects + main->size, batch * sizeof(void *));
 
-       local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+       localtry_unlock(&s->cpu_sheaves->lock);
 
        stat_add(s, ALLOC_PCS, batch);
 
@@ -5134,13 +5148,14 @@ slab_empty:
  * The object is expected to have passed slab_free_hook() already.
  */
 static __fastpath_inline
-void free_to_pcs(struct kmem_cache *s, void *object)
+bool free_to_pcs(struct kmem_cache *s, void *object)
 {
        struct slub_percpu_sheaves *pcs;
-       unsigned long flags;
 
 restart:
-       local_lock_irqsave(&s->cpu_sheaves->lock, flags);
+       if (!localtry_trylock(&s->cpu_sheaves->lock))
+               return false;
+
        pcs = this_cpu_ptr(s->cpu_sheaves);
 
        if (unlikely(pcs->main->size == s->sheaf_capacity)) {
@@ -5175,7 +5190,7 @@ restart:
                        struct slab_sheaf *to_flush = pcs->spare;
 
                        pcs->spare = NULL;
-                       local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+                       localtry_unlock(&s->cpu_sheaves->lock);
 
                        sheaf_flush(s, to_flush);
                        empty = to_flush;
@@ -5183,17 +5198,27 @@ restart:
                }
 
 alloc_empty:
-               local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+               localtry_unlock(&s->cpu_sheaves->lock);
 
                empty = alloc_empty_sheaf(s, GFP_NOWAIT);
 
                if (!empty) {
-                       sheaf_flush_main(s);
-                       goto restart;
+                       if (sheaf_flush_main(s))
+                               goto restart;
+                       else
+                               return false;
                }
 
 got_empty:
-               local_lock_irqsave(&s->cpu_sheaves->lock, flags);
+               if (!localtry_trylock(&s->cpu_sheaves->lock)) {
+                       struct node_barn *barn;
+
+                       barn = get_node(s, numa_mem_id())->barn;
+
+                       barn_put_empty_sheaf(barn, empty, true);
+                       return false;
+               }
+
                pcs = this_cpu_ptr(s->cpu_sheaves);
 
                /*
@@ -5222,9 +5247,11 @@ got_empty:
 do_free:
        pcs->main->objects[pcs->main->size++] = object;
 
-       local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+       localtry_unlock(&s->cpu_sheaves->lock);
 
        stat(s, FREE_PCS);
+
+       return true;
 }
 
 static void __rcu_free_sheaf_prepare(struct kmem_cache *s,
@@ -5283,9 +5310,10 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj)
 {
        struct slub_percpu_sheaves *pcs;
        struct slab_sheaf *rcu_sheaf;
-       unsigned long flags;
 
-       local_lock_irqsave(&s->cpu_sheaves->lock, flags);
+       if (!localtry_trylock(&s->cpu_sheaves->lock))
+               goto fail;
+
        pcs = this_cpu_ptr(s->cpu_sheaves);
 
        if (unlikely(!pcs->rcu_free)) {
@@ -5299,16 +5327,16 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj)
                        goto do_free;
                }
 
-               local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+               localtry_unlock(&s->cpu_sheaves->lock);
 
                empty = alloc_empty_sheaf(s, GFP_NOWAIT);
 
-               if (!empty) {
-                       stat(s, FREE_RCU_SHEAF_FAIL);
-                       return false;
-               }
+               if (!empty)
+                       goto fail;
+
+               if (!localtry_trylock(&s->cpu_sheaves->lock))
+                       goto fail;
 
-               local_lock_irqsave(&s->cpu_sheaves->lock, flags);
                pcs = this_cpu_ptr(s->cpu_sheaves);
 
                if (unlikely(pcs->rcu_free))
@@ -5324,19 +5352,22 @@ do_free:
        rcu_sheaf->objects[rcu_sheaf->size++] = obj;
 
        if (likely(rcu_sheaf->size < s->sheaf_capacity)) {
-               local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+               localtry_unlock(&s->cpu_sheaves->lock);
                stat(s, FREE_RCU_SHEAF);
                return true;
        }
 
        pcs->rcu_free = NULL;
-       local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+       localtry_unlock(&s->cpu_sheaves->lock);
 
        call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
 
        stat(s, FREE_RCU_SHEAF);
-
        return true;
+
+fail:
+       stat(s, FREE_RCU_SHEAF_FAIL);
+       return false;
 }
 
 /*
@@ -5348,7 +5379,6 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
 {
        struct slub_percpu_sheaves *pcs;
        struct slab_sheaf *main;
-       unsigned long flags;
        unsigned int batch, i = 0;
        bool init;
 
@@ -5371,7 +5401,9 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
        }
 
 next_batch:
-       local_lock_irqsave(&s->cpu_sheaves->lock, flags);
+       if (!localtry_trylock(&s->cpu_sheaves->lock))
+               goto fallback;
+
        pcs = this_cpu_ptr(s->cpu_sheaves);
 
        if (unlikely(pcs->main->size == s->sheaf_capacity)) {
@@ -5402,13 +5434,13 @@ next_batch:
                }
 
 no_empty:
-               local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+               localtry_unlock(&s->cpu_sheaves->lock);
 
                /*
                 * if we depleted all empty sheaves in the barn or there are too
                 * many full sheaves, free the rest to slab pages
                 */
-
+fallback:
                __kmem_cache_free_bulk(s, size, p);
                return;
        }
@@ -5420,7 +5452,7 @@ do_free:
        memcpy(main->objects + main->size, p, batch * sizeof(void *));
        main->size += batch;
 
-       local_unlock_irqrestore(&s->cpu_sheaves->lock, flags);
+       localtry_unlock(&s->cpu_sheaves->lock);
 
        stat_add(s, FREE_PCS, batch);
 
@@ -5520,9 +5552,7 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
        if (unlikely(!slab_free_hook(s, object, slab_want_init_on_free(s), false)))
                return;
 
-       if (s->cpu_sheaves)
-               free_to_pcs(s, object);
-       else
+       if (!s->cpu_sheaves || !free_to_pcs(s, object))
                do_slab_free(s, slab, object, object, 1, addr);
 }
 
@@ -6301,7 +6331,7 @@ static int init_percpu_sheaves(struct kmem_cache *s)
 
                pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
 
-               local_lock_init(&pcs->lock);
+               localtry_lock_init(&pcs->lock);
 
                nid = cpu_to_mem(cpu);