From ec8ebac28fc8177a0d43b172d44151843d169643 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Wed, 28 Aug 2024 11:28:19 +0200 Subject: [PATCH] slab: switch percpu sheaves locking to localtry_lock Instead of local_lock_irqsave(), use localtry_trylock() when potential callers include irq context, and localtry_lock() otherwise (such as when we already know the gfp flags allow blocking). This should reduce the locking (due to irq disabling/enabling) overhead. Failing to use percpu sheaves in an irq due to preempting an already locked user of sheaves should be rare so it's a favorable tradeoff. Signed-off-by: Vlastimil Babka --- mm/slub.c | 122 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 46 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 79bcc778ecc9a..76f9ff02ca55b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -450,7 +450,7 @@ struct slab_sheaf { }; struct slub_percpu_sheaves { - local_lock_t lock; + localtry_lock_t lock; struct slab_sheaf *main; /* never NULL when unlocked */ struct slab_sheaf *spare; /* empty or full, may be NULL */ struct slab_sheaf *rcu_free; @@ -2542,16 +2542,19 @@ static struct slab_sheaf *alloc_full_sheaf(struct kmem_cache *s, gfp_t gfp) static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); -static void sheaf_flush_main(struct kmem_cache *s) +/* returns true if at least partially flushed */ +static bool sheaf_flush_main(struct kmem_cache *s) { struct slub_percpu_sheaves *pcs; unsigned int batch, remaining; void *objects[PCS_BATCH_MAX]; struct slab_sheaf *sheaf; - unsigned long flags; + bool ret = false; next_batch: - local_lock_irqsave(&s->cpu_sheaves->lock, flags); + if (!localtry_trylock(&s->cpu_sheaves->lock)) + return ret; + pcs = this_cpu_ptr(s->cpu_sheaves); sheaf = pcs->main; @@ -2562,14 +2565,18 @@ next_batch: remaining = sheaf->size; - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); __kmem_cache_free_bulk(s, batch, &objects[0]); stat_add(s, SHEAF_FLUSH_MAIN, batch); + ret = true; + if (remaining) goto next_batch; + + return ret; } static void sheaf_flush(struct kmem_cache *s, struct slab_sheaf *sheaf) @@ -2606,6 +2613,8 @@ static void rcu_free_sheaf_nobarn(struct rcu_head *head) * Caller needs to make sure migration is disabled in order to fully flush * single cpu's sheaves * + * must not be called from an irq + * * flushing operations are rare so let's keep it simple and flush to slabs * directly, skipping the barn */ @@ -2613,9 +2622,8 @@ static void pcs_flush_all(struct kmem_cache *s) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *spare, *rcu_free; - unsigned long flags; - local_lock_irqsave(&s->cpu_sheaves->lock, flags); + localtry_lock(&s->cpu_sheaves->lock); pcs = this_cpu_ptr(s->cpu_sheaves); spare = pcs->spare; @@ -2624,7 +2632,7 @@ static void pcs_flush_all(struct kmem_cache *s) rcu_free = pcs->rcu_free; pcs->rcu_free = NULL; - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); if (spare) { sheaf_flush(s, spare); @@ -4567,10 +4575,11 @@ static __fastpath_inline void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp) { struct slub_percpu_sheaves *pcs; - unsigned long flags; void *object; - local_lock_irqsave(&s->cpu_sheaves->lock, flags); + if (!localtry_trylock(&s->cpu_sheaves->lock)) + return NULL; + pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(pcs->main->size == 0)) { @@ -4603,7 +4612,7 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp) } } - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); if (!can_alloc) return NULL; @@ -4625,7 +4634,11 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp) if (!full) return NULL; - local_lock_irqsave(&s->cpu_sheaves->lock, flags); + /* + * we can reach here only when gfpflags_allow_blocking + * so this must not be an irq + */ + localtry_lock(&s->cpu_sheaves->lock); pcs = this_cpu_ptr(s->cpu_sheaves); /* @@ -4659,7 +4672,7 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp) do_alloc: object = pcs->main->objects[--pcs->main->size]; - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); stat(s, ALLOC_PCS); @@ -4671,12 +4684,13 @@ unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, size_t size, void **p) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *main; - unsigned long flags; unsigned int allocated = 0; unsigned int batch; next_batch: - local_lock_irqsave(&s->cpu_sheaves->lock, flags); + if (!localtry_trylock(&s->cpu_sheaves->lock)) + return allocated; + pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(pcs->main->size == 0)) { @@ -4696,7 +4710,7 @@ next_batch: goto do_alloc; } - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); /* * Once full sheaves in barn are depleted, let the bulk @@ -4714,7 +4728,7 @@ do_alloc: main->size -= batch; memcpy(p, main->objects + main->size, batch * sizeof(void *)); - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); stat_add(s, ALLOC_PCS, batch); @@ -5134,13 +5148,14 @@ slab_empty: * The object is expected to have passed slab_free_hook() already. */ static __fastpath_inline -void free_to_pcs(struct kmem_cache *s, void *object) +bool free_to_pcs(struct kmem_cache *s, void *object) { struct slub_percpu_sheaves *pcs; - unsigned long flags; restart: - local_lock_irqsave(&s->cpu_sheaves->lock, flags); + if (!localtry_trylock(&s->cpu_sheaves->lock)) + return false; + pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(pcs->main->size == s->sheaf_capacity)) { @@ -5175,7 +5190,7 @@ restart: struct slab_sheaf *to_flush = pcs->spare; pcs->spare = NULL; - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); sheaf_flush(s, to_flush); empty = to_flush; @@ -5183,17 +5198,27 @@ restart: } alloc_empty: - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); empty = alloc_empty_sheaf(s, GFP_NOWAIT); if (!empty) { - sheaf_flush_main(s); - goto restart; + if (sheaf_flush_main(s)) + goto restart; + else + return false; } got_empty: - local_lock_irqsave(&s->cpu_sheaves->lock, flags); + if (!localtry_trylock(&s->cpu_sheaves->lock)) { + struct node_barn *barn; + + barn = get_node(s, numa_mem_id())->barn; + + barn_put_empty_sheaf(barn, empty, true); + return false; + } + pcs = this_cpu_ptr(s->cpu_sheaves); /* @@ -5222,9 +5247,11 @@ got_empty: do_free: pcs->main->objects[pcs->main->size++] = object; - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); stat(s, FREE_PCS); + + return true; } static void __rcu_free_sheaf_prepare(struct kmem_cache *s, @@ -5283,9 +5310,10 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *rcu_sheaf; - unsigned long flags; - local_lock_irqsave(&s->cpu_sheaves->lock, flags); + if (!localtry_trylock(&s->cpu_sheaves->lock)) + goto fail; + pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(!pcs->rcu_free)) { @@ -5299,16 +5327,16 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) goto do_free; } - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); empty = alloc_empty_sheaf(s, GFP_NOWAIT); - if (!empty) { - stat(s, FREE_RCU_SHEAF_FAIL); - return false; - } + if (!empty) + goto fail; + + if (!localtry_trylock(&s->cpu_sheaves->lock)) + goto fail; - local_lock_irqsave(&s->cpu_sheaves->lock, flags); pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(pcs->rcu_free)) @@ -5324,19 +5352,22 @@ do_free: rcu_sheaf->objects[rcu_sheaf->size++] = obj; if (likely(rcu_sheaf->size < s->sheaf_capacity)) { - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); stat(s, FREE_RCU_SHEAF); return true; } pcs->rcu_free = NULL; - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf); stat(s, FREE_RCU_SHEAF); - return true; + +fail: + stat(s, FREE_RCU_SHEAF_FAIL); + return false; } /* @@ -5348,7 +5379,6 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *main; - unsigned long flags; unsigned int batch, i = 0; bool init; @@ -5371,7 +5401,9 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) } next_batch: - local_lock_irqsave(&s->cpu_sheaves->lock, flags); + if (!localtry_trylock(&s->cpu_sheaves->lock)) + goto fallback; + pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(pcs->main->size == s->sheaf_capacity)) { @@ -5402,13 +5434,13 @@ next_batch: } no_empty: - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); /* * if we depleted all empty sheaves in the barn or there are too * many full sheaves, free the rest to slab pages */ - +fallback: __kmem_cache_free_bulk(s, size, p); return; } @@ -5420,7 +5452,7 @@ do_free: memcpy(main->objects + main->size, p, batch * sizeof(void *)); main->size += batch; - local_unlock_irqrestore(&s->cpu_sheaves->lock, flags); + localtry_unlock(&s->cpu_sheaves->lock); stat_add(s, FREE_PCS, batch); @@ -5520,9 +5552,7 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object, if (unlikely(!slab_free_hook(s, object, slab_want_init_on_free(s), false))) return; - if (s->cpu_sheaves) - free_to_pcs(s, object); - else + if (!s->cpu_sheaves || !free_to_pcs(s, object)) do_slab_free(s, slab, object, object, 1, addr); } @@ -6301,7 +6331,7 @@ static int init_percpu_sheaves(struct kmem_cache *s) pcs = per_cpu_ptr(s->cpu_sheaves, cpu); - local_lock_init(&pcs->lock); + localtry_lock_init(&pcs->lock); nid = cpu_to_mem(cpu); -- 2.50.1