From f6033f6b83e33a88b073539ffc22012c8e48bc06 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 14 Feb 2025 17:27:43 +0100 Subject: [PATCH] slab: determine barn status racily outside of lock The possibility of many barn operations is determined by the current number of full or empty sheaves. Taking the barn->lock just to find out that e.g. there are no empty sheaves results in unnecessary overhead and lock contention. Thus perform these checks outside of the lock with a data_race() annotated variable read and fail quickly without taking the lock. Checks for sheaf availability that racily succeed have to be obviously repeated under the lock for correctness, but we can skip repeating checks if there are too many sheaves on the given list as the limits don't need to be strict. Signed-off-by: Vlastimil Babka Reviewed-by: Suren Baghdasaryan Reviewed-by: Harry Yoo --- mm/slub.c | 57 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index a9412933dbe3..1a545ac18155 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2698,9 +2698,12 @@ static struct slab_sheaf *barn_get_empty_sheaf(struct node_barn *barn) struct slab_sheaf *empty = NULL; unsigned long flags; + if (!data_race(barn->nr_empty)) + return NULL; + spin_lock_irqsave(&barn->lock, flags); - if (barn->nr_empty) { + if (likely(barn->nr_empty)) { empty = list_first_entry(&barn->sheaves_empty, struct slab_sheaf, barn_list); list_del(&empty->barn_list); @@ -2716,38 +2719,36 @@ static int barn_put_empty_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf, bool ignore_limit) { unsigned long flags; - int ret = 0; + + /* we don't repeat the check under barn->lock as it's not critical */ + if (!ignore_limit && data_race(barn->nr_empty) >= MAX_EMPTY_SHEAVES) + return -E2BIG; spin_lock_irqsave(&barn->lock, flags); - if (!ignore_limit && barn->nr_empty >= MAX_EMPTY_SHEAVES) { - ret = -E2BIG; - } else { - list_add(&sheaf->barn_list, &barn->sheaves_empty); - barn->nr_empty++; - } + list_add(&sheaf->barn_list, &barn->sheaves_empty); + barn->nr_empty++; spin_unlock_irqrestore(&barn->lock, flags); - return ret; + return 0; } static int barn_put_full_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf, bool ignore_limit) { unsigned long flags; - int ret = 0; + + /* we don't repeat the check under barn->lock as it's not critical */ + if (!ignore_limit && data_race(barn->nr_full) >= MAX_FULL_SHEAVES) + return -E2BIG; spin_lock_irqsave(&barn->lock, flags); - if (!ignore_limit && barn->nr_full >= MAX_FULL_SHEAVES) { - ret = -E2BIG; - } else { - list_add(&sheaf->barn_list, &barn->sheaves_full); - barn->nr_full++; - } + list_add(&sheaf->barn_list, &barn->sheaves_full); + barn->nr_full++; spin_unlock_irqrestore(&barn->lock, flags); - return ret; + return 0; } static struct slab_sheaf *barn_get_full_or_empty_sheaf(struct node_barn *barn) @@ -2755,6 +2756,9 @@ static struct slab_sheaf *barn_get_full_or_empty_sheaf(struct node_barn *barn) struct slab_sheaf *sheaf = NULL; unsigned long flags; + if (!data_race(barn->nr_full) && !data_race(barn->nr_empty)) + return NULL; + spin_lock_irqsave(&barn->lock, flags); if (barn->nr_full) { @@ -2785,9 +2789,12 @@ barn_replace_empty_sheaf(struct node_barn *barn, struct slab_sheaf *empty) struct slab_sheaf *full = NULL; unsigned long flags; + if (!data_race(barn->nr_full)) + return NULL; + spin_lock_irqsave(&barn->lock, flags); - if (barn->nr_full) { + if (likely(barn->nr_full)) { full = list_first_entry(&barn->sheaves_full, struct slab_sheaf, barn_list); list_del(&full->barn_list); @@ -2810,19 +2817,23 @@ barn_replace_full_sheaf(struct node_barn *barn, struct slab_sheaf *full) struct slab_sheaf *empty; unsigned long flags; + /* we don't repeat this check under barn->lock as it's not critical */ + if (data_race(barn->nr_full) >= MAX_FULL_SHEAVES) + return ERR_PTR(-E2BIG); + if (!data_race(barn->nr_empty)) + return ERR_PTR(-ENOMEM); + spin_lock_irqsave(&barn->lock, flags); - if (barn->nr_full >= MAX_FULL_SHEAVES) { - empty = ERR_PTR(-E2BIG); - } else if (!barn->nr_empty) { - empty = ERR_PTR(-ENOMEM); - } else { + if (likely(barn->nr_empty)) { empty = list_first_entry(&barn->sheaves_empty, struct slab_sheaf, barn_list); list_del(&empty->barn_list); list_add(&full->barn_list, &barn->sheaves_full); barn->nr_empty--; barn->nr_full++; + } else { + empty = ERR_PTR(-ENOMEM); } spin_unlock_irqrestore(&barn->lock, flags); -- 2.49.0