From 02d25914f0962ee8a52e243c2193aca4a715ccb2 Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Sat, 7 Dec 2024 21:29:07 -0500 Subject: [PATCH] maple_tree: Convert forking to use the sheaf interface Use the generic interface which should result in less bulk allocations during a forking. A part of this is to abstract the freeing of the sheaf or maple state allocations into its own function so mas_destroy() and the tree duplication code can use the same functionality to return any unused resources. Signed-off-by: Liam R. Howlett --- lib/maple_tree.c | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 118aa6c49da5..8795b7850622 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -1153,6 +1153,20 @@ error: mas_set_err(mas, -ENOMEM); } +static inline void mas_empty_nodes(struct ma_state *mas) +{ + mas->node_request = 0; + if (mas->sheaf) { + kmem_cache_return_sheaf(maple_node_cache, GFP_KERNEL, + mas->sheaf); + mas->sheaf = NULL; + } + + if (mas->alloc) { + mt_free_one(mas->alloc); + mas->alloc = NULL; + } +} /* * mas_free() - Free an encoded maple node * @mas: The maple state @@ -5411,17 +5425,7 @@ void mas_destroy(struct ma_state *mas) mas->mas_flags &= ~MA_STATE_REBALANCE; } mas->mas_flags &= ~(MA_STATE_BULK|MA_STATE_PREALLOC); - - mas->node_request = 0; - if (mas->sheaf) - kmem_cache_return_sheaf(maple_node_cache, GFP_KERNEL, - mas->sheaf); - if (mas->allocated) { - kmem_cache_free_bulk(maple_node_cache, mas->allocated, - (void **)mas->alloc); - mas->allocated = 0; - } - mas->sheaf = NULL; + mas_empty_nodes(mas); } EXPORT_SYMBOL_GPL(mas_destroy); @@ -6498,7 +6502,7 @@ static inline void mas_dup_alloc(struct ma_state *mas, struct ma_state *new_mas, struct maple_node *node = mte_to_node(mas->node); struct maple_node *new_node = mte_to_node(new_mas->node); enum maple_type type; - unsigned char request, count, i; + unsigned char count, i; void __rcu **slots; void __rcu **new_slots; unsigned long val; @@ -6506,20 +6510,17 @@ static inline void mas_dup_alloc(struct ma_state *mas, struct ma_state *new_mas, /* Allocate memory for child nodes. */ type = mte_node_type(mas->node); new_slots = ma_slots(new_node, type); - request = mas_data_end(mas) + 1; - count = mt_alloc_bulk(gfp, request, (void **)new_slots); - if (unlikely(count < request)) { - memset(new_slots, 0, request * sizeof(void *)); - mas_set_err(mas, -ENOMEM); + count = mas->node_request = mas_data_end(mas) + 1; + mas_alloc_nodes(mas, gfp); + if (unlikely(mas_is_err(mas))) return; - } - /* Restore node type information in slots. */ slots = ma_slots(node, type); for (i = 0; i < count; i++) { val = (unsigned long)mt_slot_locked(mas->tree, slots, i); val &= MAPLE_NODE_MASK; - ((unsigned long *)new_slots)[i] |= val; + new_slots[i] = ma_mnode_ptr((unsigned long)mas_pop_node(mas) | + val); } } @@ -6573,7 +6574,7 @@ static inline void mas_dup_build(struct ma_state *mas, struct ma_state *new_mas, /* Only allocate child nodes for non-leaf nodes. */ mas_dup_alloc(mas, new_mas, gfp); if (unlikely(mas_is_err(mas))) - return; + goto empty_mas; } else { /* * This is the last leaf node and duplication is @@ -6606,6 +6607,8 @@ set_new_tree: /* Make them the same height */ new_mas->tree->ma_flags = mas->tree->ma_flags; rcu_assign_pointer(new_mas->tree->ma_root, root); +empty_mas: + mas_empty_nodes(mas); } /** -- 2.49.0