From: Liam R. Howlett Date: Tue, 12 Oct 2021 13:34:44 +0000 (-0400) Subject: maple_tree: Rework mas_walk() for optimization. X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=23c41eee5f467db39e3934f34eb55dff306896e1;p=users%2Fjedix%2Flinux-maple.git maple_tree: Rework mas_walk() for optimization. Pass through the node when known and reduce the code overhead of the loops. There cannot be a on non-leaf nodes thanks to the fix on the right-most node max fix Signed-off-by: Liam R. Howlett --- diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 55c6bc67ddeb..3cf1d2775677 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -2073,8 +2073,9 @@ static inline unsigned char mas_store_b_node(struct ma_state *mas, return b_end; } -static inline void mas_node_walk(struct ma_state *mas, enum maple_type type, - unsigned long *range_min, unsigned long *range_max); +static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, + enum maple_type type, unsigned long *range_min, + unsigned long *range_max); /* * mas_prev_sibling() - Find the previous node with the same parent. @@ -2155,7 +2156,8 @@ static inline void mast_topiary(struct maple_subtree_state *mast) l_index = mast->orig_l->index; mast->orig_l->index = mast->orig_l->last; mt = mte_node_type(mast->orig_l->node); - mas_node_walk(mast->orig_l, mt, &range_min, &range_max); + mas_node_walk(mast->orig_l, mas_mn(mast->orig_l), + mt, &range_min, &range_max); mast->orig_l->index = l_index; l_off = mast->orig_l->offset; r_off = mast->orig_r->offset; @@ -2329,13 +2331,13 @@ mast_ascend_free(struct maple_subtree_state *mast) if (mast->orig_r->max < mast->orig_r->last) mast->orig_r->offset = mas_data_end(mast->orig_r) + 1; else - mas_node_walk(mast->orig_r, mte_node_type(mast->orig_r->node), - &range_min, &range_max); + mas_node_walk(mast->orig_r, mas_mn(mast->orig_r), + mte_node_type(mast->orig_r->node), &range_min, &range_max); /* Set up the left side of things */ mast->orig_l->offset = 0; mast->orig_l->index = mast->l->min; - mas_node_walk(mast->orig_l, mte_node_type(mast->orig_l->node), - &range_min, &range_max); + mas_node_walk(mast->orig_l, mas_mn(mast->orig_l), + mte_node_type(mast->orig_l->node), &range_min, &range_max); } /* @@ -3084,6 +3086,7 @@ static inline void mast_fill_bnode(struct maple_subtree_state *mast, if (cp) mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1, mast->bn, mast->bn->b_end); + mast->bn->b_end--; mast->bn->type = mte_node_type(mas->node); @@ -3506,15 +3509,16 @@ static bool mas_is_span_wr(struct ma_state *mas, unsigned long piv, * The offset will be stored in the maple state. * */ -static inline void mas_node_walk(struct ma_state *mas, enum maple_type type, - unsigned long *range_min, unsigned long *range_max) +static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, + enum maple_type type, unsigned long *range_min, + unsigned long *range_max) { - struct maple_node *node; unsigned long *pivots; - unsigned char offset, count; - unsigned long min, max, index; + unsigned char count; + unsigned long min, max; + unsigned char offset; + unsigned long index; - node = mas_mn(mas); pivots = ma_pivots(node, type); if (unlikely(ma_is_dense(type))) { (*range_max) = (*range_min) = mas->index; @@ -3527,15 +3531,15 @@ static inline void mas_node_walk(struct ma_state *mas, enum maple_type type, offset = mas->offset; min = mas_safe_min(mas, pivots, offset); - count = mt_pivots[type]; - index = mas->index; max = pivots[offset]; if (unlikely(ma_dead_node(node))) return; + count = mt_pivots[type]; if (unlikely(offset == count)) goto max; + index = mas->index; if (unlikely(index <= max)) goto done; @@ -3580,13 +3584,15 @@ done: static bool mas_wr_walk(struct ma_state *mas, unsigned long *range_min, unsigned long *range_max, void *entry) { + struct maple_node *node; enum maple_type type; while (true) { type = mte_node_type(mas->node); mas->depth++; - mas_node_walk(mas, type, range_min, range_max); + node = mas_mn(mas); + mas_node_walk(mas, node, type, range_min, range_max); if (mas_is_span_wr(mas, *range_max, type, entry)) return false; @@ -3671,31 +3677,25 @@ static inline void mas_extend_null(struct ma_state *l_mas, struct ma_state *r_ma static inline bool __mas_walk(struct ma_state *mas, unsigned long *range_min, unsigned long *range_max) { - struct maple_node *node; struct maple_enode *next; + struct maple_node *node; enum maple_type type; - node = mas_mn(mas); - if (unlikely(ma_dead_node(node))) - return false; - + next = mas->node; while (true) { - type = mte_node_type(mas->node); mas->depth++; - mas_node_walk(mas, type, range_min, range_max); - next = mas_slot(mas, ma_slots(mas_mn(mas), type), mas->offset); + node = mte_to_node(next); + type = mte_node_type(next); + mas_node_walk(mas, node, type, range_min, range_max); + next = mas_slot(mas, ma_slots(node, type), mas->offset); if (unlikely(ma_dead_node(node))) return false; if (unlikely(ma_is_leaf(type))) return true; - if (unlikely(!next)) - return false; - /* Descend. */ mas->node = next; - node = mas_mn(mas); mas->max = *range_max; mas->min = *range_min; mas->offset = 0; @@ -4392,8 +4392,8 @@ no_entry: static inline void *mas_next_nentry(struct ma_state *mas, struct maple_node *node, unsigned long max, enum maple_type type) { - unsigned long pivot = 0; unsigned char count; + unsigned long pivot; unsigned long *pivots; void __rcu **slots; void *entry; @@ -4404,8 +4404,8 @@ static inline void *mas_next_nentry(struct ma_state *mas, } pivots = ma_pivots(node, type); - mas->index = mas_safe_min(mas, pivots, mas->offset); slots = ma_slots(node, type); + mas->index = mas_safe_min(mas, pivots, mas->offset); if (ma_dead_node(node)) return NULL; @@ -5873,7 +5873,7 @@ EXPORT_SYMBOL_GPL(mas_pause); */ void *mas_find(struct ma_state *mas, unsigned long max) { - if (mas_is_paused(mas)) { + if (unlikely(mas_is_paused(mas))) { if (unlikely(mas->last == ULONG_MAX)) { mas->node = MAS_NONE; return NULL;