From: Liam R. Howlett Date: Tue, 12 Oct 2021 13:57:52 +0000 (-0400) Subject: maple_tree: Add mas_wr_node_walk() for writers X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=dd732f4c79be4dbe7fa9e5f7589a5364a6a6ddaf;p=users%2Fjedix%2Flinux-maple.git maple_tree: Add mas_wr_node_walk() for writers Writers don't need to check for dead nodes. Signed-off-by: Liam R. Howlett --- diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 3cf1d2775677..eff82d985484 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -2073,10 +2073,6 @@ static inline unsigned char mas_store_b_node(struct ma_state *mas, return b_end; } -static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, - enum maple_type type, unsigned long *range_min, - unsigned long *range_max); - /* * mas_prev_sibling() - Find the previous node with the same parent. * @mas: the maple state @@ -2139,6 +2135,70 @@ static inline struct maple_enode *mte_node_or_none(struct maple_enode *enode) return ma_enode_ptr(MAS_NONE); } +/* + * mas_wr_node_walk() - Find the correct offset for the index in the @mas. + * @mas: The maple state + * @type: The node type in the @mas + * @range_min: The minimum range + * @range_max: The maximum range + * + * Uses mas_slot_locked() and does not need to worry about dead nodes. + * + */ +static inline void mas_wr_node_walk(struct ma_state *mas, enum maple_type type, + unsigned long *range_min, unsigned long *range_max) +{ + struct maple_node *node; + unsigned long *pivots; + unsigned long min; + unsigned long max; + unsigned long index; + unsigned char count; + unsigned char offset; + + if (unlikely(ma_is_dense(type))) { + (*range_max) = (*range_min) = mas->index; + mas->offset = mas->index = mas->min; + return; + } + + node = mas_mn(mas); + count = mt_pivots[type]; + offset = mas->offset; + pivots = ma_pivots(node, type); + min = mas_safe_min(mas, pivots, offset); + max = pivots[offset]; + if (unlikely(offset == count)) + goto max; + + index = mas->index; + if (unlikely(index <= max)) + goto done; + + if (unlikely(!max && offset)) + goto max; + + offset++; + min = max + 1; + while (offset < count) { + max = pivots[offset]; + if (index <= max) + goto done; + + if (unlikely(!max)) + break; + + min = max + 1; + offset++; + } + +max: + max = mas->max; +done: + *range_max = max; + *range_min = min; + mas->offset = offset; +} /* * mast_topiary() - Add the portions of the tree to the removal list; either to * be freed or discarded (destroy walk). @@ -2156,8 +2216,7 @@ static inline void mast_topiary(struct maple_subtree_state *mast) l_index = mast->orig_l->index; mast->orig_l->index = mast->orig_l->last; mt = mte_node_type(mast->orig_l->node); - mas_node_walk(mast->orig_l, mas_mn(mast->orig_l), - mt, &range_min, &range_max); + mas_wr_node_walk(mast->orig_l, mt, &range_min, &range_max); mast->orig_l->index = l_index; l_off = mast->orig_l->offset; r_off = mast->orig_r->offset; @@ -2331,13 +2390,13 @@ mast_ascend_free(struct maple_subtree_state *mast) if (mast->orig_r->max < mast->orig_r->last) mast->orig_r->offset = mas_data_end(mast->orig_r) + 1; else - mas_node_walk(mast->orig_r, mas_mn(mast->orig_r), + mas_wr_node_walk(mast->orig_r, mte_node_type(mast->orig_r->node), &range_min, &range_max); /* Set up the left side of things */ mast->orig_l->offset = 0; mast->orig_l->index = mast->l->min; - mas_node_walk(mast->orig_l, mas_mn(mast->orig_l), - mte_node_type(mast->orig_l->node), &range_min, &range_max); + mas_wr_node_walk(mast->orig_l, mte_node_type(mast->orig_l->node), + &range_min, &range_max); } /* @@ -3499,78 +3558,6 @@ static bool mas_is_span_wr(struct ma_state *mas, unsigned long piv, return true; } -/* - * mas_node_walk() - Walk a maple node to offset of the index. - * @mas: The maple state - * @type: The maple node type - * @*range_min: Pointer to store the minimum range of the offset - * @*range_max: Pointer to store the maximum range of the offset - * - * The offset will be stored in the maple state. - * - */ -static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, - enum maple_type type, unsigned long *range_min, - unsigned long *range_max) -{ - unsigned long *pivots; - unsigned char count; - unsigned long min, max; - unsigned char offset; - unsigned long index; - - pivots = ma_pivots(node, type); - if (unlikely(ma_is_dense(type))) { - (*range_max) = (*range_min) = mas->index; - if (unlikely(ma_dead_node(node))) - return; - - mas->offset = mas->index = mas->min; - return; - } - - offset = mas->offset; - min = mas_safe_min(mas, pivots, offset); - max = pivots[offset]; - if (unlikely(ma_dead_node(node))) - return; - - count = mt_pivots[type]; - if (unlikely(offset == count)) - goto max; - - index = mas->index; - if (unlikely(index <= max)) - goto done; - - if (unlikely(!max && offset)) - goto max; - - offset++; - min = max + 1; - while (offset < count) { - max = pivots[offset]; - if (unlikely(ma_dead_node(node))) - return; - - if (index <= max) - goto done; - - if (unlikely(!max)) - break; - - min = max + 1; - offset++; - } - -max: - max = mas->max; -done: - *range_max = max; - *range_min = min; - mas->offset = offset; -} - /* * mas_wr_walk(): Walk the tree for a write. * @range_min - pointer that will be set to the minimum of the slot range @@ -3584,15 +3571,13 @@ done: static bool mas_wr_walk(struct ma_state *mas, unsigned long *range_min, unsigned long *range_max, void *entry) { - struct maple_node *node; enum maple_type type; while (true) { type = mte_node_type(mas->node); mas->depth++; - node = mas_mn(mas); - mas_node_walk(mas, node, type, range_min, range_max); + mas_wr_node_walk(mas, type, range_min, range_max); if (mas_is_span_wr(mas, *range_max, type, entry)) return false; @@ -3663,6 +3648,78 @@ static inline void mas_extend_null(struct ma_state *l_mas, struct ma_state *r_ma r_mas->offset = cp_r_slot; } +/* + * mas_node_walk() - Walk a maple node to offset of the index. + * @mas: The maple state + * @type: The maple node type + * @*range_min: Pointer to store the minimum range of the offset + * @*range_max: Pointer to store the maximum range of the offset + * + * The offset will be stored in the maple state. + * + */ +static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, + enum maple_type type, unsigned long *range_min, + unsigned long *range_max) + +{ + unsigned long *pivots; + unsigned char count; + unsigned long min, max; + unsigned char offset; + unsigned long index; + + pivots = ma_pivots(node, type); + if (unlikely(ma_is_dense(type))) { + (*range_max) = (*range_min) = mas->index; + if (unlikely(ma_dead_node(node))) + return; + + mas->offset = mas->index = mas->min; + return; + } + + offset = mas->offset; + min = mas_safe_min(mas, pivots, offset); + max = pivots[offset]; + if (unlikely(ma_dead_node(node))) + return; + + count = mt_pivots[type]; + if (unlikely(offset == count)) + goto max; + + index = mas->index; + if (unlikely(index <= max)) + goto done; + + if (unlikely(!max && offset)) + goto max; + + offset++; + min = max + 1; + while (offset < count) { + max = pivots[offset]; + if (unlikely(ma_dead_node(node))) + return; + + if (index <= max) + goto done; + + if (unlikely(!max)) + break; + + min = max + 1; + offset++; + } + +max: + max = mas->max; +done: + *range_max = max; + *range_min = min; + mas->offset = offset; +} /* * __mas_walk(): Locates a value and sets the mas->node and slot accordingly. * range_min and range_max are set to the range which the entry is valid.