From: Liam R. Howlett Date: Thu, 23 Sep 2021 19:40:32 +0000 (-0400) Subject: maple_tree: Continue mas_data_end quest, and finish it. X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=b32a3bec9a7dff5ba877ae621d4c2044b22a9dcc;p=users%2Fjedix%2Flinux-maple.git maple_tree: Continue mas_data_end quest, and finish it. Signed-off-by: Liam R. Howlett --- diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 7e85984b6ccc..2b7937c5c7ef 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -1316,44 +1316,20 @@ static inline unsigned char mas_data_end(struct ma_state *mas) unsigned long *pivots; type = mte_node_type(mas->node); - if (type == maple_arange_64) - return ma_meta_end(mte_to_node(mas->node), type); - - pivots = ma_pivots(mas_mn(mas), type); - offset = mt_pivots[type] - 1; - if (likely(!pivots[offset]) || pivots[offset] == mas->max) { - int ret = ma_meta_end(mas_mn(mas), type); - if (ret) - return ret; + if (type == maple_arange_64) { + offset = ma_meta_end(mte_to_node(mas->node), type); + return offset; } - offset = mt_min_slots[type]; - if (unlikely(!pivots[offset])) - goto decrement; - - /* Higher than the min. */ + pivots = ma_pivots(mas_mn(mas), type); offset = mt_pivots[type] - 1; - /* Check exceptions outside of the loop. */ - if (unlikely(pivots[offset])) { - /* At least almost full. */ - - /* Totally full. */ - if (pivots[offset] != mas->max) - return offset + 1; - + if (unlikely(pivots[offset] == mas->max)) return offset; - } -decrement: - while (--offset) { - if (likely(pivots[offset])) - break; - } + if (unlikely(pivots[offset])) + return mt_pivots[type]; - if (likely(pivots[offset] < mas->max)) - offset++; - - return offset; + return ma_meta_end(mas_mn(mas), type); } /* @@ -1597,9 +1573,8 @@ static inline void mas_adopt_children(struct ma_state *mas, void __rcu **slots = ma_slots(mte_to_node(mas->node), type); struct maple_enode *child; unsigned char offset; - unsigned char count = mas_data_end(mas); - for (offset = 0; offset <= count; offset++) { + for (offset = 0; offset <= mas_data_end(mas); offset++) { child = mas_slot_locked(mas, slots, offset); mte_set_parent(child, parent, offset); } @@ -1655,7 +1630,7 @@ static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child) __must_hold(mas->tree->lock) { enum maple_type mt; - unsigned char offset, count; + unsigned char offset; struct maple_enode *entry; struct maple_node *node; void __rcu **slots; @@ -1663,8 +1638,7 @@ static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child) mt = mte_node_type(mas->node); node = mas_mn(mas); slots = ma_slots(node, mt); - count = mas_data_end(mas); - for (offset = mas->offset; offset <= count; offset++) { + for (offset = mas->offset; offset <= mas_data_end(mas); offset++) { entry = mas_slot_locked(mas, slots, offset); if (mte_parent(entry) == node) { *child = *mas; @@ -2093,7 +2067,6 @@ static inline bool mas_prev_sibling(struct ma_state *mas) */ static inline bool mas_next_sibling(struct ma_state *mas) { - unsigned char end; MA_STATE(parent, mas->tree, mas->index, mas->last); if (mte_is_root(mas->node)) @@ -2101,12 +2074,8 @@ static inline bool mas_next_sibling(struct ma_state *mas) parent = *mas; mas_ascend(&parent); - end = mas_data_end(&parent); parent.offset = mte_parent_slot(mas->node) + 1; - if (parent.offset > end) - return false; - - if (!mas_get_slot(&parent, parent.offset)) + if (parent.offset > mas_data_end(&parent)) return false; *mas = parent; @@ -2210,18 +2179,18 @@ static inline void mast_rebalance_next(struct maple_subtree_state *mast, static inline void mast_rebalance_prev(struct maple_subtree_state *mast, struct maple_enode *old_l) { - unsigned char end = mas_data_end(mast->orig_l); + unsigned char end = mas_data_end(mast->orig_l) + 1; unsigned char b_end = mast->bn->b_end; - mab_shift_right(mast->bn, end + 1); - mas_mab_cp(mast->orig_l, 0, end, mast->bn, 0); + mab_shift_right(mast->bn, end); + mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0); mat_add(mast->free, old_l); if (mast->orig_r->node == old_l) mast->orig_r->node = mast->orig_l->node; mast->l->min = mast->orig_l->min; mast->orig_l->index = mast->orig_l->min; - mast->bn->b_end = end + 1 + b_end; - mast->l->offset += end + 1; + mast->bn->b_end = end + b_end; + mast->l->offset += end; } /* @@ -2902,10 +2871,6 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end node = mas_mn(mas); newnode->parent = node->parent; - pivs = ma_pivots(newnode, mt); - - - slots = ma_slots(newnode, mt); pivs = ma_pivots(newnode, mt); left = mas_mn(&l_mas); @@ -2925,11 +2890,11 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end mas->min = l_mas.max + 1; eparent = mt_mk_node(mte_parent(l_mas.node), mas_parent_enum(&l_mas, l_mas.node)); + tmp += end; if (!mt_in_rcu(mas->tree)) { unsigned char max_p = mt_pivots[mt]; unsigned char max_s = mt_slots[mt]; - tmp += end; if (tmp < max_p) memset(pivs + tmp, 0, sizeof(unsigned long *) * (max_p - tmp)); @@ -2938,6 +2903,7 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end memset(slots + tmp, 0, sizeof(void *) * (max_s - tmp)); memcpy(node, newnode, sizeof(struct maple_node)); + ma_set_meta(node, mt, 0, tmp - 1); mte_set_pivot(eparent, mte_parent_slot(l_mas.node), l_pivs[split]); @@ -2945,12 +2911,14 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end tmp = split + 1; memset(l_pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp)); memset(l_slots + tmp, 0, sizeof(void *) * (max_s - tmp)); + ma_set_meta(left, mt, 0, split); goto done; } /* RCU requires replacing both l_mas, mas, and parent. */ mas->node = mt_mk_node(newnode, mt); + ma_set_meta(newnode, mt, 0, tmp); new_left = mas_pop_node(mas); new_left->parent = left->parent; @@ -2959,6 +2927,7 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end pivs = ma_pivots(new_left, mt); memcpy(slots, l_slots, sizeof(void *) * split); memcpy(pivs, l_pivs, sizeof(unsigned long) * split); + ma_set_meta(new_left, mt, 0, split); l_mas.node = mt_mk_node(new_left, mt); @@ -3934,8 +3903,6 @@ done: if (!dst_pivots[mt_pivots[mt] - 1] || dst_pivots[mt_pivots[mt] - 1] == mas->max) { if (dst_pivots[new_end] && dst_pivots[new_end] < mas->max) new_end++; - //printk("Set new end %u pivot is %lu max is %lu\n", new_end, dst_pivots[new_end], - // mas->max); ma_set_meta(newnode, maple_leaf_64, 0, new_end); } @@ -4302,7 +4269,7 @@ static inline int mas_next_node(struct ma_state *mas, unsigned long max) struct maple_node *node; struct maple_enode *enode; int level = 0; - unsigned char offset, end; + unsigned char offset; enum maple_type mt; void __rcu **slots; @@ -4326,8 +4293,7 @@ static inline int mas_next_node(struct ma_state *mas, unsigned long max) offset = mas->offset; level++; - end = mas_data_end(mas); - } while (unlikely(offset == end)); + } while (unlikely(offset == mas_data_end(mas))); node = mas_mn(mas); mt = mte_node_type(mas->node); @@ -4385,7 +4351,6 @@ static inline void *mas_next_nentry(struct ma_state *mas, unsigned long max, struct maple_node *node; unsigned long pivot = 0; unsigned long r_start; - unsigned char count; unsigned char offset; unsigned long *pivots; void __rcu **slots; @@ -4401,7 +4366,6 @@ static inline void *mas_next_nentry(struct ma_state *mas, unsigned long max, type = mte_node_type(mas->node); pivots = ma_pivots(node, type); r_start = mas_safe_min(mas, pivots, offset); - count = mas_data_end(mas); slots = ma_slots(node, type); if (mte_dead_node(mas->node)) @@ -4413,7 +4377,7 @@ static inline void *mas_next_nentry(struct ma_state *mas, unsigned long max, } - while (offset < count) { + while (offset < mas_data_end(mas)) { pivot = pivots[offset]; entry = mas_slot(mas, slots, offset); if (mte_dead_node(mas->node)) @@ -6617,7 +6581,6 @@ void mas_validate_gaps(struct ma_state *mas) pr_err(MA_PTR"[%u] %lu >= %lu - %lu + 1 (%lu)\n", mas_mn(mas), i, gap, p_end, p_start, p_end - p_start + 1); - mt_dump(mas->tree); MT_BUG_ON(mas->tree, gap > p_end - p_start + 1); } @@ -6770,7 +6733,7 @@ void mas_validate_limits(struct ma_state *mas) void mt_validate_nulls(struct maple_tree *mt) { void *entry, *last = (void *)1; - unsigned char end, offset = 0; + unsigned char offset = 0; void __rcu **slots; MA_STATE(mas, mt, 0, 0); @@ -6782,7 +6745,6 @@ void mt_validate_nulls(struct maple_tree *mt) mas_descend(&mas); slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node)); - end = mas_data_end(&mas); do { entry = mas_slot(&mas, slots, offset); if (!last && !entry) { @@ -6791,12 +6753,11 @@ void mt_validate_nulls(struct maple_tree *mt) } MT_BUG_ON(mt, !last && !entry); last = entry; - if (offset == end) { + if (offset == mas_data_end(&mas)) { mas_next_node(&mas, ULONG_MAX); if (mas_is_none(&mas)) return; offset = 0; - end = mas_data_end(&mas); slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node)); } else