maple_tree: Continue mas_data_end quest, and finish it.

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 7e85984b6ccc24a9bc38a85c4a265f028e059401..2b7937c5c7ef318bf03dd7b59bb7351da0072940 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -1316,44 +1316,20 @@ static inline unsigned char mas_data_end(struct ma_state *mas)
        unsigned long *pivots;
 
        type = mte_node_type(mas->node);
-       if (type == maple_arange_64)
-               return ma_meta_end(mte_to_node(mas->node), type);
-
-       pivots = ma_pivots(mas_mn(mas), type);
-       offset = mt_pivots[type] - 1;
-       if (likely(!pivots[offset]) || pivots[offset] == mas->max) {
-               int ret = ma_meta_end(mas_mn(mas), type);
-               if (ret)
-                       return ret;
+       if (type == maple_arange_64) {
+               offset = ma_meta_end(mte_to_node(mas->node), type);
+               return offset;
        }
 
-       offset = mt_min_slots[type];
-       if (unlikely(!pivots[offset]))
-               goto decrement;
-
-       /* Higher than the min. */
+       pivots = ma_pivots(mas_mn(mas), type);
        offset = mt_pivots[type] - 1;
-       /* Check exceptions outside of the loop. */
-       if (unlikely(pivots[offset])) {
-               /* At least almost full. */
-
-               /* Totally full. */
-               if (pivots[offset] != mas->max)
-                       return offset + 1;
-
+       if (unlikely(pivots[offset] == mas->max))
                return offset;
-       }
 
-decrement:
-       while (--offset) {
-               if (likely(pivots[offset]))
-                       break;
-       }
+       if (unlikely(pivots[offset]))
+               return mt_pivots[type];
 
-       if (likely(pivots[offset] < mas->max))
-               offset++;
-
-       return offset;
+       return ma_meta_end(mas_mn(mas), type);
 }
 
 /*
@@ -1597,9 +1573,8 @@ static inline void mas_adopt_children(struct ma_state *mas,
        void __rcu **slots = ma_slots(mte_to_node(mas->node), type);
        struct maple_enode *child;
        unsigned char offset;
-       unsigned char count = mas_data_end(mas);
 
-       for (offset = 0; offset <= count; offset++) {
+       for (offset = 0; offset <= mas_data_end(mas); offset++) {
                child = mas_slot_locked(mas, slots, offset);
                mte_set_parent(child, parent, offset);
        }
@@ -1655,7 +1630,7 @@ static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child)
        __must_hold(mas->tree->lock)
 {
        enum maple_type mt;
-       unsigned char offset, count;
+       unsigned char offset;
        struct maple_enode *entry;
        struct maple_node *node;
        void __rcu **slots;
@@ -1663,8 +1638,7 @@ static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child)
        mt = mte_node_type(mas->node);
        node = mas_mn(mas);
        slots = ma_slots(node, mt);
-       count = mas_data_end(mas);
-       for (offset = mas->offset; offset <= count; offset++) {
+       for (offset = mas->offset; offset <= mas_data_end(mas); offset++) {
                entry = mas_slot_locked(mas, slots, offset);
                if (mte_parent(entry) == node) {
                        *child = *mas;
@@ -2093,7 +2067,6 @@ static inline bool mas_prev_sibling(struct ma_state *mas)
  */
 static inline bool mas_next_sibling(struct ma_state *mas)
 {
-       unsigned char end;
        MA_STATE(parent, mas->tree, mas->index, mas->last);
 
        if (mte_is_root(mas->node))
@@ -2101,12 +2074,8 @@ static inline bool mas_next_sibling(struct ma_state *mas)
 
        parent = *mas;
        mas_ascend(&parent);
-       end = mas_data_end(&parent);
        parent.offset = mte_parent_slot(mas->node) + 1;
-       if (parent.offset > end)
-               return false;
-
-       if (!mas_get_slot(&parent, parent.offset))
+       if (parent.offset > mas_data_end(&parent))
                return false;
 
        *mas = parent;
@@ -2210,18 +2179,18 @@ static inline void mast_rebalance_next(struct maple_subtree_state *mast,
 static inline void mast_rebalance_prev(struct maple_subtree_state *mast,
                                       struct maple_enode *old_l)
 {
-       unsigned char end = mas_data_end(mast->orig_l);
+       unsigned char end = mas_data_end(mast->orig_l) + 1;
        unsigned char b_end = mast->bn->b_end;
 
-       mab_shift_right(mast->bn, end + 1);
-       mas_mab_cp(mast->orig_l, 0, end, mast->bn, 0);
+       mab_shift_right(mast->bn, end);
+       mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0);
        mat_add(mast->free, old_l);
        if (mast->orig_r->node == old_l)
                mast->orig_r->node = mast->orig_l->node;
        mast->l->min = mast->orig_l->min;
        mast->orig_l->index = mast->orig_l->min;
-       mast->bn->b_end = end + 1 + b_end;
-       mast->l->offset += end + 1;
+       mast->bn->b_end = end + b_end;
+       mast->l->offset += end;
 }
 
 /*
@@ -2902,10 +2871,6 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end
 
        node = mas_mn(mas);
        newnode->parent = node->parent;
-       pivs = ma_pivots(newnode, mt);
-
-
-
        slots = ma_slots(newnode, mt);
        pivs = ma_pivots(newnode, mt);
        left = mas_mn(&l_mas);
@@ -2925,11 +2890,11 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end
        mas->min = l_mas.max + 1;
        eparent = mt_mk_node(mte_parent(l_mas.node),
                             mas_parent_enum(&l_mas, l_mas.node));
+       tmp += end;
        if (!mt_in_rcu(mas->tree)) {
                unsigned char max_p = mt_pivots[mt];
                unsigned char max_s = mt_slots[mt];
 
-               tmp += end;
                if (tmp < max_p)
                        memset(pivs + tmp, 0,
                               sizeof(unsigned long *) * (max_p - tmp));
@@ -2938,6 +2903,7 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end
                        memset(slots + tmp, 0, sizeof(void *) * (max_s - tmp));
 
                memcpy(node, newnode, sizeof(struct maple_node));
+               ma_set_meta(node, mt, 0, tmp - 1);
                mte_set_pivot(eparent, mte_parent_slot(l_mas.node),
                              l_pivs[split]);
 
@@ -2945,12 +2911,14 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end
                tmp = split + 1;
                memset(l_pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp));
                memset(l_slots + tmp, 0, sizeof(void *) * (max_s - tmp));
+               ma_set_meta(left, mt, 0, split);
 
                goto done;
        }
 
        /* RCU requires replacing both l_mas, mas, and parent. */
        mas->node = mt_mk_node(newnode, mt);
+       ma_set_meta(newnode, mt, 0, tmp);
 
        new_left = mas_pop_node(mas);
        new_left->parent = left->parent;
@@ -2959,6 +2927,7 @@ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end
        pivs = ma_pivots(new_left, mt);
        memcpy(slots, l_slots, sizeof(void *) * split);
        memcpy(pivs, l_pivs, sizeof(unsigned long) * split);
+       ma_set_meta(new_left, mt, 0, split);
        l_mas.node = mt_mk_node(new_left, mt);
 
 
@@ -3934,8 +3903,6 @@ done:
        if (!dst_pivots[mt_pivots[mt] - 1] || dst_pivots[mt_pivots[mt] - 1] == mas->max) {
                if (dst_pivots[new_end] && dst_pivots[new_end] < mas->max)
                        new_end++;
-               //printk("Set new end %u pivot is %lu max is %lu\n", new_end, dst_pivots[new_end],
-               //      mas->max);
                ma_set_meta(newnode, maple_leaf_64, 0, new_end);
        }
 
@@ -4302,7 +4269,7 @@ static inline int mas_next_node(struct ma_state *mas, unsigned long max)
        struct maple_node *node;
        struct maple_enode *enode;
        int level = 0;
-       unsigned char offset, end;
+       unsigned char offset;
        enum maple_type mt;
        void __rcu **slots;
 
@@ -4326,8 +4293,7 @@ static inline int mas_next_node(struct ma_state *mas, unsigned long max)
 
                offset = mas->offset;
                level++;
-               end = mas_data_end(mas);
-       } while (unlikely(offset == end));
+       } while (unlikely(offset == mas_data_end(mas)));
 
        node = mas_mn(mas);
        mt = mte_node_type(mas->node);
@@ -4385,7 +4351,6 @@ static inline void *mas_next_nentry(struct ma_state *mas, unsigned long max,
        struct maple_node *node;
        unsigned long pivot = 0;
        unsigned long r_start;
-       unsigned char count;
        unsigned char offset;
        unsigned long *pivots;
        void __rcu **slots;
@@ -4401,7 +4366,6 @@ static inline void *mas_next_nentry(struct ma_state *mas, unsigned long max,
        type = mte_node_type(mas->node);
        pivots = ma_pivots(node, type);
        r_start = mas_safe_min(mas, pivots, offset);
-       count = mas_data_end(mas);
        slots = ma_slots(node, type);
 
        if (mte_dead_node(mas->node))
@@ -4413,7 +4377,7 @@ static inline void *mas_next_nentry(struct ma_state *mas, unsigned long max,
        }
 
 
-       while (offset < count) {
+       while (offset < mas_data_end(mas)) {
                pivot = pivots[offset];
                entry = mas_slot(mas, slots, offset);
                if (mte_dead_node(mas->node))
@@ -6617,7 +6581,6 @@ void mas_validate_gaps(struct ma_state *mas)
                                        pr_err(MA_PTR"[%u] %lu >= %lu - %lu + 1 (%lu)\n",
                                        mas_mn(mas), i, gap, p_end, p_start,
                                        p_end - p_start + 1);
-                                       mt_dump(mas->tree);
                                        MT_BUG_ON(mas->tree,
                                                gap > p_end - p_start + 1);
                                }
@@ -6770,7 +6733,7 @@ void mas_validate_limits(struct ma_state *mas)
 void mt_validate_nulls(struct maple_tree *mt)
 {
        void *entry, *last = (void *)1;
-       unsigned char end, offset = 0;
+       unsigned char offset = 0;
        void __rcu **slots;
        MA_STATE(mas, mt, 0, 0);
 
@@ -6782,7 +6745,6 @@ void mt_validate_nulls(struct maple_tree *mt)
                mas_descend(&mas);
 
        slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node));
-       end = mas_data_end(&mas);
        do {
                entry = mas_slot(&mas, slots, offset);
                if (!last && !entry) {
@@ -6791,12 +6753,11 @@ void mt_validate_nulls(struct maple_tree *mt)
                }
                MT_BUG_ON(mt, !last && !entry);
                last = entry;
-               if (offset == end) {
+               if (offset == mas_data_end(&mas)) {
                        mas_next_node(&mas, ULONG_MAX);
                        if (mas_is_none(&mas))
                                return;
                        offset = 0;
-                       end = mas_data_end(&mas);
                        slots = ma_slots(mte_to_node(mas.node),
                                         mte_node_type(mas.node));
                } else