maple_tree: Don't record position in mas_split() and mas_spanning_reblance()

Just walk there in the end

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 9cb5bfb7cc60523cc194da7e1da6fbfe0c66b2a3..740ccbbb2a494bc39faf06c36821bb42e63ba1c3 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -2625,8 +2625,7 @@ static inline void mast_new_root(struct maple_subtree_state *mast,
  */
 static inline void mast_cp_to_nodes(struct maple_subtree_state *mast,
        struct maple_enode *left, struct maple_enode *middle,
-       struct maple_enode *right, unsigned char split, unsigned char mid_split,
-       struct ma_state *save)
+       struct maple_enode *right, unsigned char split, unsigned char mid_split)
 {
        mast->l->node = mte_node_or_none(left);
        mast->m->node = mte_node_or_none(middle);
@@ -2641,13 +2640,6 @@ static inline void mast_cp_to_nodes(struct maple_subtree_state *mast,
                mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true);
                mast->m->min = mast->bn->pivot[split] + 1;
                mast->m->max = mast->bn->pivot[mid_split];
-               if (!save->node &&
-                   (save->offset > split) && (save->offset < mid_split)) {
-                       save->offset -= (split + 1);
-                       save->node= mast->m->node;
-                       save->min = mast->m->min;
-                       save->max = mast->m->max;
-               }
                split = mid_split;
        }
 
@@ -2655,17 +2647,6 @@ static inline void mast_cp_to_nodes(struct maple_subtree_state *mast,
                mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, true);
                mast->r->min = mast->bn->pivot[split] + 1;
                mast->r->max = mast->bn->pivot[mast->bn->b_end];
-               if (!save->node && (save->offset > split)) {
-                       save->offset -= (split + 1);
-                       save->node= mast->r->node;
-                       save->min = mast->r->min;
-                       save->max = mast->r->max;
-               }
-       }
-       if (!save->node) {
-               save->node= mast->l->node;
-               save->min = mast->l->min;
-               save->max = mast->l->max;
        }
 }
 
@@ -2738,6 +2719,75 @@ static inline void mast_setup_bnode_for_split(struct maple_subtree_state *mast)
        mast->bn->type = mte_node_type(mast->orig_l->node);
 }
 
+
+static inline void *mtree_range_walk(struct ma_state *mas)
+{
+       unsigned long *pivots;
+       unsigned char offset;
+       struct maple_node *node;
+       struct maple_enode *next, *last;
+       enum maple_type type;
+       void __rcu **slots;
+       unsigned char end;
+       unsigned long max, min;
+       unsigned long prev_max, prev_min;
+
+       last = next = mas->node;
+       prev_min = min = mas->min;
+       max = mas->max;
+       do {
+               offset = 0;
+               last = next;
+               node = mte_to_node(next);
+               type = mte_node_type(next);
+               pivots = ma_pivots(node, type);
+               if (unlikely(ma_dead_node(node)))
+                       goto dead_node;
+
+               end = ma_data_end(node, type, pivots, max);
+               if (pivots[offset] >= mas->index) {
+                       prev_max = max;
+                       prev_min = min;
+                       max = pivots[offset];
+                       goto next;
+               }
+
+               do {
+                       offset++;
+               } while((offset < end) && (pivots[offset] < mas->index));
+
+               prev_min = min;
+               prev_max = max;
+               if (offset < mt_pivots[type] && pivots[offset])
+                       max = pivots[offset];
+
+               if (offset)
+                       min = pivots[offset - 1] + 1;
+
+               if (likely(offset > end) && pivots[offset]) {
+next:
+                       max = pivots[offset];
+               }
+
+               slots = ma_slots(node, type);
+               next = mt_slot(mas->tree, slots, offset);
+               if (unlikely(ma_dead_node(node)))
+                       goto dead_node;
+       } while (!ma_is_leaf(type));
+
+       mas->offset = offset;
+       mas->index = min;
+       mas->last = max;
+       mas->min = prev_min;
+       mas->max = prev_max;
+       mas->node = last;
+       return (void *) next;
+
+dead_node:
+       mas_reset(mas);
+       return NULL;
+}
+
 /*
  * mas_spanning_rebalance() - Rebalance across two nodes which may not be peers.
  * @mas: The starting maple state
@@ -2759,7 +2809,6 @@ static inline void mast_setup_bnode_for_split(struct maple_subtree_state *mast)
 static int mas_spanning_rebalance(struct ma_state *mas,
                struct maple_subtree_state *mast, unsigned char count)
 {
-       struct ma_state restore;
        unsigned char split, mid_split;
        unsigned char slot = 0;
        struct maple_enode *left = NULL, *middle = NULL, *right = NULL;
@@ -2785,8 +2834,6 @@ static int mas_spanning_rebalance(struct ma_state *mas,
                if (!mast_sibling_rebalance_right(mast, false))
                        mast_cousin_rebalance_right(mast, false);
        }
-       restore.node = NULL;
-       restore.offset = mas->offset;
        mast->orig_l->depth = 0;
 
        while (count--) {
@@ -2795,8 +2842,7 @@ static int mas_spanning_rebalance(struct ma_state *mas,
                                        &mid_split);
                mast_set_split_parents(mast, left, middle, right, split,
                                       mid_split);
-               mast_cp_to_nodes(mast, left, middle, right, split, mid_split,
-                                &restore);
+               mast_cp_to_nodes(mast, left, middle, right, split, mid_split);
 
                /*
                 * Copy data from next level in the tree to mast->bn from next
@@ -2870,10 +2916,7 @@ new_root:
        mast->orig_l->alloc = mas->alloc;
        *mas = *mast->orig_l;
        mas_wmb_replace(mas, &free, &destroy);
-       mas->offset = restore.offset;
-       mas->min = restore.min;
-       mas->max = restore.max;
-       mas->node = restore.node;
+       mtree_range_walk(mas);
        return mast->bn->b_end;
 }
 
@@ -3155,7 +3198,7 @@ static inline void mast_fill_bnode(struct maple_subtree_state *mast,
  * @split: The location to split the big node
  */
 static inline void mast_split_data(struct maple_subtree_state *mast,
-          struct ma_state *mas, unsigned char split, struct ma_state *save)
+          struct ma_state *mas, unsigned char split)
 {
        unsigned char p_slot;
 
@@ -3165,24 +3208,14 @@ static inline void mast_split_data(struct maple_subtree_state *mast,
        mast->l->offset = mte_parent_slot(mas->node);
        mast->l->max = mast->bn->pivot[split];
        mast->r->min = mast->l->max + 1;
-       if (!mte_is_leaf(mas->node)) {
-               p_slot = mast->orig_l->offset;
-               mas_set_split_parent(mast->orig_l, mast->l->node,
-                                    mast->r->node, &p_slot, split);
-               mas_set_split_parent(mast->orig_r, mast->l->node,
-                                    mast->r->node, &p_slot, split);
-       } else {
-               if (save->offset > split) {
-                       save->node = mast->r->node;
-                       save->min = mast->r->min;
-                       save->max = mast->r->max;
-                       save->offset -= (split + 1);
-               } else {
-                       save->node = mast->l->node;
-                       save->min = mast->l->min;
-                       save->max = mast->l->max;
-               }
-       }
+       if (mte_is_leaf(mas->node))
+           return;
+
+       p_slot = mast->orig_l->offset;
+       mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node,
+                            &p_slot, split);
+       mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node,
+                            &p_slot, split);
 }
 
 /*
@@ -3198,8 +3231,7 @@ static inline void mast_split_data(struct maple_subtree_state *mast,
  * Return: True if pushed, false otherwise.
  */
 static inline bool mas_push_data(struct ma_state *mas, int height,
-                                struct maple_subtree_state *mast, bool left,
-                                struct ma_state *save)
+                                struct maple_subtree_state *mast, bool left)
 {
        unsigned char slot_total = mast->bn->b_end;
        unsigned char end, space, split;
@@ -3232,8 +3264,6 @@ static inline bool mas_push_data(struct ma_state *mas, int height,
                mab_shift_right(mast->bn, end + 1);
                mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0);
                mast->bn->b_end = slot_total + 1;
-               if (!save->node)
-                       save->offset = mas->offset + end + 1;
        } else {
                mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end);
        }
@@ -3258,7 +3288,7 @@ static inline bool mas_push_data(struct ma_state *mas, int height,
        if (left)
                mast->orig_l->offset += end + 1;
 
-       mast_split_data(mast, mas, split, save);
+       mast_split_data(mast, mas, split);
        mast_fill_bnode(mast, mas, 2);
        _mas_split_final_node(mast, mas, height + 1);
        return true;
@@ -3276,7 +3306,6 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
        struct maple_subtree_state mast;
        int height = 0;
        unsigned char mid_split, split = 0;
-       struct ma_state restore;
 
        MA_STATE(l_mas, mas->tree, mas->index, mas->last);
        MA_STATE(r_mas, mas->tree, mas->index, mas->last);
@@ -3297,8 +3326,6 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
        mast.orig_r = &prev_r_mas;
        mast.free = &mat;
        mast.bn = b_node;
-       restore.node = NULL;
-       restore.offset = mas->offset;
 
        while (height++ <= mas->depth) {
                if (mas_split_final_node(&mast, mas, height))
@@ -3308,15 +3335,15 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
                l_mas.node = mas_new_ma_node(mas, b_node);
                r_mas.node = mas_new_ma_node(mas, b_node);
                /* Try to push left. */
-               if (mas_push_data(mas, height, &mast, true, &restore))
+               if (mas_push_data(mas, height, &mast, true))
                        break;
 
                /* Try to push right. */
-               if (mas_push_data(mas, height, &mast, false, &restore))
+               if (mas_push_data(mas, height, &mast, false))
                        break;
 
                split = mab_calc_split(mas, b_node, &mid_split);
-               mast_split_data(&mast, mas, split, &restore);
+               mast_split_data(&mast, mas, split);
                /*
                 * Usually correct, mab_mas_cp in the above call overwrites
                 * r->max.
@@ -3331,10 +3358,7 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
        mat_add(mast.free, mas->node);
        mas->node = l_mas.node;
        mas_wmb_replace(mas, mast.free, NULL);
-       mas->offset = restore.offset;
-       mas->min = restore.min;
-       mas->max = restore.max;
-       mas->node = restore.node;
+       mtree_range_walk(mas);
        return 1;
 }
 
@@ -3626,73 +3650,6 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas,
        }
 }
 
-static inline void *mtree_range_walk(struct ma_state *mas)
-{
-       unsigned long *pivots;
-       unsigned char offset;
-       struct maple_node *node;
-       struct maple_enode *next, *last;
-       enum maple_type type;
-       void __rcu **slots;
-       unsigned char end;
-       unsigned long max, min;
-       unsigned long prev_max, prev_min;
-
-       last = next = mas->node;
-       prev_min = min = mas->min;
-       max = mas->max;
-       do {
-               offset = 0;
-               last = next;
-               node = mte_to_node(next);
-               type = mte_node_type(next);
-               pivots = ma_pivots(node, type);
-               if (unlikely(ma_dead_node(node)))
-                       goto dead_node;
-
-               end = ma_data_end(node, type, pivots, max);
-               if (pivots[offset] >= mas->index) {
-                       prev_max = max;
-                       prev_min = min;
-                       max = pivots[offset];
-                       goto next;
-               }
-
-               do {
-                       offset++;
-               } while((offset < end) && (pivots[offset] < mas->index));
-
-               prev_min = min;
-               prev_max = max;
-               if (offset < mt_pivots[type] && pivots[offset])
-                       max = pivots[offset];
-
-               if (offset)
-                       min = pivots[offset - 1] + 1;
-
-               if (likely(offset > end) && pivots[offset]) {
-next:
-                       max = pivots[offset];
-               }
-
-               slots = ma_slots(node, type);
-               next = mt_slot(mas->tree, slots, offset);
-               if (unlikely(ma_dead_node(node)))
-                       goto dead_node;
-       } while (!ma_is_leaf(type));
-
-       mas->offset = offset;
-       mas->index = min;
-       mas->last = max;
-       mas->min = prev_min;
-       mas->max = prev_max;
-       mas->node = last;
-       return (void *) next;
-
-dead_node:
-       mas_reset(mas);
-       return NULL;
-}
 
 static inline void *mas_state_walk(struct ma_state *mas)
 {