maple_tree: Rework mas_walk() for optimization.

Pass through the node when known and reduce the code overhead of the loops.
There cannot be a on non-leaf nodes thanks to the fix on the right-most
node max fix

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 55c6bc67ddebd951b81acee853e2308b1c24a7b4..3cf1d277567765937e35b107b980ecb56bd42247 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -2073,8 +2073,9 @@ static inline unsigned char mas_store_b_node(struct ma_state *mas,
        return b_end;
 }
 
-static inline void mas_node_walk(struct ma_state *mas, enum maple_type type,
-               unsigned long *range_min, unsigned long *range_max);
+static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node,
+               enum maple_type type, unsigned long *range_min,
+               unsigned long *range_max);
 
 /*
  * mas_prev_sibling() - Find the previous node with the same parent.
@@ -2155,7 +2156,8 @@ static inline void mast_topiary(struct maple_subtree_state *mast)
        l_index = mast->orig_l->index;
        mast->orig_l->index = mast->orig_l->last;
        mt = mte_node_type(mast->orig_l->node);
-       mas_node_walk(mast->orig_l, mt, &range_min, &range_max);
+       mas_node_walk(mast->orig_l, mas_mn(mast->orig_l),
+                     mt, &range_min, &range_max);
        mast->orig_l->index = l_index;
        l_off = mast->orig_l->offset;
        r_off = mast->orig_r->offset;
@@ -2329,13 +2331,13 @@ mast_ascend_free(struct maple_subtree_state *mast)
        if (mast->orig_r->max < mast->orig_r->last)
                mast->orig_r->offset = mas_data_end(mast->orig_r) + 1;
        else
-               mas_node_walk(mast->orig_r, mte_node_type(mast->orig_r->node),
-                             &range_min, &range_max);
+               mas_node_walk(mast->orig_r, mas_mn(mast->orig_r),
+                 mte_node_type(mast->orig_r->node), &range_min, &range_max);
        /* Set up the left side of things */
        mast->orig_l->offset = 0;
        mast->orig_l->index = mast->l->min;
-       mas_node_walk(mast->orig_l, mte_node_type(mast->orig_l->node),
-                     &range_min, &range_max);
+       mas_node_walk(mast->orig_l, mas_mn(mast->orig_l),
+               mte_node_type(mast->orig_l->node), &range_min, &range_max);
 }
 
 /*
@@ -3084,6 +3086,7 @@ static inline void mast_fill_bnode(struct maple_subtree_state *mast,
        if (cp)
                mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1,
                           mast->bn, mast->bn->b_end);
+
        mast->bn->b_end--;
        mast->bn->type = mte_node_type(mas->node);
 
@@ -3506,15 +3509,16 @@ static bool mas_is_span_wr(struct ma_state *mas, unsigned long piv,
  * The offset will be stored in the maple state.
  *
  */
-static inline void mas_node_walk(struct ma_state *mas, enum maple_type type,
-               unsigned long *range_min, unsigned long *range_max)
+static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node,
+                        enum maple_type type, unsigned long *range_min,
+                        unsigned long *range_max)
 {
-       struct maple_node *node;
        unsigned long *pivots;
-       unsigned char offset, count;
-       unsigned long min, max, index;
+       unsigned char count;
+       unsigned long min, max;
+       unsigned char offset;
+       unsigned long index;
 
-       node = mas_mn(mas);
        pivots = ma_pivots(node, type);
        if (unlikely(ma_is_dense(type))) {
                (*range_max) = (*range_min) = mas->index;
@@ -3527,15 +3531,15 @@ static inline void mas_node_walk(struct ma_state *mas, enum maple_type type,
 
        offset = mas->offset;
        min = mas_safe_min(mas, pivots, offset);
-       count = mt_pivots[type];
-       index = mas->index;
        max = pivots[offset];
        if (unlikely(ma_dead_node(node)))
                return;
 
+       count = mt_pivots[type];
        if (unlikely(offset == count))
                goto max;
 
+       index = mas->index;
        if (unlikely(index <= max))
                goto done;
 
@@ -3580,13 +3584,15 @@ done:
 static bool mas_wr_walk(struct ma_state *mas, unsigned long *range_min,
                        unsigned long *range_max, void *entry)
 {
+       struct maple_node *node;
        enum maple_type type;
 
        while (true) {
                type = mte_node_type(mas->node);
                mas->depth++;
 
-               mas_node_walk(mas, type, range_min, range_max);
+               node = mas_mn(mas);
+               mas_node_walk(mas, node, type, range_min, range_max);
                if (mas_is_span_wr(mas, *range_max, type, entry))
                        return false;
 
@@ -3671,31 +3677,25 @@ static inline void mas_extend_null(struct ma_state *l_mas, struct ma_state *r_ma
 static inline bool __mas_walk(struct ma_state *mas, unsigned long *range_min,
                unsigned long *range_max)
 {
-       struct maple_node *node;
        struct maple_enode *next;
+       struct maple_node *node;
        enum maple_type type;
 
-       node = mas_mn(mas);
-       if (unlikely(ma_dead_node(node)))
-               return false;
-
+       next = mas->node;
        while (true) {
-               type = mte_node_type(mas->node);
                mas->depth++;
-               mas_node_walk(mas, type, range_min, range_max);
-               next = mas_slot(mas, ma_slots(mas_mn(mas), type), mas->offset);
+               node = mte_to_node(next);
+               type = mte_node_type(next);
+               mas_node_walk(mas, node, type, range_min, range_max);
+               next = mas_slot(mas, ma_slots(node, type), mas->offset);
                if (unlikely(ma_dead_node(node)))
                        return false;
 
                if (unlikely(ma_is_leaf(type)))
                        return true;
 
-               if (unlikely(!next))
-                       return false;
-
                /* Descend. */
                mas->node = next;
-               node = mas_mn(mas);
                mas->max = *range_max;
                mas->min = *range_min;
                mas->offset = 0;
@@ -4392,8 +4392,8 @@ no_entry:
 static inline void *mas_next_nentry(struct ma_state *mas,
            struct maple_node *node, unsigned long max, enum maple_type type)
 {
-       unsigned long pivot = 0;
        unsigned char count;
+       unsigned long pivot;
        unsigned long *pivots;
        void __rcu **slots;
        void *entry;
@@ -4404,8 +4404,8 @@ static inline void *mas_next_nentry(struct ma_state *mas,
        }
 
        pivots = ma_pivots(node, type);
-       mas->index = mas_safe_min(mas, pivots, mas->offset);
        slots = ma_slots(node, type);
+       mas->index = mas_safe_min(mas, pivots, mas->offset);
        if (ma_dead_node(node))
                return NULL;
 
@@ -5873,7 +5873,7 @@ EXPORT_SYMBOL_GPL(mas_pause);
  */
 void *mas_find(struct ma_state *mas, unsigned long max)
 {
-       if (mas_is_paused(mas)) {
+       if (unlikely(mas_is_paused(mas))) {
                if (unlikely(mas->last == ULONG_MAX)) {
                        mas->node = MAS_NONE;
                        return NULL;