maple_tree: Make some common functions faster.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 57a16eb464e93b7728d35f3537e28e8e3298ea3d..e5531bf4dfe255c644dc78e1f5bb2d6a7a2ba3b4 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -1047,13 +1047,13 @@ static inline unsigned char mas_data_end(struct ma_state *mas)
        if (type == maple_arange_64)
                return ma_meta_end(mte_to_node(mas->node), type);
 
-       if (!pivots[offset])
+       if (unlikely(!pivots[offset]))
                goto decrement;
 
        // Higher than the min.
        offset = mt_pivots[type] - 1;
        // Check exceptions outside of the loop.
-       if (pivots[offset]) { // almost full.
+       if (unlikely(pivots[offset])) { // almost full.
                if (pivots[offset] != mas->max) // Totally full.
                        return offset + 1;
                return offset;
@@ -1061,10 +1061,10 @@ static inline unsigned char mas_data_end(struct ma_state *mas)
 
 decrement:
        while (--offset) {
-               if (pivots[offset])
+               if (likely(pivots[offset]))
                        break;
        };
-       if (pivots[offset] < mas->max)
+       if (likely(pivots[offset] < mas->max))
                offset++;
 
        return offset;
@@ -1105,7 +1105,7 @@ static inline unsigned long mas_leaf_max_gap(struct ma_state *mas)
        // Removing the pivot overflow optimizes the loop below.
        // Check the first implied pivot.
        i = 2;
-       if (!slots[0]) {
+       if (likely(!slots[0])) {
                max_gap = pivots[0] - mas->min + 1;
        } else if (!slots[1]) {
                // Checking the first slot remove the !pstart && mas->min check
@@ -1125,7 +1125,7 @@ static inline unsigned long mas_leaf_max_gap(struct ma_state *mas)
        }
 
        for (; i <= max_piv; i++) {
-               if (slots[i]) // data == no gap.
+               if (likely(slots[i])) // data == no gap.
                        continue;
 
                pstart = pivots[i - 1];
@@ -2952,6 +2952,8 @@ static inline void mas_node_walk(struct ma_state *mas, enum maple_type type,
                unsigned long *range_min, unsigned long *range_max)
 {
        unsigned long *pivots = ma_pivots(mas_mn(mas), type);
+       unsigned char offset;
+       unsigned long min, max;
 
        if (unlikely(ma_is_dense(type))) {
                (*range_max) = (*range_min) = mas->index;
@@ -2959,25 +2961,31 @@ static inline void mas_node_walk(struct ma_state *mas, enum maple_type type,
                return;
        }
 
-       (*range_min) = mas_safe_min(mas, pivots, mas->offset);
-       if (unlikely(mas->offset == mt_pivots[type]))
+       offset = mas->offset;
+       min = mas_safe_min(mas, pivots, offset);
+       if (unlikely(offset == mt_pivots[type]))
                goto max;
 
-       while (mas->offset < mt_pivots[type]) {
+       while (offset < mt_pivots[type]) {
 
-               (*range_max) = pivots[mas->offset];
-               if (!(*range_max) && mas->offset)
-                       goto max;
+               max = pivots[offset];
+               if (unlikely(!max && offset)) {
+                       break;
+               }
 
-               if (mas->index <= (*range_max))
-                       return;
+               if (mas->index <= max)
+                       goto done;
 
-               (*range_min) = (*range_max) + 1;
-               mas->offset++;
+               min = max + 1;
+               offset++;
        }
 
 max:
-       *range_max = mas->max;
+       max = mas->max;
+done:
+       *range_max = max;
+       *range_min = min;
+       mas->offset = offset;
 }
 
 /*
@@ -3088,7 +3096,7 @@ static inline bool __mas_walk(struct ma_state *mas, unsigned long *range_min,
                        return true;
 
                next = mas_get_slot(mas, mas->offset);
-               if (!next)
+               if (unlikely(!next))
                        return false;
 
                // Descend.
@@ -4426,7 +4434,7 @@ static inline void *mas_range_load(struct ma_state *mas,
 
 retry:
        if (_mas_walk(mas, range_min, range_max)) {
-               if (mas_is_ptr(mas) && mas->last == 0)
+               if (mas->last == 0 && mas_is_ptr(mas))
                        return mte_safe_root(mas->tree->ma_root);
 
                if (mas->offset >= MAPLE_NODE_SLOTS)
@@ -4458,7 +4466,7 @@ static inline void *_mas_next(struct ma_state *mas, unsigned long limit,
        void *entry = NULL;
 
 
-       if (mas_is_start(mas)) {// First run.
+       if (unlikely(mas_is_start(mas))) {// First run.
                unsigned long range_max;
 
                mas_start(mas);