maple_tree: Add mtree_range_walk() for quicker searching.

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 2c9712b89e6005f12fe270ead2885f1f47b43136..c1fce0549f611543c6cd9e3537d1e8c8f82cea79 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -3705,6 +3705,72 @@ offset_zero:
        }
 }
 
+static inline void *mtree_range_walk(struct ma_state *mas)
+{
+       unsigned long *pivots;
+       unsigned char offset;
+       struct maple_node *node;
+       struct maple_enode *next, *last;
+       enum maple_type type;
+       void __rcu **slots;
+       unsigned char end;
+       unsigned long max, min;
+       unsigned long prev_max, prev_min;
+
+       last = next = mas->node;
+       prev_min = min = 0;
+       max = ULONG_MAX;
+       do {
+               offset = 0;
+               last = next;
+               node = mte_to_node(next);
+               type = mte_node_type(next);
+               pivots = ma_pivots(node, type);
+               if (unlikely(ma_dead_node(node)))
+                       goto dead_node;
+
+               end = ma_data_end(node, type, pivots, max);
+               if (pivots[offset] >= mas->index) {
+                       prev_max = max;
+                       max = pivots[offset];
+                       goto next;
+               }
+
+               do {
+                       offset++;
+               } while((offset < end) && (pivots[offset] < mas->index));
+
+               prev_max = max;
+               prev_min = min;
+
+               if (offset < end)
+                       max = pivots[offset];
+
+               if (offset)
+                       min = pivots[offset - 1] + 1;
+next:
+               if (likely(offset > end) && pivots[offset])
+                       max = pivots[offset];
+
+               slots = ma_slots(node, type);
+               next = mt_slot(mas->tree, slots, offset);
+               if (unlikely(ma_dead_node(node)))
+                       goto dead_node;
+       } while (!ma_is_leaf(type));
+
+       mas->offset = offset;
+       mas->index = min;
+       mas->last = max;
+       mas->min = prev_min;
+       mas->max = prev_max;
+       mas->node = last;
+       return (void *) next;
+
+dead_node:
+       mas_reset(mas);
+       return NULL;
+}
+
 /*
  * mtree_lookup_walk() - Internal quick lookup that does not keep maple state up
  * to date.
@@ -4784,13 +4850,30 @@ not_found:
 
 static inline void mas_rewalk(struct ma_state *mas, unsigned long index)
 {
-       unsigned long range_max, range_min;
 
+retry:
        mas_set(mas, index);
-       mas_tree_walk(mas, &range_min, &range_max);
-       mas->last = range_max;
-       mas->index = range_min;
+       mas_start(mas);
+       if (mas_is_none(mas))
+               goto not_found;
+
+       if (mas_is_ptr(mas)) {
+               if (!mas->index)
+                       return;
+
+               goto not_found;
+       }
+
+       mtree_range_walk(mas);
+       if (mas_is_start(mas))
+               goto retry;
+
+       return;
+
+not_found:
+       mas->offset = MAPLE_NODE_SLOTS;
 }
+
 /*
  * mas_dead_node() - Check if the maple state is pointing to a dead node.
  * @mas: The maple state
@@ -6441,41 +6524,50 @@ EXPORT_SYMBOL(mtree_destroy);
  */
 void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max)
 {
-       unsigned long range_start = 0, range_end = 0;
-       void *entry = NULL;
+       MA_STATE(mas, mt, *index, *index);
+       void *entry;
 #ifdef CONFIG_DEBUG_MAPLE_TREE
        unsigned long copy = *index;
 #endif
 
-       MA_STATE(mas, mt, *index, *index);
+       trace_ma_read(__func__, &mas);
 
        if ((*index) > max)
                return NULL;
 
        rcu_read_lock();
-       if (mas_tree_walk(&mas, &range_start, &range_end)) {
-               if (unlikely(mas_is_ptr(&mas)) && !(*index))
-                       return mas_root(&mas);
-               entry = mas_get_slot(&mas, mas.offset);
-       }
+retry:
+       entry = mas_start(&mas);
+       if (unlikely(mas_is_none(&mas)))
+               goto unlock;
 
-       mas.last = range_end;
-       if (entry && !xa_is_zero(entry)) {
-               rcu_read_unlock();
-               goto done;
+       if (unlikely(mas_is_ptr(&mas))) {
+               if (index)
+                       entry = NULL;
+
+               goto unlock;
        }
 
-       mas.index = range_start;
+       entry = mtree_range_walk(&mas);
+       if (xa_is_zero(entry))
+               entry = NULL;
+
+       if (entry)
+               goto unlock;
+
+       if (!entry && unlikely(mas_is_start(&mas)))
+               goto retry;
+
        while (mas_searchable(&mas) && (mas.index < max)) {
                entry = mas_next_entry(&mas, max);
                if (likely(entry && !xa_is_zero(entry)))
                        break;
        }
-       rcu_read_unlock();
 
        if (unlikely(xa_is_zero(entry)))
                entry = NULL;
-done:
+unlock:
+       rcu_read_unlock();
        if (likely(entry)) {
                *index = mas.last + 1;
 #ifdef CONFIG_DEBUG_MAPLE_TREE
@@ -6485,6 +6577,8 @@ done:
                MT_BUG_ON(mt, (*index) && ((*index) <= copy));
 #endif
        }
+       if (xa_is_zero(entry))
+               return NULL;
 
        return entry;
 }

diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c
index f48de72cd88b475b1d433544f5054b11f2569a40..77de706a7e84a43516f205d1b684391cedab624f 100644 (file)
--- a/lib/test_maple_tree.c
+++ b/lib/test_maple_tree.c
@@ -17,6 +17,7 @@
 /* #define BENCH_NODE_STORE */
 /* #define BENCH_AWALK */
 /* #define BENCH_WALK */
+/* #define BENCH_MT_FOR_EACH */
 /* #define BENCH_FORK */
 static
 int mtree_insert_index(struct maple_tree *mt, unsigned long index, gfp_t gfp)
@@ -35477,7 +35478,7 @@ static noinline void bench_awalk(struct maple_tree *mt)
 #if defined(BENCH_WALK)
 static noinline void bench_walk(struct maple_tree *mt)
 {
-       int i, max = 2500, count = 500000000;
+       int i, max = 2500, count = 550000000;
        MA_STATE(mas, mt, 1470, 1470);
 
        for (i = 0; i < max; i += 10)
@@ -35495,6 +35496,29 @@ static noinline void bench_walk(struct maple_tree *mt)
 }
 #endif
 
+#if defined(BENCH_MT_FOR_EACH)
+static noinline void bench_mt_for_each(struct maple_tree *mt)
+{
+       int i, count = 1000000;
+       unsigned long max = 2500, index = 0;
+       void *entry;
+
+       for (i = 0; i < max; i += 5)
+               mtree_store_range(mt, i, i + 4, xa_mk_value(i), GFP_KERNEL);
+
+       for (i = 0; i < count; i++) {
+               unsigned long j = 0;
+               mt_for_each(mt, entry, index, max) {
+                       MT_BUG_ON(mt, entry != xa_mk_value(j));
+                       j+=5;
+               }
+
+               index = 0;
+       }
+
+}
+#endif
+
 static noinline void check_forking(struct maple_tree *mt)
 {
 
@@ -36776,6 +36800,13 @@ static int maple_tree_seed(void)
        mtree_destroy(&tree);
        goto skip;
 #endif
+#if defined(BENCH_MT_FOR_EACH)
+#define BENCH
+       mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+       bench_mt_for_each(&tree);
+       mtree_destroy(&tree);
+       goto skip;
+#endif
 
        test_kmem_cache_bulk();