From: Liam R. Howlett <Liam.Howlett@oracle.com>
Date: Thu, 18 Nov 2021 04:04:59 +0000 (-0500)
Subject: maple_tree: Move older walks to test code, implement with newer walks
X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=bacef98ed0e567732d1c1db1f44779996545d570;p=users%2Fjedix%2Flinux-maple.git

maple_tree: Move older walks to test code, implement with newer walks

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
---

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index c1fce0549f61..48b12e2a16b8 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -2174,7 +2174,7 @@ static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas)
 	min = mas_safe_min(mas, wr_mas->pivots, offset);
 	max = wr_mas->pivots[offset];
 	if (unlikely(offset == count))
-		goto max;
+		goto max; /* may have been set to zero */
 
 	index = mas->index;
 	if (unlikely(index <= max))
@@ -3585,44 +3585,42 @@ static bool mas_wr_walk(struct ma_wr_state *wr_mas)
  * @l_mas: The left maple state
  * @r_mas: The right maple state
  */
-static inline void mas_extend_spanning_null(struct ma_state *l_mas,
-					    struct ma_state *r_mas)
+static inline void mas_extend_spanning_null(
+		struct ma_state *l_mas, unsigned long r_min, void *content,
+		struct ma_state *r_mas, unsigned long r_max, void *r_content)
 {
 	unsigned char l_slot = l_mas->offset;
 	unsigned char r_slot = r_mas->offset;
 	unsigned char cp_r_slot = r_slot;
-	unsigned long range_max = mas_safe_pivot(r_mas, r_slot);
-	unsigned long range_min = l_mas->min;
-	void __rcu **slots = ma_slots(mte_to_node(l_mas->node),
-				    mte_node_type(l_mas->node));
-	void *content = mas_slot_locked(l_mas, slots, l_slot);
-
-	if (l_slot)
-		range_min = mas_safe_pivot(l_mas, l_slot - 1) + 1;
+	struct maple_node *l_node = mas_mn(l_mas);
+	enum maple_type l_type = mte_node_type(l_mas->node);
+	void __rcu **slots = ma_slots(l_node, l_type);
 
 	/* Expand NULL to start of the range. */
 	if (!content)
-		l_mas->index = range_min;
+		l_mas->index = r_min;
 
-	if ((l_mas->index == range_min) &&
-	    l_slot && !slots[l_slot - 1]) {
-		if (l_slot > 1)
-			l_mas->index = mas_safe_pivot(l_mas, l_slot - 2) + 1;
-		else
+	if ((l_mas->index == r_min) &&
+	    (l_slot && !slots[l_slot - 1])) {
+		if (l_slot > 1) {
+			unsigned long *pivots = ma_pivots(l_node, l_type);
+
+			l_mas->index = pivots[l_slot - 2] + 1;
+		} else
 			l_mas->index = l_mas->min;
 		l_mas->offset = l_slot - 1;
 	}
 
-	slots = ma_slots(mte_to_node(r_mas->node),
-			     mte_node_type(r_mas->node));
-	if (!mas_slot_locked(r_mas, slots, r_slot)) {
-		if (r_mas->last < range_max)
-			r_mas->last = range_max;
+	if (!r_content) {
+		if (r_mas->last < r_max)
+			r_mas->last = r_max;
 		cp_r_slot++;
 	}
 
-	if (r_mas->last == range_max &&
-	    r_mas->last < r_mas->max &&
+	slots = ma_slots(mte_to_node(r_mas->node),
+			     mte_node_type(r_mas->node));
+	if ((r_mas->last == r_max) &&
+	    (r_mas->last < r_mas->max) &&
 	    !mas_slot_locked(r_mas, slots, r_slot + 1)) {
 		r_mas->last = mas_safe_pivot(r_mas, r_slot + 1);
 		cp_r_slot++;
@@ -3635,76 +3633,6 @@ static inline void mas_extend_spanning_null(struct ma_state *l_mas,
 		r_mas->offset = cp_r_slot;
 }
 
-/*
- * mas_node_walk() - Walk a maple node to offset of the index.
- * @mas: The maple state
- * @type: The maple node type
- * @*range_min: Pointer to store the minimum range of the offset
- * @*range_max: Pointer to store the maximum range of the offset
- *
- * The offset will be stored in the maple state.
- *
- */
-static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node,
-			 enum maple_type type, unsigned long *range_min,
-			 unsigned long *range_max)
-
-{
-	unsigned long *pivots;
-	unsigned char count;
-	unsigned long prev, max;
-	unsigned char offset;
-	unsigned long index;
-
-	if (unlikely(ma_is_dense(type))) {
-		(*range_max) = (*range_min) = mas->index;
-		if (unlikely(ma_dead_node(node)))
-			return;
-
-		mas->offset = mas->index = mas->min;
-		return;
-	}
-
-	pivots = ma_pivots(node, type);
-	max = pivots[0];
-	if (unlikely(ma_dead_node(node)))
-		return;
-
-	offset = 0;
-	prev = mas->min;
-	index = mas->index;
-	if (unlikely(index <= max))
-		goto offset_zero;
-
-	count = mt_pivots[type];
-	while (++offset < count) {
-		prev = max;
-		max = pivots[offset];
-		if (unlikely(ma_dead_node(node)))
-			return;
-
-		if (index <= max)
-			goto offset_found;
-		else if (unlikely(!max))
-			goto mas_max;
-	}
-
-	prev = max;
-mas_max:
-	max = mas->max;
-offset_found:
-	prev++;
-offset_zero:
-	mas->offset = offset;
-	if (ma_is_leaf(type)) {
-		*range_max = max;
-		*range_min = prev;
-	} else {
-		mas->max = max;
-		mas->min = prev;
-	}
-}
-
 static inline void *mtree_range_walk(struct ma_state *mas)
 {
 	unsigned long *pivots;
@@ -3718,8 +3646,8 @@ static inline void *mtree_range_walk(struct ma_state *mas)
 	unsigned long prev_max, prev_min;
 
 	last = next = mas->node;
-	prev_min = min = 0;
-	max = ULONG_MAX;
+	prev_min = min = mas->min;
+	max = mas->max;
 	do {
 		offset = 0;
 		last = next;
@@ -3732,6 +3660,7 @@ static inline void *mtree_range_walk(struct ma_state *mas)
 		end = ma_data_end(node, type, pivots, max);
 		if (pivots[offset] >= mas->index) {
 			prev_max = max;
+			prev_min = min;
 			max = pivots[offset];
 			goto next;
 		}
@@ -3740,17 +3669,18 @@ static inline void *mtree_range_walk(struct ma_state *mas)
 			offset++;
 		} while((offset < end) && (pivots[offset] < mas->index));
 
-		prev_max = max;
 		prev_min = min;
-
-		if (offset < end)
+		prev_max = max;
+		if (offset < mt_pivots[type] && pivots[offset])
 			max = pivots[offset];
 
 		if (offset)
 			min = pivots[offset - 1] + 1;
+
+		if (likely(offset > end) && pivots[offset]) {
 next:
-		if (likely(offset > end) && pivots[offset])
 			max = pivots[offset];
+		}
 
 		slots = ma_slots(node, type);
 		next = mt_slot(mas->tree, slots, offset);
@@ -3771,6 +3701,27 @@ dead_node:
 	return NULL;
 }
 
+static inline void *mas_state_walk(struct ma_state *mas)
+{
+	void *entry;
+
+	entry = mas_start(mas);
+	if (mas_is_none(mas))
+		goto not_found;
+
+	if (mas_is_ptr(mas)) {
+		if (!mas->index)
+			return entry;
+
+		goto not_found;
+	}
+
+	return mtree_range_walk(mas);
+
+not_found:
+	mas->offset = MAPLE_NODE_SLOTS;
+	return NULL;
+}
 /*
  * mtree_lookup_walk() - Internal quick lookup that does not keep maple state up
  * to date.
@@ -3826,43 +3777,6 @@ dead_node:
 	return NULL;
 }
 
-/*
- * mas_descend_walk(): Locates a value and sets the mas->node and slot
- * accordingly.  range_min and range_max are set to the range which the entry is
- * valid.
- * @mas: The maple state
- * @*range_min: A pointer to store the minimum of the range
- * @*range_max: A pointer to store the maximum of the range
- *
- * Check mas->node is still valid on return of any value.
- *
- * Return: true if pointing to a valid node and offset.  False otherwise.
- */
-static inline bool mas_descend_walk(struct ma_state *mas,
-			unsigned long *range_min, unsigned long *range_max)
-{
-	struct maple_enode *next;
-	struct maple_node *node;
-	enum maple_type type;
-
-	next = mas->node;
-	while (true) {
-		node = mte_to_node(next);
-		type = mte_node_type(next);
-		mas_node_walk(mas, node, type, range_min, range_max);
-		next = mas_slot(mas, ma_slots(node, type), mas->offset);
-		if (unlikely(ma_dead_node(node)))
-			return false;
-
-		if (unlikely(ma_is_leaf(type)))
-			return true;
-
-		/* Descend. */
-		mas->node = next;
-	}
-	return false;
-}
-
 /*
  * mas_new_root() - Create a new root node that only contains the entry passed
  * in.
@@ -3923,12 +3837,12 @@ done:
  */
 static inline int mas_spanning_store(struct ma_state *mas, void *entry)
 {
-	unsigned long range_min, range_max;
 	struct maple_big_node b_node;
 	struct maple_subtree_state mast;
 	unsigned char height = mas_mt_height(mas);
 	int node_count = 1 + height * 3;
-	void *content;
+	void *content, *r_content;
+	unsigned long r_min, r_max;
 
 	/* Holds new left and right sub-tree */
 	MA_STATE(l_mas, mas->tree, mas->index, mas->index);
@@ -3958,24 +3872,25 @@ static inline int mas_spanning_store(struct ma_state *mas, void *entry)
 		r_mas.last++;
 
 	r_mas.index = r_mas.last;
-	r_mas.offset = 0;
-	mas_descend_walk(&r_mas, &range_min, &range_max);
+	r_content = mtree_range_walk(&r_mas);
+	r_max = r_mas.last;
 	r_mas.last = r_mas.index = mas->last;
 
 	/* Set up left side. */
 	l_mas = *mas;
-	l_mas.offset = 0;
-	mas_descend_walk(&l_mas, &range_min, &range_max);
-	content = mas_get_slot(&l_mas, l_mas.offset);
+	content = mtree_range_walk(&l_mas);
+	r_min = l_mas.index;
+	l_mas.index = mas->index;
+	l_mas.last = mas->last;
 
 	if (!entry) {
-		mas_extend_spanning_null(&l_mas, &r_mas);
+		mas_extend_spanning_null(&l_mas, r_min, content,
+					 &r_mas, r_max, r_content);
 		mas->index = l_mas.index;
 		mas->last = l_mas.last = r_mas.index = r_mas.last;
 		mas->offset = l_mas.offset;
 	}
 
-
 	/* Copy l_mas and store the value in b_node. */
 	b_node.b_end = mas_store_b_node(&l_mas, &b_node, entry,
 					mas_data_end(&l_mas), mas_data_end(&l_mas),
@@ -4806,72 +4721,17 @@ found:
 	return entry;
 }
 
-/*
- * mas_tree_walk() - Walk to @mas->index and set the range values.
- * @mas: The maple state.
- * @*range_min: The minimum range to be set.
- * @*range_max: The maximum range to be set.
- *
- * Ranges are only valid if there is a valid entry at @mas->index.
- *
- * Return: True if a value exists, false otherwise.
- */
-static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min,
-				 unsigned long *range_max)
-{
-	bool ret;
-
-retry:
-	ret = false;
-	mas_start(mas);
-	if (mas_is_none(mas))
-		goto not_found;
-
-	if (mas_is_ptr(mas)) {
-		*range_min = *range_max = 0;
-		if (!mas->index)
-			return true;
-
-		goto not_found;
-	}
-
-	ret = mas_descend_walk(mas, range_min, range_max);
-	if (unlikely(mte_dead_node(mas->node))) {
-		mas->node = MAS_START;
-		goto retry;
-	}
-
-	return ret;
-
-not_found:
-	mas->offset = MAPLE_NODE_SLOTS;
-	return false;
-}
-
 static inline void mas_rewalk(struct ma_state *mas, unsigned long index)
 {
 
 retry:
 	mas_set(mas, index);
-	mas_start(mas);
-	if (mas_is_none(mas))
-		goto not_found;
-
-	if (mas_is_ptr(mas)) {
-		if (!mas->index)
-			return;
-
-		goto not_found;
-	}
-
-	mtree_range_walk(mas);
+	mas_state_walk(mas);
 	if (mas_is_start(mas))
 		goto retry;
 
 	return;
 
-not_found:
-	mas->offset = MAPLE_NODE_SLOTS;
 }
 
 /*
@@ -5243,41 +5103,6 @@ done:
 	return found;
 }
 
-/*
- * mas_range_load() - Load the entry at an index and get the range.
- * @mas: The maple state
- * @*range_min: Pointer to store the minimum range the entry is valid
- * @*range_max: Pointer to store the maximum range the entry is valid
- *
- * Must hold rcu_read_lock or the write lock.
- * Find where mas->index is located and return the entry.
- * @mas->node will point to the node containing the entry.
- * range_min and range_max will be set accordingly.
- *
- * Return: The entry at mas->index or %NULL
- */
-static inline void *mas_range_load(struct ma_state *mas,
-	   unsigned long *range_min, unsigned long *range_max)
-{
-	void *entry = NULL;
-	unsigned long index = mas->index;
-
-	if (mas_is_none(mas) || mas_is_paused(mas))
-		mas->node = MAS_START;
-retry:
-	if (mas_tree_walk(mas, range_min, range_max))
-		if (unlikely(mas->node == MAS_ROOT))
-			return mas_root(mas);
-
-	if (likely(mas->offset != MAPLE_NODE_SLOTS))
-		entry = mas_get_slot(mas, mas->offset);
-
-	if (mas_dead_node(mas, index))
-		goto retry;
-
-	return entry;
-}
-
 /**
  * mas_walk() - Search for @mas->index in the tree.
  * @mas - the maple state.
@@ -5289,13 +5114,13 @@ retry:
  */
 void *mas_walk(struct ma_state *mas)
 {
-	unsigned long range_min, range_max;
 	void *entry;
 
-	/* Retries on dead nodes handled by mas_range_load */
-	entry = mas_range_load(mas, &range_min, &range_max);
-	mas->index = range_min;
-	mas->last = range_max;
+retry:
+	entry = mas_state_walk(mas);
+	if (mas_is_start(mas))
+		goto retry;
+
 	return entry;
 }
 
@@ -5970,12 +5795,11 @@ void mas_destroy(struct ma_state *mas)
 	 */
 	if (mas->mas_flags & MA_STATE_REBALANCE) {
 		unsigned char end;
-		unsigned long range_min, range_max;
 
 		if (mas_is_start(mas))
 			mas_start(mas);
 
-		mas_descend_walk(mas, &range_min, &range_max);
+		mtree_range_walk(mas);
 		end = mas_data_end(mas) + 1;
 		if (end < mt_min_slot_count(mas->node) - 1)
 			mas_destroy_rebalance(mas, end);
@@ -6200,17 +6024,19 @@ EXPORT_SYMBOL_GPL(mas_find);
  */
 void *mas_erase(struct ma_state *mas)
 {
-	unsigned long r_max, r_min;
 	void *entry = NULL;
 
-	entry = mas_range_load(mas, &r_min, &r_max);
-retry:
-	mas->node = MAS_START;
-	mas->index = r_min;
-	mas->last = r_max;
+	if (mas_is_none(mas) || mas_is_paused(mas))
+		mas->node = MAS_START;
+
+	/* Retry unnecessary when holding the write lock. */
+	entry = mas_state_walk(mas);
+write_retry:
+	/* Must reset to ensure spanning writes of last slot are detected */
+	mas_reset(mas);
 	mas_store_entry(mas, NULL);
 	if (mas_nomem(mas, GFP_KERNEL))
-		goto retry;
+		goto write_retry;
 
 	return entry;
 }
@@ -6537,27 +6363,16 @@ void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max)
 
 	rcu_read_lock();
 retry:
-	entry = mas_start(&mas);
-	if (unlikely(mas_is_none(&mas)))
-		goto unlock;
-
-	if (unlikely(mas_is_ptr(&mas))) {
-		if (index)
-			entry = NULL;
-
-		goto unlock;
-	}
+	entry = mas_state_walk(&mas);
+	if (mas_is_start(&mas))
+		goto retry;
 
-	entry = mtree_range_walk(&mas);
-	if (xa_is_zero(entry))
+	if (unlikely(xa_is_zero(entry)))
 		entry = NULL;
 
 	if (entry)
 		goto unlock;
 
-	if (!entry && unlikely(mas_is_start(&mas)))
-		goto retry;
-
 	while (mas_searchable(&mas) && (mas.index < max)) {
 		entry = mas_next_entry(&mas, max);
 		if (likely(entry && !xa_is_zero(entry)))
@@ -6577,8 +6392,6 @@ unlock:
 		MT_BUG_ON(mt, (*index) && ((*index) <= copy));
 #endif
 	}
-	if (xa_is_zero(entry))
-		return NULL;
 
 	return entry;
 }
@@ -6927,7 +6740,6 @@ void mas_validate_gaps(struct ma_state *mas)
 	if (!mte_is_leaf(mte))
 		gaps = ma_gaps(mte_to_node(mte), mte_node_type(mte));
 
-
 	for (i = 0; i < mt_slot_count(mte); i++) {
 		p_end = mas_logical_pivot(mas, pivots, i, mte_node_type(mte));
 
diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c
index 77de706a7e84..76a5276bc1b1 100644
--- a/lib/test_maple_tree.c
+++ b/lib/test_maple_tree.c
@@ -1242,6 +1242,177 @@ int mas_ce2_over_count(struct ma_state *mas_start, struct ma_state *mas_end,
 	return count;
 }
 
+/*
+ * mas_node_walk() - Walk a maple node to offset of the index.
+ * @mas: The maple state
+ * @type: The maple node type
+ * @*range_min: Pointer to store the minimum range of the offset
+ * @*range_max: Pointer to store the maximum range of the offset
+ *
+ * The offset will be stored in the maple state.
+ *
+ */
+static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node,
+			 enum maple_type type, unsigned long *range_min,
+			 unsigned long *range_max)
+
+{
+	unsigned long *pivots;
+	unsigned char count;
+	unsigned long prev, max;
+	unsigned char offset;
+	unsigned long index;
+
+	if (unlikely(ma_is_dense(type))) {
+		(*range_max) = (*range_min) = mas->index;
+		if (unlikely(ma_dead_node(node)))
+			return;
+
+		mas->offset = mas->index = mas->min;
+		return;
+	}
+
+	pivots = ma_pivots(node, type);
+	max = pivots[0];
+	if (unlikely(ma_dead_node(node)))
+		return;
+
+	offset = 0;
+	prev = mas->min;
+	index = mas->index;
+	if (unlikely(index <= max))
+		goto offset_zero;
+
+	count = mt_pivots[type];
+	while (++offset < count) {
+		prev = max;
+		max = pivots[offset];
+		if (unlikely(ma_dead_node(node)))
+			return;
+
+		if (index <= max)
+			goto offset_found;
+		else if (unlikely(!max))
+			goto mas_max;
+	}
+
+	prev = max;
+mas_max:
+	max = mas->max;
+offset_found:
+	prev++;
+offset_zero:
+	mas->offset = offset;
+	if (ma_is_leaf(type)) {
+		*range_max = max;
+		*range_min = prev;
+	} else {
+		mas->max = max;
+		mas->min = prev;
+	}
+}
+
+/*
+ * mas_descend_walk(): Locates a value and sets the mas->node and slot
+ * accordingly.  range_min and range_max are set to the range which the entry is
+ * valid.
+ * @mas: The maple state
+ * @*range_min: A pointer to store the minimum of the range
+ * @*range_max: A pointer to store the maximum of the range
+ *
+ * Check mas->node is still valid on return of any value.
+ *
+ * Return: true if pointing to a valid node and offset.  False otherwise.
+ */
+static inline bool mas_descend_walk(struct ma_state *mas,
+			unsigned long *range_min, unsigned long *range_max)
+{
+	struct maple_enode *next;
+	struct maple_node *node;
+	enum maple_type type;
+
+	next = mas->node;
+	while (true) {
+		node = mte_to_node(next);
+		type = mte_node_type(next);
+		mas_node_walk(mas, node, type, range_min, range_max);
+		next = mas_slot(mas, ma_slots(node, type), mas->offset);
+		if (unlikely(ma_dead_node(node)))
+			return false;
+
+		if (unlikely(ma_is_leaf(type)))
+			return true;
+
+		/* Descend. */
+		mas->node = next;
+	}
+	return false;
+}
+
+/*
+ * mas_tree_walk() - Walk to @mas->index and set the range values.
+ * @mas: The maple state.
+ * @*range_min: The minimum range to be set.
+ * @*range_max: The maximum range to be set.
+ *
+ * Ranges are only valid if there is a valid entry at @mas->index.
+ *
+ * Return: True if a value exists, false otherwise.
+ */
+static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min,
+				 unsigned long *range_max)
+{
+	bool ret;
+
+retry:
+	ret = false;
+	mas_start(mas);
+	if (mas_is_none(mas))
+		goto not_found;
+
+	if (mas_is_ptr(mas)) {
+		*range_min = *range_max = 0;
+		if (!mas->index)
+			return true;
+
+		goto not_found;
+	}
+
+	ret = mas_descend_walk(mas, range_min, range_max);
+	if (unlikely(mte_dead_node(mas->node))) {
+		mas->node = MAS_START;
+		goto retry;
+	}
+
+	return ret;
+
+not_found:
+	mas->offset = MAPLE_NODE_SLOTS;
+	return false;
+}
+
+static inline void *mas_range_load(struct ma_state *mas,
+	   unsigned long *range_min, unsigned long *range_max)
+
+{
+	void *entry = NULL;
+	unsigned long index = mas->index;
+
+	if (mas_is_none(mas) || mas_is_paused(mas))
+		mas->node = MAS_START;
+retry:
+	if (mas_tree_walk(mas, range_min, range_max))
+		if (unlikely(mas->node == MAS_ROOT))
+			return mas_root(mas);
+
+	if (likely(mas->offset != MAPLE_NODE_SLOTS))
+		entry = mas_get_slot(mas, mas->offset);
+
+	if (mas_dead_node(mas, index))
+		goto retry;
+
+	return entry;
+}
 static noinline void check_erase2_testset(struct maple_tree *mt,
 		unsigned long *set, unsigned long size)
 {