maple_tree: Add mas_wr_node_walk() for writers

Writers don't need to check for dead nodes.

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 3cf1d277567765937e35b107b980ecb56bd42247..eff82d9854841dd4a1dbea22fafb07d106a0a939 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -2073,10 +2073,6 @@ static inline unsigned char mas_store_b_node(struct ma_state *mas,
        return b_end;
 }
 
-static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node,
-               enum maple_type type, unsigned long *range_min,
-               unsigned long *range_max);
-
 /*
  * mas_prev_sibling() - Find the previous node with the same parent.
  * @mas: the maple state
@@ -2139,6 +2135,70 @@ static inline struct maple_enode *mte_node_or_none(struct maple_enode *enode)
        return ma_enode_ptr(MAS_NONE);
 }
 
+/*
+ * mas_wr_node_walk() - Find the correct offset for the index in the @mas.
+ * @mas: The maple state
+ * @type: The node type in the @mas
+ * @range_min: The minimum range
+ * @range_max: The maximum range
+ *
+ * Uses mas_slot_locked() and does not need to worry about dead nodes.
+ *
+ */
+static inline void mas_wr_node_walk(struct ma_state *mas, enum maple_type type,
+                       unsigned long *range_min, unsigned long *range_max)
+{
+       struct maple_node *node;
+       unsigned long *pivots;
+       unsigned long min;
+       unsigned long max;
+       unsigned long index;
+       unsigned char count;
+       unsigned char offset;
+
+       if (unlikely(ma_is_dense(type))) {
+               (*range_max) = (*range_min) = mas->index;
+               mas->offset = mas->index = mas->min;
+               return;
+       }
+
+       node = mas_mn(mas);
+       count = mt_pivots[type];
+       offset = mas->offset;
+       pivots = ma_pivots(node, type);
+       min = mas_safe_min(mas, pivots, offset);
+       max = pivots[offset];
+       if (unlikely(offset == count))
+               goto max;
+
+       index = mas->index;
+       if (unlikely(index <= max))
+               goto done;
+
+       if (unlikely(!max && offset))
+               goto max;
+
+       offset++;
+       min = max + 1;
+       while (offset < count) {
+               max = pivots[offset];
+               if (index <= max)
+                       goto done;
+
+               if (unlikely(!max))
+                       break;
+
+               min = max + 1;
+               offset++;
+       }
+
+max:
+       max = mas->max;
+done:
+       *range_max = max;
+       *range_min = min;
+       mas->offset = offset;
+}
 /*
  * mast_topiary() - Add the portions of the tree to the removal list; either to
  * be freed or discarded (destroy walk).
@@ -2156,8 +2216,7 @@ static inline void mast_topiary(struct maple_subtree_state *mast)
        l_index = mast->orig_l->index;
        mast->orig_l->index = mast->orig_l->last;
        mt = mte_node_type(mast->orig_l->node);
-       mas_node_walk(mast->orig_l, mas_mn(mast->orig_l),
-                     mt, &range_min, &range_max);
+       mas_wr_node_walk(mast->orig_l, mt, &range_min, &range_max);
        mast->orig_l->index = l_index;
        l_off = mast->orig_l->offset;
        r_off = mast->orig_r->offset;
@@ -2331,13 +2390,13 @@ mast_ascend_free(struct maple_subtree_state *mast)
        if (mast->orig_r->max < mast->orig_r->last)
                mast->orig_r->offset = mas_data_end(mast->orig_r) + 1;
        else
-               mas_node_walk(mast->orig_r, mas_mn(mast->orig_r),
+               mas_wr_node_walk(mast->orig_r,
                  mte_node_type(mast->orig_r->node), &range_min, &range_max);
        /* Set up the left side of things */
        mast->orig_l->offset = 0;
        mast->orig_l->index = mast->l->min;
-       mas_node_walk(mast->orig_l, mas_mn(mast->orig_l),
-               mte_node_type(mast->orig_l->node), &range_min, &range_max);
+       mas_wr_node_walk(mast->orig_l, mte_node_type(mast->orig_l->node),
+                        &range_min, &range_max);
 }
 
 /*
@@ -3499,78 +3558,6 @@ static bool mas_is_span_wr(struct ma_state *mas, unsigned long piv,
        return true;
 }
 
-/*
- * mas_node_walk() - Walk a maple node to offset of the index.
- * @mas: The maple state
- * @type: The maple node type
- * @*range_min: Pointer to store the minimum range of the offset
- * @*range_max: Pointer to store the maximum range of the offset
- *
- * The offset will be stored in the maple state.
- *
- */
-static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node,
-                        enum maple_type type, unsigned long *range_min,
-                        unsigned long *range_max)
-{
-       unsigned long *pivots;
-       unsigned char count;
-       unsigned long min, max;
-       unsigned char offset;
-       unsigned long index;
-
-       pivots = ma_pivots(node, type);
-       if (unlikely(ma_is_dense(type))) {
-               (*range_max) = (*range_min) = mas->index;
-               if (unlikely(ma_dead_node(node)))
-                       return;
-
-               mas->offset = mas->index = mas->min;
-               return;
-       }
-
-       offset = mas->offset;
-       min = mas_safe_min(mas, pivots, offset);
-       max = pivots[offset];
-       if (unlikely(ma_dead_node(node)))
-               return;
-
-       count = mt_pivots[type];
-       if (unlikely(offset == count))
-               goto max;
-
-       index = mas->index;
-       if (unlikely(index <= max))
-               goto done;
-
-       if (unlikely(!max && offset))
-               goto max;
-
-       offset++;
-       min = max + 1;
-       while (offset < count) {
-               max = pivots[offset];
-               if (unlikely(ma_dead_node(node)))
-                       return;
-
-               if (index <= max)
-                       goto done;
-
-               if (unlikely(!max))
-                       break;
-
-               min = max + 1;
-               offset++;
-       }
-
-max:
-       max = mas->max;
-done:
-       *range_max = max;
-       *range_min = min;
-       mas->offset = offset;
-}
-
 /*
  * mas_wr_walk(): Walk the tree for a write.
  * @range_min - pointer that will be set to the minimum of the slot range
@@ -3584,15 +3571,13 @@ done:
 static bool mas_wr_walk(struct ma_state *mas, unsigned long *range_min,
                        unsigned long *range_max, void *entry)
 {
-       struct maple_node *node;
        enum maple_type type;
 
        while (true) {
                type = mte_node_type(mas->node);
                mas->depth++;
 
-               node = mas_mn(mas);
-               mas_node_walk(mas, node, type, range_min, range_max);
+               mas_wr_node_walk(mas, type, range_min, range_max);
                if (mas_is_span_wr(mas, *range_max, type, entry))
                        return false;
 
@@ -3663,6 +3648,78 @@ static inline void mas_extend_null(struct ma_state *l_mas, struct ma_state *r_ma
                r_mas->offset = cp_r_slot;
 }
 
+/*
+ * mas_node_walk() - Walk a maple node to offset of the index.
+ * @mas: The maple state
+ * @type: The maple node type
+ * @*range_min: Pointer to store the minimum range of the offset
+ * @*range_max: Pointer to store the maximum range of the offset
+ *
+ * The offset will be stored in the maple state.
+ *
+ */
+static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node,
+                        enum maple_type type, unsigned long *range_min,
+                        unsigned long *range_max)
+
+{
+       unsigned long *pivots;
+       unsigned char count;
+       unsigned long min, max;
+       unsigned char offset;
+       unsigned long index;
+
+       pivots = ma_pivots(node, type);
+       if (unlikely(ma_is_dense(type))) {
+               (*range_max) = (*range_min) = mas->index;
+               if (unlikely(ma_dead_node(node)))
+                       return;
+
+               mas->offset = mas->index = mas->min;
+               return;
+       }
+
+       offset = mas->offset;
+       min = mas_safe_min(mas, pivots, offset);
+       max = pivots[offset];
+       if (unlikely(ma_dead_node(node)))
+               return;
+
+       count = mt_pivots[type];
+       if (unlikely(offset == count))
+               goto max;
+
+       index = mas->index;
+       if (unlikely(index <= max))
+               goto done;
+
+       if (unlikely(!max && offset))
+               goto max;
+
+       offset++;
+       min = max + 1;
+       while (offset < count) {
+               max = pivots[offset];
+               if (unlikely(ma_dead_node(node)))
+                       return;
+
+               if (index <= max)
+                       goto done;
+
+               if (unlikely(!max))
+                       break;
+
+               min = max + 1;
+               offset++;
+       }
+
+max:
+       max = mas->max;
+done:
+       *range_max = max;
+       *range_min = min;
+       mas->offset = offset;
+}
 /*
  * __mas_walk(): Locates a value and sets the mas->node and slot accordingly.
  * range_min and range_max are set to the range which the entry is valid.