maple_tree: Start using ma_wr_state for spanning_store

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index a77208d3c6f536ef602220f0a293f817f26f806c..e5814f6ad967d68be31836251927f2b20aeae660 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -3522,8 +3522,13 @@ static bool mas_is_span_wr(struct ma_wr_state *wr_mas)
                 */
                if ((last == ULONG_MAX) && (last == max))
                        return false;
-       } else if ((piv == last) && entry) {
-               return false;
+       } else if (piv == last) {
+               if (entry)
+                       return false;
+
+               /* Detect spanning store wr walk */
+               if ((last == ULONG_MAX) && (wr_mas->mas->index == ULONG_MAX))
+                       return false;
        }
 
        trace_ma_write(__func__, wr_mas->mas, piv, entry);
@@ -3572,14 +3577,14 @@ static bool mas_wr_walk(struct ma_wr_state *wr_mas)
  */
 static inline void mas_extend_spanning_null(
                struct ma_state *l_mas, unsigned long r_min, void *content,
-               struct ma_state *r_mas, unsigned long r_max, void *r_content)
+               struct ma_wr_state *r_wr_mas)
 {
        unsigned char l_slot = l_mas->offset;
-       unsigned char r_slot = r_mas->offset;
-       unsigned char cp_r_slot = r_slot;
        struct maple_node *l_node = mas_mn(l_mas);
        enum maple_type l_type = mte_node_type(l_mas->node);
        void __rcu **slots = ma_slots(l_node, l_type);
+       struct ma_state *r_mas = r_wr_mas->mas;
+
 
        /* Expand NULL to start of the range. */
        if (!content)
@@ -3596,26 +3601,16 @@ static inline void mas_extend_spanning_null(
                l_mas->offset = l_slot - 1;
        }
 
-       if (!r_content) {
-               if (r_mas->last < r_max)
-                       r_mas->last = r_max;
-               cp_r_slot++;
-       }
-
-       slots = ma_slots(mte_to_node(r_mas->node),
-                            mte_node_type(r_mas->node));
-       if ((r_mas->last == r_max) &&
+       if (!r_wr_mas->content) {
+               if (r_mas->last < r_wr_mas->r_max)
+                       r_mas->last = r_wr_mas->r_max;
+               r_mas->offset++;
+       } else if ((r_mas->last == r_wr_mas->r_max) &&
            (r_mas->last < r_mas->max) &&
-           !mas_slot_locked(r_mas, slots, r_slot + 1)) {
-               r_mas->last = mas_safe_pivot(r_mas, r_slot + 1);
-               cp_r_slot++;
+           !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) {
+               r_mas->last = mas_safe_pivot(r_mas, r_mas->offset + 1);
+               r_mas->offset++;
        }
-
-       if (r_slot && !r_mas->last)
-               r_mas->last = r_mas->max;
-
-       if (l_mas != r_mas)
-               r_mas->offset = cp_r_slot;
 }
 
 static inline void *mtree_range_walk(struct ma_state *mas)
@@ -3820,15 +3815,17 @@ done:
  *
  * Return: 0 on error, positive on success.
  */
-static inline int mas_spanning_store(struct ma_state *mas, void *entry)
+static inline int mas_spanning_store(struct ma_wr_state *wr_mas, void *entry)
 {
        struct maple_big_node b_node;
        struct maple_subtree_state mast;
-       unsigned char height = mas_mt_height(mas);
-       int node_count = 1 + height * 3;
-       void *content, *r_content;
-       unsigned long r_min, r_max;
+       unsigned char height;
+       int node_count;
+       void *l_content;
+       unsigned long r_min;
        unsigned char l_end;
+       struct ma_state *mas = wr_mas->mas;
+       struct ma_wr_state r_wr_mas;
 
        /* Holds new left and right sub-tree */
        MA_STATE(l_mas, mas->tree, mas->index, mas->index);
@@ -3842,6 +3839,8 @@ static inline int mas_spanning_store(struct ma_state *mas, void *entry)
         * Node rebalancing may occur due to this store, so there may be two new
         * entries per level plus a new root.
         */
+       height = mas_mt_height(mas);
+       node_count = 1 + height * 3;
        mas_node_count(mas, node_count);
        if (mas_is_err(mas))
                return 0;
@@ -3858,20 +3857,19 @@ static inline int mas_spanning_store(struct ma_state *mas, void *entry)
                r_mas.last++;
 
        r_mas.index = r_mas.last;
-       r_content = mtree_range_walk(&r_mas);
-       r_max = r_mas.last;
+       r_wr_mas.mas = &r_mas;
+       mas_wr_walk(&r_wr_mas);
        r_mas.last = r_mas.index = mas->last;
 
        /* Set up left side. */
        l_mas = *mas;
-       content = mtree_range_walk(&l_mas);
+       l_content = mtree_range_walk(&l_mas);
        r_min = l_mas.index;
        l_mas.index = mas->index;
        l_mas.last = mas->last;
 
        if (!entry) {
-               mas_extend_spanning_null(&l_mas, r_min, content,
-                                        &r_mas, r_max, r_content);
+               mas_extend_spanning_null(&l_mas, r_min, l_content, &r_wr_mas);
                mas->index = l_mas.index;
                mas->last = l_mas.last = r_mas.index = r_mas.last;
                mas->offset = l_mas.offset;
@@ -3880,9 +3878,9 @@ static inline int mas_spanning_store(struct ma_state *mas, void *entry)
        /* Copy l_mas and store the value in b_node. */
        l_end = mas_data_end(&l_mas);
        b_node.b_end = mas_store_b_node(&l_mas, &b_node, entry, l_end, l_end,
-                                       content);
+                                       l_content);
        /* Copy r_mas into b_node. */
-       mas_mab_cp(&r_mas, r_mas.offset, mt_slot_count(r_mas.node),
+       mas_mab_cp(&r_mas, r_mas.offset, r_wr_mas.node_end,
                   &b_node, b_node.b_end + 1);
        /* Stop spanning searches by searching for just index. */
        l_mas.index = l_mas.last = mas->index;
@@ -4399,7 +4397,7 @@ static inline void *mas_store_entry(struct ma_state *mas, void *entry)
        wr_mas.mas = mas;
        wr_mas.entry = entry;
        if (!mas_wr_walk(&wr_mas)) {
-               mas_spanning_store(mas, entry);
+               mas_spanning_store(&wr_mas, entry);
                return wr_mas.content;
        }