maple_tree: Continue mas_store optimization

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 2b7937c5c7ef318bf03dd7b59bb7351da0072940..b3cb945d74ca05f5801455c8a19e1198e1c28878 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -3942,7 +3942,7 @@ static inline bool mas_slot_store(struct ma_state *mas, void *entry,
        unsigned char offset = mas->offset;
 
        if ((max > mas->last) && ((min != mas->index) || (offset != end)))
-               goto try_node_store;
+               return false;
 
        if (offset == end - 1)
                lmax = mas->max;
@@ -3951,43 +3951,35 @@ static inline bool mas_slot_store(struct ma_state *mas, void *entry,
 
        /* going to overwrite too many slots. */
        if (lmax < mas->last)
-               goto try_node_store;
+               return false;
 
        if (min == mas->index) {
                /* overwriting two or more ranges with one. */
                if (lmax == mas->last)
-                       goto try_node_store;
+                       return false;
 
                /* Overwriting all of offset and a portion of offset + 1. */
                rcu_assign_pointer(slots[offset], entry);
                pivots[offset] = mas->last;
                goto done;
-       } else {
-               /* Doesn't end on the next range end. */
-               if (lmax != mas->last)
-                       goto try_node_store;
-
-               /* Overwriting a portion of offset and all of offset + 1 */
-               if (offset + 1 < mt_pivots[mt]) {
-                       if (entry || pivots[offset + 1])
-                               pivots[offset + 1] = mas->last;
-               }
-               rcu_assign_pointer(slots[offset + 1], entry);
-               pivots[offset] = mas->index - 1;
-               mas->offset++; /* Keep mas accurate. */
-               goto done;
        }
 
-       return false;
+       /* Doesn't end on the next range end. */
+       if (lmax != mas->last)
+               return false;
+
+       /* Overwriting a portion of offset and all of offset + 1 */
+       if ((offset + 1 < mt_pivots[mt]) && (entry || pivots[offset + 1]))
+                       pivots[offset + 1] = mas->last;
 
+       rcu_assign_pointer(slots[offset + 1], entry);
+       pivots[offset] = mas->index - 1;
+       mas->offset++; /* Keep mas accurate. */
 
 done:
        trace_ma_write(__func__, mas, 0, entry);
        mas_update_gap(mas);
        return true;
-
-try_node_store:
-       return mas_node_store(mas, entry, min, max, end, content, mt, slots, pivots);
 }
 
 /*
@@ -4049,14 +4041,12 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite
        offset_end = mas->offset;
        end_piv = r_max;
        end = mas_data_end(mas);
-
        while ((mas->last > end_piv) && (offset_end < end))
                end_piv = pivots[++offset_end];
 
        if (mas->last > end_piv)
                end_piv = mas->max;
 
-
        // FIXME: Try finding end offset out here and passing it through.
        // Maybe a struct for writes?
        // lmax and offset_end ?
@@ -4086,10 +4076,12 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite
                }
        }
 
+       /* New root for a single pointer */
        if (!mas->index && mas->last == ULONG_MAX) {
                mas_new_root(mas, entry);
                return content;
        }
+       /* Direct replacement */
        if (r_min == mas->index && r_max == mas->last) {
                rcu_assign_pointer(slots[mas->offset], entry);
                if (!!entry ^ !!content)
@@ -4097,7 +4089,7 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite
                return content;
        }
 
-       /* Appending can skip a lot. */
+       /* Attempt to append */
        if (entry && (end < mt_slots[mt] - 1) && (mas->offset == end)) {
                if ((mas->index != r_min) && (mas->last == r_max)) {
                        if (end + 1 < mt_pivots[mt])
@@ -4129,14 +4121,14 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite
                }
        }
 
+       /* slot and node store will not fit, go to the slow path */
        if (unlikely(mas->offset + 1 >= mt_slots[mt]))
                goto slow_path;
 
        if ((offset_end - mas->offset <= 1) &&
            mas_slot_store(mas, entry, r_min, r_max, end_piv, end, content, mt, slots))
                return content;
-       else if ((mas->offset + 1 >= mt_slots[mt]) &&
-                mas_node_store(mas, entry, r_min, r_max, end, content, mt, slots, pivots))
+       else if (mas_node_store(mas, entry, r_min, r_max, end, content, mt, slots, pivots))
                return content;
 
        if (mas_is_err(mas))