maple_tree: mas_store cleanup part many.

node_store cleanup of end calc
mas store offset_end consistent when needed.
kick to slowpath sooner if possible.
mas_node, pass offset_end through.

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index b3cb945d74ca05f5801455c8a19e1198e1c28878..6083438538b2254395c83b7fb74b6e13a0448caa 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -3805,17 +3805,21 @@ static inline bool mas_node_store(struct ma_state *mas, void *entry,
                                  unsigned long min, unsigned long max,
                                  unsigned char end, void *content,
                                  enum maple_type mt, void __rcu **slots,
-                                 unsigned long *pivots)
+                                 unsigned long *pivots, unsigned char offset_end)
 {
        void __rcu **dst_slots;
        unsigned long *dst_pivots;
-       unsigned char dst_offset, new_end = end;
-       unsigned char offset, offset_end;
+       unsigned char dst_offset;
+       unsigned char new_end = end;
+       unsigned char offset;
        struct maple_node reuse, *newnode;
        unsigned char copy_size, max_piv = mt_pivots[mt];
 
-       offset = offset_end = mas->offset;
+       offset = mas->offset;
        if (mas->last == max) {
+               /* runs right to the end of the node */
+               if (mas->last == mas->max)
+                       new_end = offset;
                /* don't copy this offset */
                offset_end++;
        } else if (mas->last < max) {
@@ -3824,16 +3828,11 @@ static inline bool mas_node_store(struct ma_state *mas, void *entry,
                        mas_bulk_rebalance(mas, end, mt);
 
                new_end++;
-               offset_end = offset;
-       } else if (mas->last == mas->max) {
-               /* runs right to the end of the node */
-               new_end = offset;
-               /* no data beyond this range */
-               offset_end = end + 1;
        } else {
-               while (mas_logical_pivot(mas, pivots, ++offset_end, mt) <=
-                      mas->last)
-                       new_end--;
+               if (_mas_safe_pivot(mas, pivots, offset_end, mt) == mas->last)
+                       offset_end++;
+
+               new_end -= offset_end - offset - 1;
        }
 
        /* new range starts within a range */
@@ -4054,12 +4053,15 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite
        if (!entry) {
                /* Check next slot(s) if we are overwriting the end */
                if ((mas->last == end_piv) && !slots[offset_end + 1]) {
-                       if (offset_end + 1 == end)
+                       offset_end++;
+                       if (offset_end == end)
                                mas->last = mas->max;
                        else
-                               mas->last = pivots[offset_end + 1];
-               } else if ((mas->last > end_piv) && !slots[offset_end])
+                               mas->last = pivots[offset_end];
+               } else if ((mas->last > end_piv) && !slots[offset_end]) {
                        mas->last = end_piv;
+                       offset_end++;
+               }
 
                if (!content) {
                        /* If this one is null, the next and prev are not */
@@ -4122,13 +4124,14 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite
        }
 
        /* slot and node store will not fit, go to the slow path */
-       if (unlikely(mas->offset + 1 >= mt_slots[mt]))
+       if (unlikely(offset_end + 1 >= mt_slots[mt]))
                goto slow_path;
 
        if ((offset_end - mas->offset <= 1) &&
            mas_slot_store(mas, entry, r_min, r_max, end_piv, end, content, mt, slots))
                return content;
-       else if (mas_node_store(mas, entry, r_min, r_max, end, content, mt, slots, pivots))
+       else if (mas_node_store(mas, entry, r_min, r_max, end, content, mt, slots, pivots,
+                               offset_end))
                return content;
 
        if (mas_is_err(mas))