maple_tree: mas_store changes part 2

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 27d61ce2b4f6c420352b9cf30c6256dfd4fd24a8..564d2aed62ea14967a606a577d35db97783e7d62 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -3909,6 +3909,7 @@ done:
  */
 static inline bool mas_slot_store(struct ma_state *mas, void *entry,
                                  unsigned long min, unsigned long max,
+                                 unsigned long end_piv,
                                  unsigned char end, void *content,
                                  enum maple_type mt, void __rcu **slots)
 {
@@ -3917,20 +3918,8 @@ static inline bool mas_slot_store(struct ma_state *mas, void *entry,
        unsigned long lmax; /* Logical max. */
        unsigned char offset = mas->offset;
 
-       if (max > mas->last) {
-               if ((min != mas->index) || (offset != end))
-                       goto try_node_store;
-
-               /* Appending */
-               if (end + 1 < mt_pivots[mt])
-                       pivots[end + 1] = pivots[end];
-
-               rcu_assign_pointer(slots[end + 1], content);
-               pivots[end] = mas->last;
-               rcu_assign_pointer(slots[end], entry);
-               mas_update_gap(mas);
-               return true;
-       }
+       if ((max > mas->last) && ((min != mas->index) || (offset != end)))
+               goto try_node_store;
 
        if (offset == end - 1)
                lmax = mas->max;
@@ -4037,6 +4026,7 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite
        offset_end = mas->offset;
        end_piv = r_max;
        end = mas_data_end(mas);
+
        while ((mas->last > end_piv) && (offset_end < end))
                end_piv = pivots[++offset_end];
 
@@ -4080,15 +4070,39 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite
                return content;
        }
 
+       /* Appending can skip a lot. */
+       if ((end < mt_slots[mt] - 1) && (mas->offset == end)) {
+               if ((mas->index != r_min) && (mas->last == r_max)) {
+                       if (end + 1 < mt_pivots[mt])
+                               pivots[end + 1] = pivots[end];
+
+                       rcu_assign_pointer(slots[end + 1], entry);
+                       pivots[end] = mas->index - 1;
+                       if (!content || !entry)
+                               mas_update_gap(mas);
+                       return content;
+               } else if ((mas->index == r_min) && (mas->last < r_max)) {
+                       if (end + 1 < mt_pivots[mt])
+                               pivots[end + 1] = pivots[end];
+
+                       rcu_assign_pointer(slots[end + 1], content);
+                       pivots[end] = mas->last;
+                       rcu_assign_pointer(slots[end], entry);
+                       if (!content || !entry)
+                               mas_update_gap(mas);
+                       return content;
+               }
+       }
+
        if (unlikely(mas->offset + 1 >= mt_slots[mt]))
                goto slow_path;
 
        if ((offset_end - mas->offset <= 1) &&
-           mas_slot_store(mas, entry, r_min, r_max, end, content, mt, slots))
+           mas_slot_store(mas, entry, r_min, r_max, end_piv, end, content, mt, slots))
                return content;
        else if ((mas->offset + 1 >= mt_slots[mt]) &&
                 mas_node_store(mas, entry, r_min, r_max, end, content, mt, slots, pivots))
-                return content;
+               return content;
 
        if (mas_is_err(mas))
                return content;