maple_tree: Add metadata to arange nodes.

Add a metadata into the arange64 nodes to tell the end and the largest gap slot.

This is used to speed up various activities on the node, especially finding the
end of the data and updating the gap.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>

diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index 6ca90fc6693c74c25515bb9c4d7cb5d5de881b6c..496e539bc54ebc6a231ff67c153796404cecce27 100644 (file)
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -85,6 +85,7 @@ struct maple_arange_64 {
        unsigned long pivot[MAPLE_ARANGE64_SLOTS - 1];
        void __rcu *slot[MAPLE_ARANGE64_SLOTS];
        unsigned long gap[MAPLE_ARANGE64_SLOTS];
+       unsigned char meta;
 };
 
 #define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1)

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 64d4de3c4bbf050b3195a02376db9050dfee6a52..008d676781543280ff5c89a8c62b7fa47c68b969 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -530,6 +530,7 @@ mas_logical_pivot(struct ma_state *mas, unsigned long *pivots,
                  unsigned char piv, enum maple_type type)
 {
        unsigned long lpiv = _mas_safe_pivot(mas, pivots, piv, type);
+
        if (!lpiv && piv)
                return mas->max;
        return lpiv;
@@ -650,6 +651,37 @@ static inline void mte_set_slot(const struct maple_enode *mn,
        ma_set_slot(mte_to_node(mn), slot, mte_node_type(mn), val);
 }
 
+#define MA_META_END_MASK       0b1111
+#define MA_META_GAP_SHIFT      4
+static inline void ma_set_meta(struct maple_node *mn, enum maple_type mt,
+                              unsigned char offset, unsigned char end)
+{
+
+       BUG_ON(mt != maple_arange_64);
+       mn->ma64.meta = (offset << MA_META_GAP_SHIFT) | end;
+}
+static inline unsigned char ma_meta_end(struct maple_node *mn,
+                                       enum maple_type mt)
+{
+
+       BUG_ON(mt != maple_arange_64);
+       return mn->ma64.meta & MA_META_END_MASK;
+}
+static inline unsigned char ma_meta_gap(struct maple_node *mn,
+                                       enum maple_type mt)
+{
+
+       BUG_ON(mt != maple_arange_64);
+       return mn->ma64.meta >> MA_META_GAP_SHIFT;
+}
+static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt,
+                                  unsigned char offset)
+{
+
+       BUG_ON(mt != maple_arange_64);
+       mn->ma64.meta = (offset << MA_META_GAP_SHIFT) |
+               (mn->ma64.meta & MA_META_END_MASK);
+}
 /*
  * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes.
  *
@@ -1007,25 +1039,31 @@ static inline unsigned char mas_data_end(struct ma_state *mas)
 {
        enum maple_type type = mte_node_type(mas->node);
        unsigned char offset = mt_min_slots[type];
-       unsigned long piv, *pivots = ma_pivots(mas_mn(mas), type);
+       unsigned long *pivots = ma_pivots(mas_mn(mas), type);
 
-       if (pivots[offset]) {
-               do {
-                       piv = mas_logical_pivot(mas, pivots, offset, type);
-                       if (piv >= mas->max)
-                               break;
+       if (type == maple_arange_64)
+               return ma_meta_end(mte_to_node(mas->node), type);
 
-               } while (++offset < mt_slots[type]);
-       } else {
-               while (--offset) {
-                       if (pivots[offset]) {
-                               if (pivots[offset] < mas->max)
-                                       offset++;
-                               break;
-                       }
-               };
+       if (!pivots[offset])
+               goto decrement;
+
+       // Higher than the min.
+       offset = mt_pivots[type] - 1;
+       // Check exceptions outside of the loop.
+       if (pivots[offset]) { // almost full.
+               if (pivots[offset] != mas->max) // Totally full.
+                       return offset + 1;
+               return offset;
        }
 
+decrement:
+       while (--offset) {
+               if (pivots[offset])
+                       break;
+       };
+       if (pivots[offset] < mas->max)
+               offset++;
+
        return offset;
 }
 
@@ -1042,6 +1080,7 @@ static inline unsigned long mas_leaf_max_gap(struct ma_state *mas)
        unsigned long *pivots = ma_pivots(mn, mt);
        void **slots = ma_slots(mn, mt);
        unsigned char i;
+       unsigned char max_piv;
 
        if (ma_is_dense(mt)) {
                for (i = 0; i < mt_slot_count(mas->node); i++) {
@@ -1058,13 +1097,22 @@ static inline unsigned long mas_leaf_max_gap(struct ma_state *mas)
                return max_gap;
        }
 
+       // Removing the pivot overflow optimizes the loop below.
+       max_piv = mt_pivots[mt] - 1;
+       if (pivots[max_piv] && pivots[max_piv] != mas->max &&
+           !slots[max_piv + 1])
+               max_gap = mas->max - pivots[max_piv];
+
        pstart = mas->min;
-       for (i = 0; i < mt_slots[mt]; i++) {
-               pend = mas_logical_pivot(mas, pivots, i, mt);
+       for (i = 0; i < mt_pivots[mt]; i++) {
+               pend = pivots[i];
+               if (!pend && i)
+                       pend = mas->max;
 
                if (slots[i])
                        goto next;
 
+
                gap = pend - pstart + 1;
                if (gap > max_gap)
                        max_gap = gap;
@@ -1077,34 +1125,40 @@ next:
        return max_gap;
 }
 static inline unsigned long
-ma_max_gap(unsigned long *pivots, unsigned long *gaps, enum maple_type mt)
+ma_max_gap(struct maple_node *node, unsigned long *gaps, enum maple_type mt,
+           unsigned char *offset)
 {
-       unsigned char i = mt_slots[mt] - 1;
-       unsigned long max_gap = gaps[i--];
-       do {
-               if (!pivots[i])
-                       continue;
+       unsigned char i = ma_meta_end(node, mt);
+       unsigned long max_gap = 0;
 
-               if (gaps[i] > max_gap)
+       do {
+               if (gaps[i] > max_gap) {
                        max_gap = gaps[i];
+                       *offset = i;
+               }
        } while(i--);
 
        return max_gap;
 }
+
 /*
  * mas_max_gap() - find the largest gap in a non-leaf node and set the slot.
  */
 static inline unsigned long mas_max_gap(struct ma_state *mas)
 {
-       unsigned long *gaps, *pivots;
+       unsigned long *gaps;//, *pivots;
+       unsigned char offset;
        enum maple_type mt;
        if (mte_is_leaf(mas->node))
                return mas_leaf_max_gap(mas);
 
        mt = mte_node_type(mas->node);
        gaps = ma_gaps(mas_mn(mas), mt);
-       pivots = ma_pivots(mas_mn(mas), mt);
-       return ma_max_gap(pivots, gaps, mt);
+       offset = ma_meta_gap(mas_mn(mas), mt);
+       if (offset == 15)
+               return 0;
+
+       return gaps[offset];
 }
 static inline unsigned long mas_tree_gap(struct ma_state *mas)
 {
@@ -1122,44 +1176,52 @@ static inline unsigned long mas_tree_gap(struct ma_state *mas)
        }
        return mas_max_gap(mas);
 }
-
-static inline void mas_parent_gap(struct ma_state *mas, unsigned char slot,
+static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset,
                unsigned long new)
 {
-       unsigned long old_max_gap = 0;
-       struct maple_node *pnode, *gpnode = NULL; // parent and grand parent nodes.
+       unsigned long meta_gap = 0;
+       struct maple_node *pnode;
        struct maple_enode *penode;
-       unsigned long *ppivots, *pgaps, *gpgaps = NULL;
-       enum maple_type pmt, gpmt;
-       unsigned char pslot = slot;
+       unsigned long *pgaps;
+       unsigned char meta_offset;
+       enum maple_type pmt;
 
        pnode = mte_parent(mas->node);
-       pmt = gpmt = mas_parent_enum(mas, mas->node);
+       pmt = mas_parent_enum(mas, mas->node);
        penode = mt_mk_node(pnode, pmt);
        pgaps = ma_gaps(pnode, pmt);
 
 ascend:
-       if (!mte_is_root(penode)) {
-               gpnode = mte_parent(penode);
-               gpmt = mas_parent_enum(mas, penode);
-               gpgaps = ma_gaps(gpnode, gpmt);
-               pslot = mte_parent_slot(penode);
-               old_max_gap = gpgaps[pslot];
-       }
-       pgaps[slot] = new;
-       if (mte_is_root(penode))
+       meta_offset = ma_meta_gap(pnode, pmt);
+       if (meta_offset == 15)
+               meta_gap = 0;
+       else
+               meta_gap = pgaps[meta_offset];
+
+       pgaps[offset] = new;
+
+       if (meta_gap == new)
                return;
 
-       ppivots = ma_pivots(pnode, pmt);
-       new = ma_max_gap(ppivots, pgaps, pmt);
-       if (new == old_max_gap)
+       if (offset != meta_offset) {
+           if (meta_gap > new)
+               return;
+
+           ma_set_meta_gap(pnode, pmt, offset);
+       } else if (new < meta_gap) {
+               meta_offset = 15;
+               new = ma_max_gap(pnode, pgaps, pmt, &meta_offset);
+               ma_set_meta_gap(pnode, pmt, meta_offset);
+       }
+
+       if (ma_is_root(pnode))
                return;
 
        /* Go to the parent node. */
-       pnode = gpnode;
-       pmt = gpmt;
-       pgaps = gpgaps;
-       slot = pslot;
+       pnode = mte_parent(penode);
+       pmt = mas_parent_enum(mas, penode);
+       pgaps = ma_gaps(pnode, pmt);
+       offset = mte_parent_slot(penode);
        penode = mt_mk_node(pnode, pmt);
        goto ascend;
 }
@@ -1181,7 +1243,7 @@ static inline void mas_update_gap(struct ma_state *mas)
                return;
 
        max_gap = mas_max_gap(mas);
-       /* Get the gap reported in the parent */
+
        pslot = mte_parent_slot(mas->node);
        p_gap = ma_gaps(mte_parent(mas->node),
                        mas_parent_enum(mas, mas->node))[pslot];
@@ -1499,21 +1561,34 @@ static inline void mab_mas_cp(struct maple_big_node *b_node,
        unsigned long *pivots = ma_pivots(node, mt);
        unsigned long *gaps = NULL;
 
+       if (mab_end - mab_start > mt_pivots[mt])
+               mab_end--;
+
        for (i = mab_start; i <= mab_end; i++, j++) {
                if (j && !b_node->pivot[i])
                        break;
 
-               if (j < mt_pivots[mt])
-                       pivots[j] = b_node->pivot[i];
+               pivots[j] = b_node->pivot[i];
        }
 
        memcpy(slots, b_node->slot + mab_start,
               sizeof(void*) * (i - mab_start));
 
        if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) {
+               unsigned long max_gap = 0;
+               unsigned char offset = 15, tmp;
+               unsigned char end = i - mab_start;
                gaps = ma_gaps(mas_mn(mas), mt);
-               memcpy(gaps, b_node->gap + mab_start,
-                      sizeof(unsigned long) * (i - mab_start));
+               for (tmp = 0; tmp < end; tmp++) {
+                       gaps[tmp] = b_node->gap[mab_start + tmp];
+                       if (gaps[tmp] > max_gap) {
+                               offset = tmp;
+                               max_gap = gaps[tmp];
+                       }
+               }
+//             memcpy(gaps, b_node->gap + mab_start,
+//                    sizeof(unsigned long) * end);
+               ma_set_meta(node, mt, offset, end - 1);
        }
        mas->max = b_node->pivot[--i];
 }
@@ -3770,17 +3845,6 @@ bool _mas_rev_awalk(struct ma_state *mas, unsigned long size)
        if (!ma_is_leaf(type))
                gaps = ma_gaps(node, type);
 
-       if (offset == mt_pivots[type]) {
-               // Initial start, walk until the end of the data.
-               while (--offset) {
-                       if (pivots[offset]) {
-                               if (pivots[offset] < mas->max)
-                                       offset++;
-                               break;
-                       }
-               }
-       }
-
        min = _mas_safe_pivot(mas, pivots, offset, type) + 1;
        do {
                max = min - 1;
@@ -3815,7 +3879,7 @@ bool _mas_rev_awalk(struct ma_state *mas, unsigned long size)
                break;
        } while (offset--);
 
-       if (offset >= mt_slots[type]) {  // node exhausted.
+       if (offset >= mt_slots[type]) {  // Overflow, node exhausted.
                offset = 0;
                goto ascend;
        }
@@ -3824,7 +3888,7 @@ bool _mas_rev_awalk(struct ma_state *mas, unsigned long size)
        mas->node = mas_slot(mas, slots, offset);
        mas->min = min;
        mas->max = max;
-       mas->offset = mt_pivot_count(mas->node);
+       mas->offset = mas_data_end(mas);
        return false;
 
 ascend:
@@ -3839,7 +3903,7 @@ static inline bool _mas_awalk(struct ma_state *mas, unsigned long size)
 {
        enum maple_type type = mte_node_type(mas->node);
        unsigned long pivot, min, gap = 0;
-       unsigned char offset = 0, pivot_count = mt_pivots[type];
+       unsigned char offset = 0;
        unsigned long *gaps = NULL, *pivots = ma_pivots(mas_mn(mas), type);
        void **slots = ma_slots(mas_mn(mas), type);
        bool found = false;
@@ -3855,7 +3919,7 @@ static inline bool _mas_awalk(struct ma_state *mas, unsigned long size)
        }
 
        min = mas_safe_min(mas, pivots, offset);
-       for (; offset <= pivot_count; offset++) {
+       for (; offset < mt_slots[type]; offset++) {
                pivot = _mas_safe_pivot(mas, pivots, offset, type);
                if (offset && !pivot)
                        break;
@@ -4038,7 +4102,7 @@ void mas_rev_awalk(struct ma_state *mas, unsigned long size)
 {
        struct maple_enode *last = NULL;
 
-       mas->offset = mt_pivot_count(mas->node);
+       mas->offset = mas_data_end(mas);
 
        /* There are 4 options:
         * go to child (descend)
@@ -4299,8 +4363,8 @@ no_gap:
  * Returns: 0 on success, -EBUSY otherwise.
  */
 static inline int mas_rev_alloc(struct ma_state *mas, unsigned long min,
-               unsigned long max, void *entry,
-               unsigned long size, unsigned long *index)
+                               unsigned long max, void *entry,
+                               unsigned long size, unsigned long *index)
 {
        int ret = 0;
 
@@ -4330,8 +4394,8 @@ no_gap:
  * range_min and range_max will be set accordingly.
  *
  */
-static inline void *mas_range_load(struct ma_state *mas, unsigned long *range_min,
-               unsigned long *range_max)
+static inline void *mas_range_load(struct ma_state *mas,
+          unsigned long *range_min, unsigned long *range_max)
 {
        void *entry = NULL;
 
@@ -5064,7 +5128,7 @@ void mt_dump_arange64(void *entry, unsigned long min, unsigned long max,
        pr_cont(" contents: ");
        for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++)
                pr_cont("%lu ", node->gap[i]);
-       pr_cont("| ");
+       pr_cont("| %02X | ", node->meta);
        for (i = 0; i < MAPLE_ARANGE64_SLOTS - 1; i++)
                pr_cont(MA_PTR" %lu ", node->slot[i], node->pivot[i]);
        pr_cont(MA_PTR"\n", node->slot[i]);