maple_tree: Reduce mas_split() memory usage

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 309d5cb53b87d76fdf0422f42bec0b3b7d3e156f..51437e26ef0daab3900f138c6504ed116b32aa6f 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -146,6 +146,18 @@ struct maple_subtree_state {
        struct maple_big_node *bn;
 };
 
+struct maple_split_state {
+       struct maple_enode *l_node;
+       struct maple_enode *r_node;
+       unsigned char offset;
+       struct ma_state *l;             /* New left side of subtree */
+       struct ma_state *m;             /* New middle of subtree (rare) */
+       struct ma_state *r;             /* New right side of subtree */
+       struct ma_topiary *free;        /* nodes to be freed */
+       struct ma_topiary *destroy;     /* Nodes to be destroyed (walked and freed) */
+       struct maple_big_node *bn;
+};
+
 /* Functions */
 static inline struct maple_node *mt_alloc_one(gfp_t gfp)
 {
@@ -2638,7 +2650,7 @@ static inline void mab_set_b_end(struct maple_big_node *b_node,
 }
 
 /*
- * mas_set_split_parent() - combine_then_separate helper function.  Sets the parent
+ * mte_set_split_parent() - combine_then_separate helper function.  Sets the parent
  * of @mas->node to either @left or @right, depending on @slot and @split
  *
  * @mas - the maple state with the node that needs a parent
@@ -2647,18 +2659,18 @@ static inline void mab_set_b_end(struct maple_big_node *b_node,
  * @slot - the slot the mas->node was placed
  * @split - the split location between @left and @right
  */
-static inline void mas_set_split_parent(struct ma_state *mas,
+static inline void mte_set_split_parent(struct maple_enode *enode,
                                        struct maple_enode *left,
                                        struct maple_enode *right,
                                        unsigned char *slot, unsigned char split)
 {
-       if (mas_is_none(mas))
+       if (enode == MAS_NONE)
                return;
 
        if ((*slot) <= split)
-               mte_set_parent(mas->node, left, *slot);
+               mte_set_parent(enode, left, *slot);
        else if (right)
-               mte_set_parent(mas->node, right, (*slot) - split - 1);
+               mte_set_parent(enode, right, (*slot) - split - 1);
 
        (*slot)++;
 }
@@ -2718,13 +2730,13 @@ static inline void mast_set_split_parents(struct maple_subtree_state *mast,
        slot = mast->l->offset;
 
        mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
-       mas_set_split_parent(mast->l, l, r, &slot, split);
+       mte_set_split_parent(mast->l->node, l, r, &slot, split);
 
        mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
-       mas_set_split_parent(mast->m, l, r, &slot, split);
+       mte_set_split_parent(mast->m->node, l, r, &slot, split);
 
        mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
-       mas_set_split_parent(mast->r, l, r, &slot, split);
+       mte_set_split_parent(mast->r->node, l, r, &slot, split);
 }
 
 /*
@@ -3264,107 +3276,102 @@ done:
 
 /*
  * mas_split_final_node() - Split the final node in a subtree operation.
- * @mast: the maple subtree state
+ * @mss: the maple subtree state
  * @mas: The maple state
  * @height: The height of the tree in case it's a new root.
  */
-static inline bool mas_split_final_node(struct maple_subtree_state *mast,
+static inline bool mas_split_final_node(struct maple_split_state *mss,
                                        struct ma_state *mas, int height)
 {
        struct maple_enode *ancestor;
 
        if (mte_is_root(mas->node)) {
                if (mt_is_alloc(mas->tree))
-                       mast->bn->type = maple_arange_64;
+                       mss->bn->type = maple_arange_64;
                else
-                       mast->bn->type = maple_range_64;
+                       mss->bn->type = maple_range_64;
                mas->depth = height;
        }
        /*
         * Only a single node is used here, could be root.
         * The Big_node data should just fit in a single node.
         */
-       ancestor = mas_new_ma_node(mas, mast->bn);
-       mte_set_parent(mast->l->node, ancestor, mast->l->offset);
-       mte_set_parent(mast->r->node, ancestor, mast->r->offset);
+       ancestor = mas_new_ma_node(mas, mss->bn);
+       mte_set_parent(mss->l->node, ancestor, mss->l->offset);
+       mte_set_parent(mss->r->node, ancestor, mss->r->offset);
        mte_to_node(ancestor)->parent = mas_mn(mas)->parent;
 
-       mast->l->node = ancestor;
-       mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true);
-       mas->offset = mast->bn->b_end - 1;
+       mss->l->node = ancestor;
+       mab_mas_cp(mss->bn, 0, mt_slots[mss->bn->type] - 1, mss->l, true);
+       mas->offset = mss->bn->b_end - 1;
        return true;
 }
 
 /*
- * mast_fill_bnode() - Copy data into the big node in the subtree state
- * @mast: The maple subtree state
+ * mss_fill_bnode() - Copy data into the big node in the subtree state
+ * @mss: The maple split state
  * @mas: the maple state
  * @skip: The number of entries to skip for new nodes insertion.
  */
-static inline void mast_fill_bnode(struct maple_subtree_state *mast,
-                                        struct ma_state *mas,
-                                        unsigned char skip)
+static inline void mss_fill_bnode(struct maple_split_state *mss,
+                                 struct ma_state *mas, unsigned char skip)
 {
        bool cp = true;
        struct maple_enode *old = mas->node;
        unsigned char split;
 
-       memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap));
-       memset(mast->bn->slot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->slot));
-       memset(mast->bn->pivot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->pivot));
-       mast->bn->b_end = 0;
-
+       memset(mss->bn, 0, sizeof(struct maple_big_node));
        if (mte_is_root(mas->node)) {
                cp = false;
        } else {
                mas_ascend(mas);
-               mat_add(mast->free, old);
+               mat_add(mss->free, old);
                mas->offset = mte_parent_slot(mas->node);
        }
 
-       if (cp && mast->l->offset)
-               mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0);
+       if (cp && mss->l->offset)
+               mas_mab_cp(mas, 0, mss->l->offset - 1, mss->bn, 0);
 
-       split = mast->bn->b_end;
-       mab_set_b_end(mast->bn, mast->l, mast->l->node);
-       mast->r->offset = mast->bn->b_end;
-       mab_set_b_end(mast->bn, mast->r, mast->r->node);
-       if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max)
+       split = mss->bn->b_end;
+       mab_set_b_end(mss->bn, mss->l, mss->l->node);
+       mss->r->offset = mss->bn->b_end;
+       mab_set_b_end(mss->bn, mss->r, mss->r->node);
+       if (mss->bn->pivot[mss->bn->b_end - 1] == mas->max)
                cp = false;
 
        if (cp)
                mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1,
-                          mast->bn, mast->bn->b_end);
+                          mss->bn, mss->bn->b_end);
 
-       mast->bn->b_end--;
-       mast->bn->type = mte_node_type(mas->node);
+       mss->bn->b_end--;
+       mss->bn->type = mte_node_type(mas->node);
 }
 
 /*
- * mast_split_data() - Split the data in the subtree state big node into regular
+ * mss_split_data() - Split the data in the subtree state big node into regular
  * nodes.
- * @mast: The maple subtree state
+ * @mss: The maple split state
  * @mas: The maple state
  * @split: The location to split the big node
  */
-static inline void mast_split_data(struct maple_subtree_state *mast,
+static inline void mss_split_data(struct maple_split_state *mss,
           struct ma_state *mas, unsigned char split)
 {
        unsigned char p_slot;
 
-       mab_mas_cp(mast->bn, 0, split, mast->l, true);
-       mte_set_pivot(mast->r->node, 0, mast->r->max);
-       mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false);
-       mast->l->offset = mte_parent_slot(mas->node);
-       mast->l->max = mast->bn->pivot[split];
-       mast->r->min = mast->l->max + 1;
+       mab_mas_cp(mss->bn, 0, split, mss->l, true);
+       mte_set_pivot(mss->r->node, 0, mss->r->max);
+       mab_mas_cp(mss->bn, split + 1, mss->bn->b_end, mss->r, false);
+       mss->l->offset = mte_parent_slot(mas->node);
+       mss->l->max = mss->bn->pivot[split];
+       mss->r->min = mss->l->max + 1;
        if (mte_is_leaf(mas->node))
                return;
 
-       p_slot = mast->orig_l->offset;
-       mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node,
+       p_slot = mss->offset;
+       mte_set_split_parent(mss->l_node, mss->l->node, mss->r->node,
                             &p_slot, split);
-       mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node,
+       mte_set_split_parent(mss->r_node, mss->l->node, mss->r->node,
                             &p_slot, split);
 }
 
@@ -3381,14 +3388,14 @@ static inline void mast_split_data(struct maple_subtree_state *mast,
  * Return: True if pushed, false otherwise.
  */
 static inline bool mas_push_data(struct ma_state *mas, int height,
-                                struct maple_subtree_state *mast, bool left)
+                                struct maple_split_state *mss, bool left)
 {
-       unsigned char slot_total = mast->bn->b_end;
+       unsigned char slot_total = mss->bn->b_end;
        unsigned char end, space, split;
 
        MA_STATE(tmp_mas, mas->tree, mas->index, mas->last);
        tmp_mas = *mas;
-       tmp_mas.depth = mast->l->depth;
+       tmp_mas.depth = mss->l->depth;
 
        if (left && !mas_prev_sibling(&tmp_mas))
                return false;
@@ -3399,7 +3406,7 @@ static inline bool mas_push_data(struct ma_state *mas, int height,
        slot_total += end;
        space = 2 * mt_slot_count(mas->node) - 2;
        /* -2 instead of -1 to ensure there isn't a triple split */
-       if (ma_is_leaf(mast->bn->type))
+       if (ma_is_leaf(mss->bn->type))
                space--;
 
        if (mas->max == ULONG_MAX)
@@ -3408,39 +3415,39 @@ static inline bool mas_push_data(struct ma_state *mas, int height,
        if (slot_total >= space)
                return false;
 
-       /* Get the data; Fill mast->bn */
-       mast->bn->b_end++;
+       /* Get the data; Fill mss->bn */
+       mss->bn->b_end++;
        if (left) {
-               mab_shift_right(mast->bn, end + 1);
-               mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0);
-               mast->bn->b_end = slot_total + 1;
+               mab_shift_right(mss->bn, end + 1);
+               mas_mab_cp(&tmp_mas, 0, end, mss->bn, 0);
+               mss->bn->b_end = slot_total + 1;
        } else {
-               mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end);
+               mas_mab_cp(&tmp_mas, 0, end, mss->bn, mss->bn->b_end);
        }
 
-       /* Configure mast for splitting of mast->bn */
-       split = mt_slots[mast->bn->type] - 2;
+       /* Configure mss for splitting of mss->bn */
+       split = mt_slots[mss->bn->type] - 2;
        if (left) {
                /*  Switch mas to prev node  */
-               mat_add(mast->free, mas->node);
+               mat_add(mss->free, mas->node);
                *mas = tmp_mas;
-               /* Start using mast->l for the left side. */
-               tmp_mas.node = mast->l->node;
-               *mast->l = tmp_mas;
+               /* Start using mss->l for the left side. */
+               tmp_mas.node = mss->l->node;
+               *mss->l = tmp_mas;
        } else {
-               mat_add(mast->free, tmp_mas.node);
-               tmp_mas.node = mast->r->node;
-               *mast->r = tmp_mas;
+               mat_add(mss->free, tmp_mas.node);
+               tmp_mas.node = mss->r->node;
+               *mss->r = tmp_mas;
                split = slot_total - split;
        }
-       split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]);
+       split = mab_no_null_split(mss->bn, split, mt_slots[mss->bn->type]);
        /* Update parent slot for split calculation. */
        if (left)
-               mast->orig_l->offset += end + 1;
+               mss->offset += end + 1;
 
-       mast_split_data(mast, mas, split);
-       mast_fill_bnode(mast, mas, 2);
-       mas_split_final_node(mast, mas, height + 1);
+       mss_split_data(mss, mas, split);
+       mss_fill_bnode(mss, mas, 2);
+       mas_split_final_node(mss, mas, height + 1);
        return true;
 }
 
@@ -3453,9 +3460,10 @@ static inline bool mas_push_data(struct ma_state *mas, int height,
 static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
 {
 
-       struct maple_subtree_state mast;
+       struct maple_split_state mss;
        int height = 0;
        unsigned char mid_split, split = 0;
+       unsigned long prev_l_min;
 
        /*
         * Splitting is handled differently from any other B-tree; the Maple
@@ -3476,8 +3484,6 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
         */
        MA_STATE(l_mas, mas->tree, mas->index, mas->last);
        MA_STATE(r_mas, mas->tree, mas->index, mas->last);
-       MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last);
-       MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last);
        MA_TOPIARY(mat, mas->tree);
 
        trace_ma_op(__func__, mas);
@@ -3487,16 +3493,18 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
        if (mas_is_err(mas))
                return 0;
 
-       mast.l = &l_mas;
-       mast.r = &r_mas;
-       mast.orig_l = &prev_l_mas;
-       mast.orig_r = &prev_r_mas;
-       mast.free = &mat;
-       mast.bn = b_node;
+       mss.l = &l_mas;
+       mss.r = &r_mas;
+       mss.l_node = mas->node;
+       mss.r_node = mas->node;
+       mss.offset = mas->offset;
+       prev_l_min = mas->min;
+       mss.free = &mat;
+       mss.bn = b_node;
 
        while (height++ <= mas->depth) {
                if (mt_slots[b_node->type] > b_node->b_end) {
-                       mas_split_final_node(&mast, mas, height);
+                       mas_split_final_node(&mss, mas, height);
                        break;
                }
 
@@ -3511,29 +3519,30 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
                 * is a significant savings.
                 */
                /* Try to push left. */
-               if (mas_push_data(mas, height, &mast, true))
+               if (mas_push_data(mas, height, &mss, true))
                        break;
 
                /* Try to push right. */
-               if (mas_push_data(mas, height, &mast, false))
+               if (mas_push_data(mas, height, &mss, false))
                        break;
 
-               split = mab_calc_split(mas, b_node, &mid_split, prev_l_mas.min);
-               mast_split_data(&mast, mas, split);
+               split = mab_calc_split(mas, b_node, &mid_split, prev_l_min);
+               mss_split_data(&mss, mas, split);
                /*
                 * Usually correct, mab_mas_cp in the above call overwrites
                 * r->max.
                 */
-               mast.r->max = mas->max;
-               mast_fill_bnode(&mast, mas, 1);
-               prev_l_mas = *mast.l;
-               prev_r_mas = *mast.r;
+               mss.r->max = mas->max;
+               mss_fill_bnode(&mss, mas, 1);
+               mss.l_node = mss.l->node;
+               mss.r_node = mss.r->node;
+               mss.offset = mss.l->offset;
        }
 
        /* Set the original node as dead */
-       mat_add(mast.free, mas->node);
+       mat_add(mss.free, mas->node);
        mas->node = l_mas.node;
-       mas_wmb_replace(mas, mast.free, NULL);
+       mas_wmb_replace(mas, mss.free, NULL);
        mtree_range_walk(mas);
        return 1;
 }