From: Liam R. Howlett <Liam.Howlett@oracle.com>
Date: Tue, 22 Mar 2022 16:37:31 +0000 (-0400)
Subject: maple_tree: Reduce mas_split() memory usage
X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=1f2c541a306c89f68f6040bbc5db7af707bf792d;p=users%2Fjedix%2Flinux-maple.git
maple_tree: Reduce mas_split() memory usage
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
---
diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index e1743803c851..75f7be445dbb 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -146,6 +146,18 @@ struct maple_subtree_state {
struct maple_big_node *bn;
};
+struct maple_split_state {
+ struct maple_enode *l_node;
+ struct maple_enode *r_node;
+ unsigned char offset;
+ struct ma_state *l; /* New left side of subtree */
+ struct ma_state *m; /* New middle of subtree (rare) */
+ struct ma_state *r; /* New right side of subtree */
+ struct ma_topiary *free; /* nodes to be freed */
+ struct ma_topiary *destroy; /* Nodes to be destroyed (walked and freed) */
+ struct maple_big_node *bn;
+};
+
/* Functions */
static inline struct maple_node *mt_alloc_one(gfp_t gfp)
{
@@ -2650,7 +2662,7 @@ static inline void mab_set_b_end(struct maple_big_node *b_node,
}
/*
- * mas_set_split_parent() - combine_then_separate helper function. Sets the parent
+ * mte_set_split_parent() - combine_then_separate helper function. Sets the parent
* of @mas->node to either @left or @right, depending on @slot and @split
*
* @mas - the maple state with the node that needs a parent
@@ -2659,18 +2671,18 @@ static inline void mab_set_b_end(struct maple_big_node *b_node,
* @slot - the slot the mas->node was placed
* @split - the split location between @left and @right
*/
-static inline void mas_set_split_parent(struct ma_state *mas,
+static inline void mte_set_split_parent(struct maple_enode *enode,
struct maple_enode *left,
struct maple_enode *right,
unsigned char *slot, unsigned char split)
{
- if (mas_is_none(mas))
+ if (enode == MAS_NONE)
return;
if ((*slot) <= split)
- mte_set_parent(mas->node, left, *slot);
+ mte_set_parent(enode, left, *slot);
else if (right)
- mte_set_parent(mas->node, right, (*slot) - split - 1);
+ mte_set_parent(enode, right, (*slot) - split - 1);
(*slot)++;
}
@@ -2730,13 +2742,13 @@ static inline void mast_set_split_parents(struct maple_subtree_state *mast,
slot = mast->l->offset;
mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
- mas_set_split_parent(mast->l, l, r, &slot, split);
+ mte_set_split_parent(mast->l->node, l, r, &slot, split);
mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
- mas_set_split_parent(mast->m, l, r, &slot, split);
+ mte_set_split_parent(mast->m->node, l, r, &slot, split);
mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
- mas_set_split_parent(mast->r, l, r, &slot, split);
+ mte_set_split_parent(mast->r->node, l, r, &slot, split);
}
/*
@@ -3276,107 +3288,102 @@ done:
/*
* mas_split_final_node() - Split the final node in a subtree operation.
- * @mast: the maple subtree state
+ * @mss: the maple subtree state
* @mas: The maple state
* @height: The height of the tree in case it's a new root.
*/
-static inline bool mas_split_final_node(struct maple_subtree_state *mast,
+static inline bool mas_split_final_node(struct maple_split_state *mss,
struct ma_state *mas, int height)
{
struct maple_enode *ancestor;
if (mte_is_root(mas->node)) {
if (mt_is_alloc(mas->tree))
- mast->bn->type = maple_arange_64;
+ mss->bn->type = maple_arange_64;
else
- mast->bn->type = maple_range_64;
+ mss->bn->type = maple_range_64;
mas->depth = height;
}
/*
* Only a single node is used here, could be root.
* The Big_node data should just fit in a single node.
*/
- ancestor = mas_new_ma_node(mas, mast->bn);
- mte_set_parent(mast->l->node, ancestor, mast->l->offset);
- mte_set_parent(mast->r->node, ancestor, mast->r->offset);
+ ancestor = mas_new_ma_node(mas, mss->bn);
+ mte_set_parent(mss->l->node, ancestor, mss->l->offset);
+ mte_set_parent(mss->r->node, ancestor, mss->r->offset);
mte_to_node(ancestor)->parent = mas_mn(mas)->parent;
- mast->l->node = ancestor;
- mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true);
- mas->offset = mast->bn->b_end - 1;
+ mss->l->node = ancestor;
+ mab_mas_cp(mss->bn, 0, mt_slots[mss->bn->type] - 1, mss->l, true);
+ mas->offset = mss->bn->b_end - 1;
return true;
}
/*
- * mast_fill_bnode() - Copy data into the big node in the subtree state
- * @mast: The maple subtree state
+ * mss_fill_bnode() - Copy data into the big node in the subtree state
+ * @mss: The maple split state
* @mas: the maple state
* @skip: The number of entries to skip for new nodes insertion.
*/
-static inline void mast_fill_bnode(struct maple_subtree_state *mast,
- struct ma_state *mas,
- unsigned char skip)
+static inline void mss_fill_bnode(struct maple_split_state *mss,
+ struct ma_state *mas, unsigned char skip)
{
bool cp = true;
struct maple_enode *old = mas->node;
unsigned char split;
- memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap));
- memset(mast->bn->slot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->slot));
- memset(mast->bn->pivot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->pivot));
- mast->bn->b_end = 0;
-
+ memset(mss->bn, 0, sizeof(struct maple_big_node));
if (mte_is_root(mas->node)) {
cp = false;
} else {
mas_ascend(mas);
- mat_add(mast->free, old);
+ mat_add(mss->free, old);
mas->offset = mte_parent_slot(mas->node);
}
- if (cp && mast->l->offset)
- mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0);
+ if (cp && mss->l->offset)
+ mas_mab_cp(mas, 0, mss->l->offset - 1, mss->bn, 0);
- split = mast->bn->b_end;
- mab_set_b_end(mast->bn, mast->l, mast->l->node);
- mast->r->offset = mast->bn->b_end;
- mab_set_b_end(mast->bn, mast->r, mast->r->node);
- if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max)
+ split = mss->bn->b_end;
+ mab_set_b_end(mss->bn, mss->l, mss->l->node);
+ mss->r->offset = mss->bn->b_end;
+ mab_set_b_end(mss->bn, mss->r, mss->r->node);
+ if (mss->bn->pivot[mss->bn->b_end - 1] == mas->max)
cp = false;
if (cp)
mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1,
- mast->bn, mast->bn->b_end);
+ mss->bn, mss->bn->b_end);
- mast->bn->b_end--;
- mast->bn->type = mte_node_type(mas->node);
+ mss->bn->b_end--;
+ mss->bn->type = mte_node_type(mas->node);
}
/*
- * mast_split_data() - Split the data in the subtree state big node into regular
+ * mss_split_data() - Split the data in the subtree state big node into regular
* nodes.
- * @mast: The maple subtree state
+ * @mss: The maple split state
* @mas: The maple state
* @split: The location to split the big node
*/
-static inline void mast_split_data(struct maple_subtree_state *mast,
+static inline void mss_split_data(struct maple_split_state *mss,
struct ma_state *mas, unsigned char split)
{
unsigned char p_slot;
- mab_mas_cp(mast->bn, 0, split, mast->l, true);
- mte_set_pivot(mast->r->node, 0, mast->r->max);
- mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false);
- mast->l->offset = mte_parent_slot(mas->node);
- mast->l->max = mast->bn->pivot[split];
- mast->r->min = mast->l->max + 1;
+ mab_mas_cp(mss->bn, 0, split, mss->l, true);
+ mte_set_pivot(mss->r->node, 0, mss->r->max);
+ mab_mas_cp(mss->bn, split + 1, mss->bn->b_end, mss->r, false);
+ mss->l->offset = mte_parent_slot(mas->node);
+ mss->l->max = mss->bn->pivot[split];
+ mss->r->min = mss->l->max + 1;
if (mte_is_leaf(mas->node))
return;
- p_slot = mast->orig_l->offset;
- mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node,
+ p_slot = mss->offset;
+ mte_set_split_parent(mss->l_node, mss->l->node, mss->r->node,
&p_slot, split);
- mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node,
+ mte_set_split_parent(mss->r_node, mss->l->node, mss->r->node,
&p_slot, split);
}
@@ -3393,14 +3400,14 @@ static inline void mast_split_data(struct maple_subtree_state *mast,
* Return: True if pushed, false otherwise.
*/
static inline bool mas_push_data(struct ma_state *mas, int height,
- struct maple_subtree_state *mast, bool left)
+ struct maple_split_state *mss, bool left)
{
- unsigned char slot_total = mast->bn->b_end;
+ unsigned char slot_total = mss->bn->b_end;
unsigned char end, space, split;
MA_STATE(tmp_mas, mas->tree, mas->index, mas->last);
tmp_mas = *mas;
- tmp_mas.depth = mast->l->depth;
+ tmp_mas.depth = mss->l->depth;
if (left && !mas_prev_sibling(&tmp_mas))
return false;
@@ -3411,7 +3418,7 @@ static inline bool mas_push_data(struct ma_state *mas, int height,
slot_total += end;
space = 2 * mt_slot_count(mas->node) - 2;
/* -2 instead of -1 to ensure there isn't a triple split */
- if (ma_is_leaf(mast->bn->type))
+ if (ma_is_leaf(mss->bn->type))
space--;
if (mas->max == ULONG_MAX)
@@ -3420,39 +3427,39 @@ static inline bool mas_push_data(struct ma_state *mas, int height,
if (slot_total >= space)
return false;
- /* Get the data; Fill mast->bn */
- mast->bn->b_end++;
+ /* Get the data; Fill mss->bn */
+ mss->bn->b_end++;
if (left) {
- mab_shift_right(mast->bn, end + 1);
- mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0);
- mast->bn->b_end = slot_total + 1;
+ mab_shift_right(mss->bn, end + 1);
+ mas_mab_cp(&tmp_mas, 0, end, mss->bn, 0);
+ mss->bn->b_end = slot_total + 1;
} else {
- mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end);
+ mas_mab_cp(&tmp_mas, 0, end, mss->bn, mss->bn->b_end);
}
- /* Configure mast for splitting of mast->bn */
- split = mt_slots[mast->bn->type] - 2;
+ /* Configure mss for splitting of mss->bn */
+ split = mt_slots[mss->bn->type] - 2;
if (left) {
/* Switch mas to prev node */
- mat_add(mast->free, mas->node);
+ mat_add(mss->free, mas->node);
*mas = tmp_mas;
- /* Start using mast->l for the left side. */
- tmp_mas.node = mast->l->node;
- *mast->l = tmp_mas;
+ /* Start using mss->l for the left side. */
+ tmp_mas.node = mss->l->node;
+ *mss->l = tmp_mas;
} else {
- mat_add(mast->free, tmp_mas.node);
- tmp_mas.node = mast->r->node;
- *mast->r = tmp_mas;
+ mat_add(mss->free, tmp_mas.node);
+ tmp_mas.node = mss->r->node;
+ *mss->r = tmp_mas;
split = slot_total - split;
}
- split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]);
+ split = mab_no_null_split(mss->bn, split, mt_slots[mss->bn->type]);
/* Update parent slot for split calculation. */
if (left)
- mast->orig_l->offset += end + 1;
+ mss->offset += end + 1;
- mast_split_data(mast, mas, split);
- mast_fill_bnode(mast, mas, 2);
- mas_split_final_node(mast, mas, height + 1);
+ mss_split_data(mss, mas, split);
+ mss_fill_bnode(mss, mas, 2);
+ mas_split_final_node(mss, mas, height + 1);
return true;
}
@@ -3465,9 +3472,10 @@ static inline bool mas_push_data(struct ma_state *mas, int height,
static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
{
- struct maple_subtree_state mast;
+ struct maple_split_state mss;
int height = 0;
unsigned char mid_split, split = 0;
+ unsigned long prev_l_min;
/*
* Splitting is handled differently from any other B-tree; the Maple
@@ -3488,8 +3496,6 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
*/
MA_STATE(l_mas, mas->tree, mas->index, mas->last);
MA_STATE(r_mas, mas->tree, mas->index, mas->last);
- MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last);
- MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last);
MA_TOPIARY(mat, mas->tree);
trace_ma_op(__func__, mas);
@@ -3499,16 +3505,18 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
if (mas_is_err(mas))
return 0;
- mast.l = &l_mas;
- mast.r = &r_mas;
- mast.orig_l = &prev_l_mas;
- mast.orig_r = &prev_r_mas;
- mast.free = &mat;
- mast.bn = b_node;
+ mss.l = &l_mas;
+ mss.r = &r_mas;
+ mss.l_node = mas->node;
+ mss.r_node = mas->node;
+ mss.offset = mas->offset;
+ prev_l_min = mas->min;
+ mss.free = &mat;
+ mss.bn = b_node;
while (height++ <= mas->depth) {
if (mt_slots[b_node->type] > b_node->b_end) {
- mas_split_final_node(&mast, mas, height);
+ mas_split_final_node(&mss, mas, height);
break;
}
@@ -3523,29 +3531,30 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
* is a significant savings.
*/
/* Try to push left. */
- if (mas_push_data(mas, height, &mast, true))
+ if (mas_push_data(mas, height, &mss, true))
break;
/* Try to push right. */
- if (mas_push_data(mas, height, &mast, false))
+ if (mas_push_data(mas, height, &mss, false))
break;
- split = mab_calc_split(mas, b_node, &mid_split, prev_l_mas.min);
- mast_split_data(&mast, mas, split);
+ split = mab_calc_split(mas, b_node, &mid_split, prev_l_min);
+ mss_split_data(&mss, mas, split);
/*
* Usually correct, mab_mas_cp in the above call overwrites
* r->max.
*/
- mast.r->max = mas->max;
- mast_fill_bnode(&mast, mas, 1);
- prev_l_mas = *mast.l;
- prev_r_mas = *mast.r;
+ mss.r->max = mas->max;
+ mss_fill_bnode(&mss, mas, 1);
+ mss.l_node = mss.l->node;
+ mss.r_node = mss.r->node;
+ mss.offset = mss.l->offset;
}
/* Set the original node as dead */
- mat_add(mast.free, mas->node);
+ mat_add(mss.free, mas->node);
mas->node = l_mas.node;
- mas_wmb_replace(mas, mast.free, NULL);
+ mas_wmb_replace(mas, mss.free, NULL);
mtree_range_walk(mas);
return 1;
}