From: Liam R. Howlett <Liam.Howlett@Oracle.com>
Date: Fri, 18 Jan 2019 15:55:38 +0000 (-0500)
Subject: maple_tree: Split support
X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=11d3093b0e7bcf7dbeccaea736ce17412e9e6ebc;p=users%2Fjedix%2Flinux-maple.git

maple_tree: Split support

Initial split support.  Walks up the tree splitting as necessary to
support the addition of a node.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
---

diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index 8176228b60f4..3de603ee8631 100644
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -171,9 +171,27 @@ void mtree_destroy(struct maple_tree *);
  */
 static inline bool mtree_empty(const struct maple_tree *mt)
 {
-	return mt->ma_root == NULL;
+	return rcu_dereference(mt->ma_root) == NULL;
 }
 
+struct ma_cp {
+	struct maple_node *src;
+	struct maple_node *dst;
+	unsigned char src_start;
+	unsigned char src_end;
+	unsigned char dst_start;
+	unsigned char dst_end;
+};
+#define MA_CP(name, snode, dnode, sstart, send)                         \
+	struct ma_cp name = {					        \
+		.src = snode,						\
+		.dst = dnode,						\
+		.src_start = sstart,					\
+		.src_end = send,					\
+		.dst_start = 0,					        \
+		.dst_end = MAPLE_RANGE64_SLOTS - 1                      \
+	}
+
 /* Advanced API */
 
 struct ma_state {
diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index d0a8f127a8ee..76096601e097 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -42,6 +42,11 @@ static inline bool mt_is_leaf(const void *entry)
 	return mt_node_type(entry) < maple_range_16;
 }
 
+static inline enum maple_type mt_node_hole(const void *entry)
+{
+	return (unsigned long)entry & 4;
+
+}
 static inline bool mt_is_reserved(const void *entry)
 {
 	return ((unsigned long)entry < 4096) && xa_is_internal(entry);
@@ -108,32 +113,33 @@ static inline unsigned int mt_slot_shift(struct maple_node *node)
 static inline void mt_set_parent(struct maple_node *node,
 				 struct maple_node *parent, short slot)
 {
-	unsigned int bitmask = mt_slot_mask(node);
-	unsigned int slot_shift = mt_slot_shift(node);
+	unsigned long bitmask = mt_slot_mask(node);
+	unsigned long slot_shift = mt_slot_shift(node);
 	unsigned long val = (unsigned long) parent;
 
 	BUG_ON(slot > MAPLE_NODE_SLOTS); // Only 4 bits to use.
 	val &= ~bitmask; // Remove any old slot number.
 	val |= (slot << slot_shift); // Set the slot.
-	node->parent = (struct maple_node *)val;
+	mt_to_node(node)->parent = (struct maple_node *)val;
 }
 
 static inline unsigned int mt_parent_slot(struct maple_node *node)
 {
 	unsigned int bitmask = mt_slot_mask(node);
 	unsigned int slot_shift = mt_slot_shift(node);
-	unsigned long val = (unsigned long) node->parent;
+	unsigned long val = (unsigned long) mt_to_node(node)->parent;
 
 	return (val & bitmask) >> slot_shift;
 }
 
 static inline void *mt_parent(struct maple_node *node)
 {
-	unsigned int bitmask = mt_slot_mask(node) | 7;
+	unsigned long bitmask = mt_slot_mask(node) | 7;
 
-	return (void *)((unsigned long)(node->parent) & ~bitmask);
+	return (void *)((unsigned long)(mt_to_node(node)->parent) & ~bitmask);
 
 }
+
 static inline struct maple_node *ma_get_alloc(const struct ma_state *ms)
 {
 	return (struct maple_node *)((unsigned long)ms->alloc & ~0x7F);
@@ -178,6 +184,24 @@ static inline bool ma_is_root(struct maple_node *node)
 		return true;
 	return false;
 }
+
+static inline void ma_encoded_parent(struct ma_state *mas)
+{
+	struct maple_node *parent, *gparent;
+	unsigned char slot;
+
+	if (ma_is_root(mt_to_node(mas->node)->parent)) {
+		mas->node = rcu_dereference(mas->tree->ma_root);
+		return;
+	}
+
+	parent = mt_parent(mas->node);
+	gparent = mt_parent(parent);
+	slot = mt_parent_slot(parent);
+	mas->node = rcu_dereference(gparent->mr64.slot[slot]);
+	return;
+}
+
 static inline struct maple_node *ma_next_alloc(struct ma_state *ms)
 {
 	int cnt;
@@ -283,35 +307,34 @@ static struct maple_node *mas_node_cnt(struct ma_state *ms, int count)
 	return ms->alloc;
 }
 
-static void *mas_start(struct ma_state *ms)
+static void *mas_start(struct ma_state *mas)
 {
 	void *entry;
 
-	if (mas_is_err(ms))
+	if (mas_is_err(mas))
 		return NULL;
 
-	if (ms->node == MAS_START) {
-		entry = ms->tree->ma_root;
+	if (mas->node == MAS_START) {
+		entry = mas->tree->ma_root;
 		if (!xa_is_node(entry)) {
-			if (ms->index > 0)
+			if (mas->index > 0)
 				return NULL;
-			ms->node = MAS_ROOT;
-			ma_set_slot(ms, MAPLE_NODE_SLOTS);
+			mas->node = MAS_ROOT;
+			ma_set_slot(mas, MAPLE_NODE_SLOTS);
 		} else {
 			entry = mt_safe_root(entry);
 		}
 	} else {
-		entry = ms->node;
+		entry = mas->node;
 	}
 
 	return entry;
 }
 
-unsigned char ma_data_end_r64(const struct ma_state *ms)
+unsigned char ma_data_end_r64(const struct maple_range_64 *mr64,
+			      unsigned long last)
 {
-	struct maple_range_64 *mr64 = &mt_to_node(ms->node)->mr64;
 	unsigned char data_end = 0;
-	unsigned long last = ms->max;
 
 	for (data_end = 0; data_end < MAPLE_RANGE64_SLOTS - 1; data_end++) {
 		last = mr64->pivot[data_end];
@@ -319,21 +342,131 @@ unsigned char ma_data_end_r64(const struct ma_state *ms)
 			return data_end;
 	}
 
-	if (mr64->slot[data_end + 1] != NULL)
+	if (mr64->slot[data_end] != NULL)
 		data_end++;
 
 	return data_end;
 }
 
+static int ma_calc_split(struct ma_state *mas)
+{
+	int i;
+	struct maple_range_64 *full64 = &mt_to_node(mas->node)->mr64;
+	unsigned long min = mas->min;
+	unsigned long max = min;
+
+	for (i = 3; i < MAPLE_RANGE64_SLOTS - 1; i++) {
+		max = full64->pivot[i];
+		if ((max - min) >= 8)
+			break;
+	}
+	return i;
+}
+void ma_copy(struct ma_state *mas, struct ma_cp *cp)
+{
+	struct maple_range_64 *src64 = &cp->src->mr64;
+	struct maple_range_64 *dst64;
+	unsigned char sloc = cp->src_start; // src location
+	unsigned char dloc = cp->dst_start; // dst location
+
+	if (!cp->dst) {
+		/* Allocate a new node */
+		mas_node_cnt(mas, 1);
+		if (mas_is_err(mas))
+			return;
+		cp->dst = ma_next_alloc(mas);
+	}
+
+	dst64 = &cp->dst->mr64;
+	while (sloc <= cp->src_end &&
+			dloc <= cp->dst_end) {
+
+		if (sloc != 0 && sloc < MAPLE_RANGE64_SLOTS - 2 &&
+		    src64->pivot[sloc] == 0)
+			break;
+
+		if (sloc < MAPLE_RANGE64_SLOTS - 1 &&
+		    dloc < MAPLE_RANGE64_SLOTS - 1)
+			dst64->pivot[dloc] = src64->pivot[sloc];
+		else if (dloc > MAPLE_RANGE64_SLOTS - 1)
+			dst64->pivot[dloc] = mas->max;
+
+		RCU_INIT_POINTER(dst64->slot[dloc], src64->slot[sloc]);
+
+		/* Increment counters */
+		sloc++;
+		dloc++;
+	}
+	cp->dst_start = dloc;
+}
+static int ma_cp_data_64(struct ma_state *mas, struct maple_node *left,
+			  struct maple_node *right, int off)
+{
+
+	int split = ma_calc_split(mas);
+
+	MA_CP(cp, mt_to_node(mas->node), left, off, split);
+	ma_copy(mas, &cp);
+	cp.src_start = split + 1;
+	cp.src_end = MAPLE_RANGE64_SLOTS - 1;
+	cp.dst = right;
+	cp.dst_start = 0;
+	ma_copy(mas, &cp);
+	return split;
+}
+
+static void ma_adopt_children(struct maple_node *parent)
+{
+
+	struct maple_range_64 *p64 = &parent->mr64;
+	struct maple_node *child;
+	unsigned char slot;
+
+	for (slot = 0; slot < MAPLE_RANGE64_SLOTS; slot++) {
+		if (slot != 0 && slot < MAPLE_RANGE64_SLOTS - 1 &&
+		    p64->pivot[slot] == 0)
+			break;
+
+		child = p64->slot[slot];
+		if (child)
+			mt_set_parent(child, parent, slot);
+	}
+}
+
 /* Private
- *
- * Splitting is done in a lazy-fashion.  As such, the parent may not have room
- * for two entries and will require splitting itself.  Rinse & repeat.
+ * Replace mn with mas->node in the tree
  */
-static int ma_split(struct ma_state *mas, unsigned char slot,
-		     unsigned char num)
+void mt_replace(struct ma_state *mas)
 {
-	return 0;
+	struct maple_node *mn = mt_to_node(mas->node);
+	struct maple_node *parent;
+	struct maple_range_64 *p64;
+	unsigned char slot = mt_parent_slot(mn);
+	struct maple_node *prev;
+
+	if (ma_is_root(mas->node)) {
+		prev = mas->tree->ma_root;
+	} else {
+		parent = mt_parent(mas->node);
+		p64 = &parent->mr64;
+		prev = p64->slot[slot];
+	}
+
+	if (prev == mn)
+		return;
+
+	if (!mt_is_leaf(mas->node))
+		ma_adopt_children(mn);
+
+	if (ma_is_root(mas->node)) {
+		mn->parent = (struct maple_node *)
+			      ((unsigned long)mas->tree | MA_ROOT_PARENT);
+		rcu_assign_pointer(mas->tree->ma_root, mt_mk_root(mas->node));
+	} else {
+		rcu_assign_pointer(parent->slot[slot], mas->node);
+	}
+
+	mt_free(mt_to_node(prev));
 }
 /* Private
  *
@@ -346,27 +479,246 @@ static int ma_split(struct ma_state *mas, unsigned char slot,
 static struct maple_range_64 *mas_partial_copy(struct ma_state *mas,
 					       unsigned char end)
 {
-	struct maple_node *smn = mt_to_node(mas->node);
-	struct maple_range_64 *src = &smn->mr64;
-	struct maple_range_64 *dst = NULL;
-	struct maple_node *dmn;
-	int i = 0;
-	/* Allocate a new node */
-	mas_node_cnt(mas, 1);
+	MA_CP(cp, mt_to_node(mas->node), NULL, 0, end);
+	ma_copy(mas, &cp);
 	if (mas_is_err(mas))
 		return NULL;
+	mas->node = mt_mk_node(cp.dst, mt_node_type(mas->node));
+	cp.dst->parent = cp.src->parent;
+	return &cp.dst->mr64;
+}
+
+static void ma_link(struct maple_node *new, struct maple_node *parent,
+		unsigned char slot, unsigned long pivot, enum maple_type type)
+{
+	struct maple_range_64 *p64 = &parent->mr64;
+	struct maple_node *new_enc = mt_mk_node(new, type);
+
+	mt_set_parent(new, parent, slot);
+	p64->pivot[slot] = pivot;
+	RCU_INIT_POINTER(p64->slot[slot], new_enc);
+	if (!mt_is_leaf(new_enc))
+		ma_adopt_children(new);
+
+}
+static int ma_split(struct ma_state *mas, unsigned char slot,
+		     unsigned char num, int depth)
+{
+	struct maple_node *enc_full = mas->node; // Encoded full node.
+	struct maple_node *full = mt_to_node(mas->node);
+	unsigned char split, p_slot = 0, p_end = 0;
+	struct maple_node *old_parent, *new_parent, *left, *right;
+	enum maple_type ctype; // Child type.
+	unsigned long pivot;
+
+	if (ma_is_root(full)) {
+		old_parent = full;
+		mas->node = mt_safe_root(mas->node);
+	} else {
+		old_parent = mt_parent(mas->node);
+		p_end = ma_data_end_r64(&old_parent->mr64, UINT_MAX);
+
+		if (p_end >= MAPLE_RANGE64_SLOTS - 1) {
+			/* Must split the parent */
+			unsigned long max = mas->max;
+			unsigned long min = mas->min;
+
+			ma_encoded_parent(mas);
+			p_slot = mt_parent_slot(enc_full);
+			split = ma_split(mas, p_slot, p_end, 1);
+			if (mas_is_err(mas))
+				return 0;
+			if (split <= p_slot)
+				p_slot -= split;
+			// Split will return the parent.
+			old_parent = mt_to_node(mas->node);
+			mas->node = old_parent->mr64.slot[p_slot];
+		}
+	}
+
+	enc_full = mas->node;
+	mas_node_cnt(mas, 3);
+	if (mas_is_err(mas))
+		return 0;
+
+	// Allocations.
+	right = ma_next_alloc(mas);
+	left = ma_next_alloc(mas);
+	new_parent = ma_next_alloc(mas);
+
+	// Record the node type for the children types.
+	ctype = mt_node_type(enc_full);
+	// copy the data and calculate the split location.
+	split = ma_cp_data_64(mas, left, right, 0);
+	// Copy the parents information
+	if (!ma_is_root(full)) {
+		// Copy the parent data and leave a hole.
+		p_slot = mt_parent_slot(enc_full);
+		MA_CP(cp, old_parent, new_parent, 0, p_slot);
+		ma_copy(mas, &cp);
+		cp.dst_start += 1;
+		cp.src_start = p_slot + 1;
+		cp.src_end = p_end;
+		ma_copy(mas, &cp);
+		// Update encoded slots in children
+		ma_adopt_children(new_parent);
+	}
+
+	// Copy grand parent to the parent, including slot encoding.
+	new_parent->parent = old_parent->parent;
+	// Set up the link to the right in the new parent
+	if (ma_is_root(full))
+		pivot = ULONG_MAX;
+	else
+		pivot = old_parent->mr64.pivot[p_slot];
+
+	ma_link(right, new_parent, p_slot + 1, pivot, ctype);
+
+	// Set up the link to the left in the new parent
+	if (split < MAPLE_RANGE64_SLOTS - 1)
+		pivot = full->mr64.pivot[split];
+	else
+		pivot = left->mr64.pivot[split - 1];
+
+	ma_link(left, new_parent, p_slot, pivot, ctype);
+
+	// Set up maple state for replacement of node.
+	// Note: old_parent may be the old parent or the full node.
+	if (ma_is_root(old_parent)) {
+		mas->node = mt_mk_node(new_parent, maple_range_64);
+	} else {
+		struct maple_node *gparent = mt_parent(old_parent);
+
+		old_parent = gparent->mr64.slot[mt_parent_slot(old_parent)];
+		mas->node = mt_mk_node(new_parent, mt_node_type(old_parent));
+	}
+	// Replace the parent node & free the old parent.
+	mt_replace(mas);
+
+	// Set up the ma_state for the return.  Point to the correct node for
+	// the insert.
+	if (mas->index <= pivot)
+		mas->node = mt_mk_node(left, ctype);
+	else
+		mas->node = mt_mk_node(right, ctype);
+
+	// Free the full node.
+	mt_free(full);
+	return split;
+}
+/* Private
+ *
+ * Splitting is done in a lazy-fashion.  As such, the parent may not have room
+ * for two entries and will require splitting itself.  Rinse & repeat.
+ *
+ *
+ * If this is root, increase the branch height.
+ * If this is not root, promote the middle item as a pivot to the parent.
+ * Copy 1/2 data to each left & right children.
+ *
+ *
+ */
+static int _ma_split(struct ma_state *mas, unsigned char slot,
+		     unsigned char num, int depth)
+{
+	struct maple_node *full = mas->node;
+	struct maple_node *mn = mt_to_node(mas->node);
+	struct maple_node *old_parent, *new_parent, *left, *right;
+	struct maple_range_64 *nparent64;
+	unsigned char p_slot;
+	unsigned char p_end;
+	unsigned char split;
+
+	if (ma_is_root(full)) {
+		old_parent = mn;
+		mas->node = mt_safe_root(mas->node);
+		p_slot = 0;
+		p_end = 0;
+	} else {
+		old_parent = mt_parent(mn);
+		p_slot = mt_parent_slot(mn);
+		p_end = ma_data_end_r64(&old_parent->mr64, UINT_MAX);
+	}
+
+
+	if (p_end >= MAPLE_RANGE64_SLOTS - 1) {
+		mas->node = mn;
+		ma_encoded_parent(mas);
+		unsigned char split = ma_split(mas, p_slot, p_end, depth + 1);
+		if (mas_is_err(mas))
+			return 0;
+
+		if (split < slot)
+			slot -= split;
 
-	dmn = ma_next_alloc(mas);
-	dst = &dmn->mr64;
-	dmn->parent = smn->parent;
-	for (i = 0; i < end;i++)
-	{
-		RCU_INIT_POINTER(dst->slot[i],
-				src->slot[i]);
-		dst->pivot[i] = src->pivot[i];
+		mn = mt_to_node(mas->node);
+		mas->node = full;
+		old_parent = mt_parent(mn);
+		p_slot = mt_parent_slot(mn);
+		p_end = ma_data_end_r64(&old_parent->mr64, UINT_MAX);
+	}
+
+	mas_node_cnt(mas, 3);
+	if (mas_is_err(mas))
+		return 0;
+
+	new_parent = ma_next_alloc(mas);
+	left = ma_next_alloc(mas);
+	right = ma_next_alloc(mas);
+
+	split = ma_cp_data_64(mas, left, right, 0);
+
+	new_parent->parent = old_parent->parent;
+	nparent64 = &new_parent->mr64;
+
+	if (!ma_is_root(full)) {
+		MA_CP(cp, old_parent, new_parent, 0, p_slot);
+		ma_copy(mas, &cp);
+		cp.dst_start += 1;
+		cp.src_start = p_slot+1;
+		cp.src_end = p_end;
+		ma_copy(mas, &cp);
+	}
+
+	mt_set_parent(left, new_parent, p_slot);
+	if (split >= MAPLE_RANGE64_SLOTS - 1)
+		nparent64->pivot[p_slot] = mn->mr64.pivot[split - 1];
+	else
+		nparent64->pivot[p_slot] = mn->mr64.pivot[split];
+
+	RCU_INIT_POINTER(nparent64->slot[p_slot],
+			mt_mk_node(left, mt_node_type(full)));
+
+	mt_set_parent(right, new_parent, ++p_slot);
+	nparent64->pivot[p_slot] = mas->max;
+	RCU_INIT_POINTER(nparent64->slot[p_slot],
+			mt_mk_node(right, mt_node_type(full)));
+
+	if (ma_is_root(old_parent)) {
+		mas->node = mt_mk_node(new_parent, maple_range_64);
+		mas->node = mt_mk_root(mas->node);
+		if (!mt_is_leaf(left)) {
+			ma_adopt_children(left);
+			ma_adopt_children(right);
+		}
+	} else {
+		struct maple_node *gparent = mt_parent(old_parent);
+		old_parent = gparent->mr64.slot[mt_parent_slot(old_parent)];
+		mas->node = mt_mk_node(new_parent, mt_node_type(old_parent));
 	}
-	mas->node = mt_mk_node(dmn, maple_leaf_64);
-	return dst;
+	mt_replace(mas); // Replace old with new parent & free.
+
+	mas->node = nparent64->slot[p_slot-1];
+	mas->max = nparent64->pivot[p_slot-1];
+	if (split <= slot) {
+		mas->node = nparent64->slot[p_slot];
+		mas->min = nparent64->pivot[p_slot-1];
+		mas->max = nparent64->pivot[p_slot];
+	} else {
+	}
+
+	mt_free(mn); // Free full node.
+	return split;
 }
 /* Private
  *
@@ -381,14 +733,15 @@ static struct maple_range_64 *mas_partial_copy(struct ma_state *mas,
  * 4. Write the entry.
  *
  */
-static int _ma_insert(struct ma_state *ms, void *entry, unsigned char slot)
+static int _ma_insert(struct ma_state *mas, void *entry, unsigned char slot)
 {
-	struct maple_range_64 *mr64 = &mt_to_node(ms->node)->mr64;
+	struct maple_range_64 *mr64 = &mt_to_node(mas->node)->mr64;
+	struct maple_node *p_mn;
 	struct maple_range_64 *p_mr64;
-	int o_end = ma_data_end_r64(ms); // Old end
+	int o_end = ma_data_end_r64(mr64, mas->max); // Old end
 	int n_end = o_end;               // New end
-	unsigned long max = ms->max;
-	unsigned long min = ms->min;
+	unsigned long max = mas->max;
+	unsigned long min = mas->min;
 	int ret = 1;
 
 	/* Calculate the range of the slot */
@@ -402,45 +755,59 @@ static int _ma_insert(struct ma_state *ms, void *entry, unsigned char slot)
 		min = mr64->pivot[slot - 1];
 
 	/* Figure out how many slots are needed for the entry. */
-	if (max != ms->last)
+	if (max != mas->last)
 		n_end++;
 
-	if (min != ms->index - 1)
+	if (mas->index && min != mas->index - 1)
 		n_end++;
 
 	if (n_end > MAPLE_RANGE64_SLOTS -1) {
-		return ma_split(ms, slot, n_end);
-	}
+		unsigned char split = ma_split(mas, slot, o_end, 0);
+		if (mas_is_err(mas))
+			return 0;
+
 
+		if (split <= slot)
+			slot -= split;
+
+		n_end -= split;
+		mr64 = &mt_to_node(mas->node)->mr64;
+		o_end = ma_data_end_r64(mr64, mas->max); // Old end is not so old now.
+		if (o_end == 1)
+			o_end = 0; // This node is empty, append at the start.
+	}
 	/* Save the node in case we are not appending. */
 	p_mr64 = mr64;
+	p_mn = mt_to_node(mas->node);
 
 	/* Creates a new node and copies until slot (inclusively) */
 	if (slot == o_end) {
-		/* Appending */
-		o_end = n_end;
+		o_end = n_end; /* Appending */
 	} else {
 		/* Not appending */
-		mr64 = mas_partial_copy(ms, slot);
+		if (p_mr64 == mr64) {
+			mr64 = mas_partial_copy(mas, slot);
+			if (mas_is_err(mas))
+				return 0;
+		}
+
 		o_end = slot;
-		if (mas_is_err(ms))
-			return 0;
 	}
 
-	if (min != ms->index - 1) {
+	if (mas->index && min != mas->index - 1) {
 		/* When writing a NULL entry, the order must be reversed to
 		 * ensure readers don't get incorrect data on appends
 		 */
 		/* Write the entry */
 		RCU_INIT_POINTER(mr64->slot[++slot], entry);
-		mr64->pivot[slot] = ms->last;
+		mr64->pivot[slot] = mas->last;
 		/* Write NULL entry */
 		RCU_INIT_POINTER(mr64->slot[--slot], NULL);
 		if (o_end == n_end) // Append.
 			wmb();
 
-		mr64->pivot[slot] = ms->index - 1;
-		slot+=2;
+		mr64->pivot[slot] = mas->index - 1;
+		slot += 2;
 		ret = 2;
 	} else {
 		/* Write the entry */
@@ -448,28 +815,28 @@ static int _ma_insert(struct ma_state *ms, void *entry, unsigned char slot)
 		if (o_end == n_end) // Append.
 			wmb();
 
-		mr64->pivot[slot++] = ms->last;
+		mr64->pivot[slot++] = mas->last;
 	}
 
 	/* Skip possible duplicate entry that contains a NULL */
-	if (p_mr64->pivot[o_end] == ms->last)
+	if (o_end != n_end && p_mr64->pivot[o_end] == mas->last)
 		o_end++;
 
 	/* Copy remainder of node if this isn't an append */
-	for ( ; o_end < n_end; slot++, o_end++) {
-		if(p_mr64->pivot[o_end] == 0)
-			break;
-		RCU_INIT_POINTER(mr64->slot[slot],
-				p_mr64->slot[o_end]);
-		if (slot < MAPLE_RANGE64_SLOTS - 1)
-			mr64->pivot[slot] = p_mr64->pivot[o_end];
-	}
+	MA_CP(cp, p_mn, mt_to_node(mas->node), o_end, MAPLE_RANGE64_SLOTS - 1);
+	cp.dst_start = slot;
+	cp.dst_end = n_end;
+	ma_copy(mas, &cp);
+
+	if (p_mr64 != mr64)
+		mt_replace(mas);
+
 	return ret;
 }
 
 static void ma_root_expand(struct ma_state *ms, void *entry)
 {
-	void *r_entry = ms->tree->ma_root;  // root entry
+	void *r_entry = rcu_dereference(ms->tree->ma_root);  // root entry
 	struct maple_node *mn;
 
 	mas_node_cnt(ms, 1);
@@ -501,8 +868,7 @@ static void mas_update_limits(struct ma_state *ms, unsigned char slot)
 	if (mas_is_start(ms))
 		return;
 
-	mr64 = &ms->node->mr64;
-
+	mr64 = &mt_to_node(ms->node)->mr64;
 	if (slot > 0)
 		ms->min = mr64->pivot[slot - 1] + 1;
 
@@ -516,10 +882,10 @@ static void mas_update_limits(struct ma_state *ms, unsigned char slot)
  */
 static int mas_coalesce(struct ma_state *mas)
 {
-	struct maple_node *smn = mt_to_node(mas->node);
-	struct maple_range_64 *src = &smn->mr64;
+	struct maple_node *src_mn = mt_to_node(mas->node);
+	struct maple_range_64 *src = &src_mn->mr64;
 	unsigned char s_slot, d_slot = 0;
-	unsigned long last = mas->max;
+	unsigned long last = UINT_MAX;
 	struct maple_range_64 *dst = NULL;
 	int ret = 0;
 
@@ -554,37 +920,26 @@ static int mas_coalesce(struct ma_state *mas)
 		}
 	}
 done:
-	if (dst)
+	if (dst) {
 		ret = s_slot - d_slot;
-	return ret;
-}
-
-static void ma_adopt_children(struct maple_node *parent)
-{
-
-	struct maple_range_64 *p64 = &parent->mr64;
-	struct maple_node *child;
-	unsigned char slot;
-
-	for (slot = 0; slot < MAPLE_RANGE64_SLOTS; slot++) {
-		if (slot < MAPLE_RANGE64_SLOTS - 1 &&
-		    slot != 0 && p64->pivot[slot] == 0)
-			break;
-		child = p64->slot[slot];
-		mt_set_parent(child, parent, slot);
+		mas->node = mt_mk_node(mt_to_node(mas->node),
+				       mt_node_type(mas->node));
+		mt_replace(mas);
 	}
+	return ret;
 }
 
 static bool mas_search_slots(struct ma_state *ms, unsigned long val)
 {
 	struct maple_range_64 *mr64;
 	int i = 0;
+	bool ret = false;
 
 	mr64 = &mt_to_node(ms->node)->mr64;
-	for (i = 0; i <= MAPLE_RANGE64_SLOTS - 1; i++) {
+	for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) {
 		if (i != 0 && mr64->pivot[i] == 0) {
 			ma_set_slot(ms, MAPLE_NODE_SLOTS);
-			return false;
+			return ret;
 		}
 
 		if (val <= mr64->pivot[i])
@@ -596,36 +951,31 @@ static bool mas_search_slots(struct ma_state *ms, unsigned long val)
 			i++;
 	}
 
+	if (mr64->slot[i])
+		ret = true;
+
 	ma_set_slot(ms, i);
-	return true;
+	return ret;
 }
-void mas_traverse(struct ma_state *mas)
+
+bool mas_traverse(struct ma_state *mas)
 {
 	unsigned char slot = ma_get_slot(mas);
 
-	mas->node = mt_to_node(mas->node)->slot[slot];
 	mas_update_limits(mas, slot);
+	if (mt_is_leaf(mas->node))
+		return false;
+	mas->node = mt_to_node(mas->node)->mr64.slot[slot];
+	return true;
 }
 
 bool _mas_walk(struct ma_state *mas)
 {
-	void *p_entry; // Previous entry.
-
 	mas->node = mas_start(mas);
-
-	if (mt_is_leaf(mas->node)) {
-		mas_search_slots(mas, mas->index);
-		return true;
-	}
-
 	do {
-		p_entry = mas->node;
-		if (!mas_search_slots(mas, mas->index)) {
-			mas->node = p_entry;
+		if (!mas_search_slots(mas, mas->index))
 			return mt_is_leaf(mas->node);
-		}
-		mas_traverse(mas);
-	} while (!mt_is_leaf(mas->node));
+	} while (mas_traverse(mas));
 
 	return true;
 }
@@ -643,58 +993,15 @@ bool ma_reserved(void *entry)
 
 	return false;
 }
-/* Private
- * Replace mn with mas->node in the tree
- */
-void mt_may_replace(struct ma_state *mas, void *p_entry, bool leaf)
-{
-	struct maple_node *mn;
-	struct maple_node *pmn = mt_to_node(p_entry);
-
-	/* There is no new node to replace */
-	if (mas->node == p_entry)
-		return;
-
-	mn = mt_to_node(mas->node);
-
-	/* The targeted node was split and is already in the tree */
-	if (mn == mt_parent(pmn))
-		goto in_tree;
-
-	mn->parent = pmn->parent;
-	if (!leaf)
-		ma_adopt_children(mn);
-
-	if (ma_is_root(p_entry)) {
-		mn->parent = (struct maple_node *)
-			      ((unsigned long)mas->tree | MA_ROOT_PARENT);
-		rcu_assign_pointer(mas->tree->ma_root, mt_mk_root(mas->node));
-	} else {
-		struct maple_node *parent = mt_parent(mn->parent);
-		unsigned char slot = mt_parent_slot(mn);
-
-		RCU_INIT_POINTER(parent->slot[slot], mas->node);
-	}
-
-in_tree:
-	mt_free(pmn);
-}
 
 void *ma_insert(struct ma_state *mas, void *entry)
 {
-	void *p_entry; // Previous entry.
 	unsigned char slot = MAPLE_NODE_SLOTS;
 	struct maple_range_64 *src;
 	bool leaf;
 
-
-	if (ma_reserved(entry))
-		mas_set_err(mas, -EINVAL);
-
 	mas->node = mas_start(mas);
-
-
-	if (!xa_is_node(mas->tree->ma_root)) {
+	if (!xa_is_node(rcu_dereference(mas->tree->ma_root))) {
 		if (mas->last == 0) {
 			if (mas->node != NULL)
 				goto exists;
@@ -715,7 +1022,6 @@ void *ma_insert(struct ma_state *mas, void *entry)
 		}
 	}
 
-	p_entry = mas->node;
 	mas_coalesce(mas);
 	if (mas_is_err(mas))
 		goto error;
@@ -725,8 +1031,6 @@ void *ma_insert(struct ma_state *mas, void *entry)
 	if (mas_is_err(mas))
 		goto error;
 
-	/* Replace the node in the tree, if necessary. */
-	mt_may_replace(mas, p_entry, leaf);
 	return NULL;
 
 error:
@@ -750,12 +1054,12 @@ void *mas_walk(struct ma_state *mas)
 	unsigned char slot = MAPLE_NODE_SLOTS;
 	bool leaf = false;
 
-	if (mas->tree->ma_root == NULL)
+	if (rcu_dereference(mas->tree->ma_root) == NULL)
 		return NULL;
 
 	if (!xa_is_node(mas->tree->ma_root)) {
 		if (mas->last == 0)
-			return mas->tree->ma_root;
+			return rcu_dereference(mas->tree->ma_root);
 		return NULL;
 	}
 
@@ -769,7 +1073,7 @@ void *mas_walk(struct ma_state *mas)
 	if (leaf == true && slot != MAPLE_NODE_SLOTS) {
 		struct maple_range_64 mr = mt_to_node(mas->node)->mr64;
 
-		entry = mr.slot[ma_get_slot(mas)];
+		entry = mr.slot[slot];
 	}
 	ma_set_slot(mas, 0);
 	return entry;
@@ -780,12 +1084,14 @@ void *mas_walk(struct ma_state *mas)
  */
 int ma_erase(struct ma_state *mas)
 {
-	bool leaf = _mas_walk(mas);
-	int slot = ma_get_slot(mas);
-	struct maple_range_64 *mr64 = &mt_to_node(mas->node)->mr64;
+	struct maple_range_64 *mr64;
 	unsigned long piv_val;
 	int cnt = -EINVAL;
-	void *p_entry; // Previous entry.
+	int slot;
+
+	_mas_walk(mas);
+	slot = ma_get_slot(mas);
+	mr64 = &mt_to_node(mas->node)->mr64;
 
 	if (slot == MAPLE_NODE_SLOTS)
 		return cnt;
@@ -807,29 +1113,28 @@ int ma_erase(struct ma_state *mas)
 		mr64->pivot[slot] = piv_val;
 		cnt++;
 	}
-	p_entry = mas->node;
+
 	mas_coalesce(mas);
 	if (mas_is_err(mas))
 		goto error;
-
-	mt_may_replace(mas, p_entry, leaf);
-
 error:
 	return cnt;
 }
 
-void ma_destroy_walk(struct ma_state *mas)
+void ma_destroy_walk(struct maple_node *enc_mn)
 {
-	struct maple_node *mn = mas->node;
-	unsigned int type = mt_node_type(mas->node);
+	struct maple_node *mn = mt_to_node(enc_mn);
+	unsigned int type = mt_node_type(enc_mn);
 	int i;
 
 	switch (type) {
 	case maple_range_64:
 		for (i = 0; i < MAPLE_RANGE64_SLOTS; i++) {
-			if (i > 0 && mn->mr64.pivot[i] == 0)
+			if (i > 0 && i < MAPLE_RANGE64_SLOTS - 1 &&
+			    mn->mr64.pivot[i] == 0)
 				break;
-			ma_destroy_walk(mn->mr64.slot[i]);
+			if (rcu_dereference(mn->mr64.slot[i]))
+				ma_destroy_walk(rcu_dereference(mn->mr64.slot[i]));
 		}
 		break;
 	case maple_leaf_64:
@@ -850,7 +1155,7 @@ void mtree_init(struct maple_tree *mt)
 {
 	spin_lock_init(&mt->ma_lock);
 	mt->ma_flags = 0;
-	mt->ma_root = NULL;
+	rcu_assign_pointer(mt->ma_root, NULL);
 }
 
 EXPORT_SYMBOL(mtree_init);
@@ -858,9 +1163,9 @@ EXPORT_SYMBOL(mtree_init);
 void *mtree_load(struct maple_tree *mt, unsigned long index)
 {
 	void *entry;
-	MA_STATE(ms, mt, index, index);
+	MA_STATE(mas, mt, index, index);
 	rcu_read_lock();
-	entry = mas_walk(&ms);
+	entry = mas_walk(&mas);
 	rcu_read_unlock();
 	return entry;
 }
@@ -872,7 +1177,7 @@ int mtree_insert_range(struct maple_tree *mt, unsigned long first,
 
 	MA_STATE(ms, mt, first, last);
 
-	if (WARN_ON_ONCE(xa_is_internal(entry)))
+	if (WARN_ON_ONCE(ma_reserved(entry)))
 		return -EINVAL;
 
 	if (first > last)
@@ -911,13 +1216,13 @@ int mtree_erase(struct maple_tree *mt, unsigned long index)
 
 void mtree_destroy(struct maple_tree *mt)
 {
-	MA_STATE(ms, mt, 0, 0);
 	mtree_lock(mt);
-	if (xa_is_node(mt->ma_root)) {
-		ms.node = mt_to_node(mt->ma_root);
-		ma_destroy_walk(&ms);
+	struct maple_node *destroyed = mt->ma_root;
+	rcu_assign_pointer(mt->ma_root, NULL);
+	if (xa_is_node(destroyed)) {
+		ma_destroy_walk(mt_safe_root(destroyed));
 	}
-	mt->ma_root = NULL;
+	mt->ma_flags = 0;
 	mtree_unlock(mt);
 }
 EXPORT_SYMBOL(mtree_destroy);
diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c
index 4da1e1d36180..73af106a9e4f 100644
--- a/lib/test_maple_tree.c
+++ b/lib/test_maple_tree.c
@@ -219,10 +219,10 @@ static noinline void check_double_insert_leaf(struct maple_tree *mt)
 	MT_BUG_ON(mt, !mtree_empty(mt));
 
 	i = huge;
-	while (i > 0x80) {
-		check_insert(mt, i, xa_mk_value(i));
+	while (i > 4096) {
+		check_insert(mt, i, i);
 		for (j = huge; j >= i; j /= 2) {
-			check_load(mt, j, xa_mk_value(j));
+			check_load(mt, j, j);
 			check_load(mt, j+1, NULL);
 		}
 		i /= 2;
@@ -243,8 +243,9 @@ static int maple_tree_seed(void)
 	pr_info("\nTEST STARTING\n\n");
 
 	mtree_init(&tree);
-	//check_new_node(&tree);
-	//check_double_insert_leaf(&tree);
+	check_new_node(&tree);
+	check_double_insert_leaf(&tree);
+
 	check_load(&tree, set[0], NULL);       // See if 15 -> NULL
 
 	check_insert(&tree, set[9], &tree);     // Insert 0
@@ -325,7 +326,7 @@ static int maple_tree_seed(void)
 	mt_set_non_kernel(1);
 	check_erase(&tree, set[1]);
 	check_load(&tree, set[1], NULL);
-	return 0;
+	check_insert(&tree, set[1], &tree);
 	check_insert(&tree, set[4], ptr); // 1000 < Should split.
 	check_load(&tree, set[0], ptr);
 	check_load(&tree, set[1], &tree);