From: Liam R. Howlett Date: Tue, 11 Feb 2025 17:18:30 +0000 (-0500) Subject: wip, moving to struct splitting X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=79e2bff1cd61d8fb30674044f17d2a3877ad1e08;p=users%2Fjedix%2Flinux-maple.git wip, moving to struct splitting Signed-off-by: Liam R. Howlett --- diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 5f76c8bce6e2..7133cc824235 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -3124,45 +3124,57 @@ struct ma_node_part { unsigned char skip; }; -struct ma_node_state { +struct ma_node_info { struct maple_node *node; struct maple_enode *enode; - unsigned long min, max; + unsigned long min; + unsigned long max; unsigned long max_gap; void __rcu **slots; unsigned long *pivots; unsigned long *gaps; - unsigned char offset; /* Current operating offset */ - unsigned char insert; /* FIXME: Rename to offset and op_off or something */ enum maple_type type; unsigned char end; + unsigned char insert_off; + unsigned char offset; /* Operating position */ bool alloc; }; +struct ma_node_state { + union { + struct ma_node_info *info; + struct ma_node_part *part; + }; + unsigned char start; /* The start offset */ + unsigned char size; /* The size to copy or insert */ + struct ma_node_info *dst; /* The destination node */ + bool use_part; +}; + /* * Unsafe from reader side */ -static inline void mns_set_end(struct ma_node_state *mns) +static inline void mni_set_end(struct ma_node_info *mni) { unsigned char offset; - if (mns->type == maple_arange_64) { - mns->end = ma_meta_end(mns->node, mns->type); + if (mni->type == maple_arange_64) { + mni->end = ma_meta_end(mni->node, mni->type); return; } - offset = mt_pivots[mns->type] - 1; - if (likely(!mns->pivots[offset])) { - mns->end = ma_meta_end(mns->node, mns->type); + offset = mt_pivots[mni->type] - 1; + if (likely(!mni->pivots[offset])) { + mni->end = ma_meta_end(mni->node, mni->type); return; } - if (likely(mns->pivots[offset] == mns->max)) { - mns->end = offset; + if (likely(mni->pivots[offset] == mni->max)) { + mni->end = offset; return; } - mns->end = mt_pivots[mns->type]; + mni->end = mt_pivots[mni->type]; } /* @@ -3174,7 +3186,7 @@ static inline void mns_set_end(struct ma_node_state *mns) */ static __always_inline void mns_node_part_leaf_init(struct ma_node_part *ma_part, - struct ma_wr_state *wr_mas, struct ma_node_state *src) + struct ma_wr_state *wr_mas, struct ma_node_info *src) { ma_part->pos = 0; ma_part->size = 0; @@ -3203,8 +3215,8 @@ void mns_node_part_leaf_init(struct ma_node_part *ma_part, } static inline -void mns_node_part_init(struct ma_node_part *ma_part, - struct ma_node_state *left, struct ma_node_state *right) +void mni_node_part_init(struct ma_node_part *ma_part, + struct ma_node_info *left, struct ma_node_info *right) { ma_part->slots[0] = left->enode; ma_part->pivots[0] = left->max; @@ -3222,8 +3234,8 @@ void mns_node_part_init(struct ma_node_part *ma_part, } static __always_inline -void mns_insert_part(struct ma_node_part *part, - struct ma_node_state *dst) +void mni_insert_part(struct ma_node_part *part, + struct ma_node_info *dst) { printk("insert pos %u/%u %u/%u\n", part->pos, part->size, dst->offset, part->dst_max_off); @@ -3270,60 +3282,97 @@ void mns_insert_part(struct ma_node_part *part, } static inline -void _mns_node_init(struct ma_node_state *mns, struct maple_node *node, +void _mni_node_init(struct ma_node_info *mni, struct maple_node *node, enum maple_type type) { - mns->node = node; - mns->type = type; - mns->max_gap = 0; - mns->offset = 0; - mns->slots = ma_slots(node, type); - mns->pivots = ma_pivots(node, type); - mns->gaps = ma_gaps(node, type); - mns->alloc = false; + mni->max_gap = 0; + mni->insert_off = 255; + mni->offset = 0; + mni->slots = ma_slots(node, type); + mni->pivots = ma_pivots(node, type); + mni->gaps = ma_gaps(node, type); + mni->alloc = false; } static inline -void mns_node_init(struct ma_node_state *mns, struct maple_node *node, +void mni_node_init(struct ma_node_info *mni, struct maple_node *node, enum maple_type type) { - _mns_node_init(mns, node, type); - mns->enode = mt_mk_node(node, type); + mni->node = node; + mni->type = type; + _mni_node_init(mni, node, type); } static inline -void mns_mas_init(struct ma_node_state *mns, struct ma_state *mas) +void mns_mas_init(struct ma_node_state *mns, struct ma_node_info *mni, + struct ma_state *mas) { - struct maple_node *node = mte_to_node(mas->node); - enum maple_type type = mte_node_type(mas->node); + mni->node = mte_to_node(mas->node); + mni->type = mte_node_type(mas->node); + _mni_node_init(mni, mni->node, mni->type); + mni->enode = mas->node; + mni->end = mas->end; + mni->max = mas->max; + mni->min = mas->min; - _mns_node_init(mns, node, type); - mns->enode = mas->node; - mns->insert = mas->offset; + mns->info = mni; + mns->start = 0; + mns->size = 0; + mns->dst = NULL; + mns->use_part = false; +} + +static inline +void mns_mni_init(struct ma_node_state *mns, struct ma_node_info *dst, + unsigned char start, unsigned char len) +{ + mns->dst = ni; + mns->start = start; + mns->size = len + 1; +} + +static inline +bool mns_ends_in_null(struct ma_node_state *mns) +{ + void __rcu **s_slots; + unsigned long *s_piv; + + if (mns->use_part) { + s_slots = ns->part->slots; + s_piv = ns->part->pivots; + } else { + s_slots = ns->info->slots; + s_piv = ns->info->pivots; + } + + if (s_slots[mns->start + mns->size - 1]) + return false; + + return true; } /* FIXME: This comment. * Parent ma_node_state init */ static inline -void mns_pmns_init(struct ma_node_state *mns, struct ma_node_state *pmns, +void mni_pmni_init(struct ma_node_info *mni, struct ma_node_info *pmni, unsigned char p_off, struct maple_tree *mt) { - if (p_off == pmns->end) { - mns->max = pmns->max; - mns->min = pmns->pivots[p_off - 1]; + if (p_off == pmni->end) { + mni->max = pmni->max; + mni->min = pmni->pivots[p_off - 1]; } else { - mns->max = pmns->pivots[p_off]; + mni->max = pmni->pivots[p_off]; if (p_off) - mns->min = pmns->pivots[p_off - 1]; + mni->min = pmni->pivots[p_off - 1]; else - mns->min = pmns->min; + mni->min = pmni->min; } - mns->enode = mt_slot_locked(mt, pmns->slots, p_off); - mns->insert = p_off; - _mns_node_init(mns, mte_to_node(mns->enode), - mte_node_type(mns->enode)); + mni->enode = mt_slot_locked(mt, pmni->slots, p_off); + mni->insert_off = p_off; + _mni_node_init(mni, mte_to_node(mni->enode), + mte_node_type(mni->enode)); } /* @@ -3333,7 +3382,7 @@ void mns_pmns_init(struct ma_node_state *mns, struct ma_node_state *pmns, * */ static __always_inline -void mns_cp(struct ma_node_state *src, struct ma_node_state *dst, +void mni_cp(struct ma_node_info *src, struct ma_node_info *dst, unsigned char len) { unsigned long max; @@ -3380,7 +3429,7 @@ void mns_cp(struct ma_node_state *src, struct ma_node_state *dst, * metadata needs the largest gap for non-leaves. */ static __always_inline -void mns_finalise(struct ma_node_state *p) +void mni_finalise(struct ma_node_info *p) { unsigned long max_gap; unsigned char len; @@ -3508,29 +3557,29 @@ static __always_inline unsigned char mas_wr_new_end(struct ma_wr_state *wr_mas) return new_end; } -static inline void mas_wr_converged(struct ma_node_state *src, - struct ma_node_state *dst, struct ma_node_part *ma_part, +static inline void mas_wr_converged(struct ma_node_info *src, + struct ma_node_info *dst, struct ma_node_part *ma_part, struct ma_state *mas) { - mns_node_init(dst, mas_pop_node(mas), src->type); + mni_node_init(dst, mas_pop_node(mas), src->type); printk("%s: %p -> %p\n", __func__, src->node, dst->node); if (mas->offset) - mns_cp(src, dst, mas->offset); + mni_cp(src, dst, mas->offset); - mns_insert_part(ma_part, dst); + mni_insert_part(ma_part, dst); src->offset += ma_part->skip; if (src->offset <= mas->end) - mns_cp(src, dst, mas->end - src->offset + 1); + mni_cp(src, dst, mas->end - src->offset + 1); dst->node->parent = src->node->parent; - mns_finalise(dst); + mni_finalise(dst); mas_set_height(mas); } -static void mas_wr_split_no_null(struct ma_node_state *src, - struct ma_node_state *left, struct ma_node_state *right, +static void mas_wr_split_no_null(struct ma_node_info *src, + struct ma_node_info *left, struct ma_node_info *right, unsigned char total, struct ma_node_part *ma_part) { if (!ma_is_leaf(src->type)) @@ -3545,11 +3594,11 @@ static void mas_wr_split_no_null(struct ma_node_state *src, if ((end - 1 > min) && (left->offset < mt_slots[left->type])) { if (ma_part->unfinished || - src->insert == src->offset) { + src->insert_off == src->offset) { ma_part->dst_max_off = src->offset; - mns_insert_part(ma_part, left); + mni_insert_part(ma_part, left); } else { - mns_cp(src, left, 1); + mni_cp(src, left, 1); } } else { left->offset--; @@ -3568,48 +3617,48 @@ static void mas_wr_split_no_null(struct ma_node_state *src, right->min = left->max + 1; } -static inline void mns_in_left(struct ma_node_state *src, - struct ma_node_state *left, struct ma_node_state *right, +static inline void mni_in_left(struct ma_node_info *src, + struct ma_node_info *left, struct ma_node_info *right, struct ma_state *mas, unsigned char split, unsigned char new_end, struct ma_node_part *ma_part) { ma_part->dst_max_off = split; if (mas->offset) - mns_cp(src, left, mas->offset); + mni_cp(src, left, mas->offset); - mns_insert_part(ma_part, left); + mni_insert_part(ma_part, left); src->offset+= ma_part->skip; //printk("\t\tsrc L offset adjusted to %u\n", src->offset); if (left->offset <= split) - mns_cp(src, left, split - left->offset + 1); + mni_cp(src, left, split - left->offset + 1); mas_wr_split_no_null(src, left, right, new_end, ma_part); if (ma_part->unfinished) - mns_insert_part(ma_part, right); + mni_insert_part(ma_part, right); right->min = left->max + 1; - mns_cp(src, right, mas->end - src->offset + 1); + mni_cp(src, right, mas->end - src->offset + 1); } -static inline void mns_in_right(struct ma_node_state *src, - struct ma_node_state *left, struct ma_node_state *right, +static inline void mni_in_right(struct ma_node_info *src, + struct ma_node_info *left, struct ma_node_info *right, struct ma_state *mas, unsigned char split, unsigned char new_end, struct ma_node_part *ma_part) { unsigned char cp; cp = mas->offset - split - 1; - mns_cp(src, left, split + 1); + mni_cp(src, left, split + 1); mas_wr_split_no_null(src, left, right, new_end, ma_part); right->min = left->max + 1; if (cp) - mns_cp(src, right, cp); + mni_cp(src, right, cp); - mns_insert_part(ma_part, right); + mni_insert_part(ma_part, right); src->offset+= ma_part->skip; //printk("\t\tsrc offset adjusted to %u\n", src->offset); if (src->offset <= mas->end) - mns_cp(src, right, mas->end - src->offset + 1); + mni_cp(src, right, mas->end - src->offset + 1); } /* @@ -3645,20 +3694,19 @@ unsigned char mas_wr_rebalance_calc(unsigned char data_size, } static inline -void mas_wr_ascend_init(struct ma_state *mas, - struct ma_node_state *ns) +void mas_wr_ascend_init(struct ma_state *mas, struct ma_node_info *ni) { mas_ascend(mas); - mns_mas_init(ns, mas); - ns->min = mas->min; - ns->max = mas->max; + ni->node = mte_to_node(mas->node); + ni->type = mte_node_type(mas->node); + _mni_node_init(ni, ni->node, ni->type); } static void mas_wr_rebalance_nodes( - struct ma_node_state *l_src, - struct ma_node_state *r_src, - struct ma_node_state *left, - struct ma_node_state *right, + struct ma_node_info *l_src, + struct ma_node_info *r_src, + struct ma_node_info *left, + struct ma_node_info *right, bool left_store, unsigned char split, struct ma_node_part *ma_part, @@ -3690,16 +3738,16 @@ static void mas_wr_rebalance_nodes( if (mas_off <= split) { /* Store will end up in left */ printk("%s %d\n", __func__, __LINE__); if (mas_off) - mns_cp(l_src, left, mas_off); + mni_cp(l_src, left, mas_off); ma_part->dst_max_off = split; - mns_insert_part(ma_part, left); + mni_insert_part(ma_part, left); l_src->offset+= ma_part->skip; printk("%d\n", __LINE__); printk("\tright min %lu left max %lu\n", right->min, left->max); if (left->offset <= split) - mns_cp(l_src, left, split - left->offset + 1); + mni_cp(l_src, left, split - left->offset + 1); printk("\tright min %lu left max %lu\n", right->min, left->max); mas_wr_split_no_null(l_src, left, right, @@ -3707,58 +3755,58 @@ static void mas_wr_rebalance_nodes( right->min = left->max + 1; printk("\tright min %lu left max %lu\n", right->min, left->max); if (ma_part->unfinished) - mns_insert_part(ma_part, right); + mni_insert_part(ma_part, right); if (l_end >= l_src->offset) - mns_cp(l_src, right, l_end - l_src->offset + 1); + mni_cp(l_src, right, l_end - l_src->offset + 1); } else { /* Store will end up in right */ printk("%s %d\n", __func__, __LINE__); - mns_cp(l_src, left, split + 1); + mni_cp(l_src, left, split + 1); mas_wr_split_no_null(l_src, left, right, r_end + new_end + 1, ma_part); right->min = left->max + 1; - mns_cp(l_src, right, mas_off - l_src->offset); - mns_insert_part(ma_part, right); + mni_cp(l_src, right, mas_off - l_src->offset); + mni_insert_part(ma_part, right); //printk("%d\n", __LINE__); l_src->offset+= ma_part->skip; if (l_end >= l_src->offset) - mns_cp(l_src, right, l_end - l_src->offset + 1); + mni_cp(l_src, right, l_end - l_src->offset + 1); } - mns_cp(r_src, right, r_end + 1); + mni_cp(r_src, right, r_end + 1); } else { /* Store is targeting r_src */ printk("%s %d\n", __func__, __LINE__); if (split <= l_end) { /* Store will end up in right */ printk("%s %d\n", __func__, __LINE__); - mns_cp(l_src, left, split + 1); + mni_cp(l_src, left, split + 1); mas_wr_split_no_null(l_src, left, right, l_end + new_end + 1, ma_part); - mns_cp(l_src, right, l_end - l_src->offset + 1); + mni_cp(l_src, right, l_end - l_src->offset + 1); right->min = left->max + 1; - mns_cp(r_src, right, mas_off); - mns_insert_part(ma_part, right); + mni_cp(r_src, right, mas_off); + mni_insert_part(ma_part, right); //printk("%d\n", __LINE__); r_src->offset+= ma_part->skip; if (r_src->offset <= r_end) - mns_cp(r_src, right, r_end - r_src->offset + 1); + mni_cp(r_src, right, r_end - r_src->offset + 1); } else { /* Store will end up in left */ unsigned char r_split; printk("%s %d\n", __func__, __LINE__); r_split = split - l_end - 1; - mns_cp(l_src, left, l_end + 1); + mni_cp(l_src, left, l_end + 1); if (mas_off <= r_split) { if (mas_off) - mns_cp(r_src, left, mas_off); + mni_cp(r_src, left, mas_off); ma_part->dst_max_off = split; - mns_insert_part(ma_part, left); + mni_insert_part(ma_part, left); //printk("%d\n", __LINE__); r_src->offset+= ma_part->skip; if (r_src->offset < r_split) - mns_cp(r_src, left, + mni_cp(r_src, left, r_split - r_src->offset); mas_wr_split_no_null(r_src, left, right, @@ -3766,26 +3814,86 @@ static void mas_wr_rebalance_nodes( ma_part); if (ma_part->unfinished) - mns_insert_part(ma_part, right); + mni_insert_part(ma_part, right); right->min = left->max + 1; } else { - mns_cp(r_src, left, r_split + 1); + mni_cp(r_src, left, r_split + 1); mas_wr_split_no_null(r_src, left, right, l_end + new_end + 1, ma_part); right->min = left->max + 1; if (mas_off > r_src->offset) - mns_cp(r_src, right, + mni_cp(r_src, right, mas_off - r_src->offset); - mns_insert_part(ma_part, right); + mni_insert_part(ma_part, right); //printk("%d\n", __LINE__); r_src->offset+= ma_part->skip; } if (r_src->offset <= r_end) - mns_cp(r_src, right, r_end - r_src->offset + 1); + mni_cp(r_src, right, r_end - r_src->offset + 1); + } + } +} + +/* + * Assemble all the states into the dst within those states + * + */ +static inline +void mns_assemble(struct ma_node_state **states, unsigned char len) +{ + unsigned char i; + unsigned long max; + struct ma_node_info *last_dst; + + last_dst = states[0]->dst; + for (i = 0; i < len; i++) { + struct ma_node_state *ns = states[i]; + void __rcu **s_slots, **d_slots; + unsigned long *s_piv, *d_piv; + unsigned long *s_gap, *d_gap; + unsigned char size; + unsigned char piv_overflow = 0; + + if (last_dst != ns->dst) + ns->dst->min = max + 1; + + d_slots = ns->dst->slots + ns->dst->offset; + d_piv = ns->dst->pivots + ns->dst->offset; + d_gap = ns->dst->gaps + ns->dst->offset; + size = ns->size; + + if (ns->use_part) { + s_slots = ns->part->slots + ns->start; + s_piv = ns->part->pivots + ns->start; + s_gap = ns->part->gaps + ns->start; + max = s_piv[size - 1]; + } else { + s_slots = ns->info->slots + ns->start; + s_piv = ns->info->pivots + ns->start; + s_gap = ns->info->gaps + ns->start; + if (ns->start + size >= mt_pivots[ns->info->type]) { + piv_overflow = 1; + max = ns->info->max; + } else + max = s_piv[size - 1]; } + + memcpy(d_slots, s_slots, size * sizeof(void __rcu *)); + if (d_gap) + memcpy(d_gap, s_gap, size * sizeof(unsigned long)); + + if (ns->dst->offset + size >= mt_pivots[ns->dst->type]) + piv_overflow = 1; + else if (piv_overflow) /* Source overflow */ + *(d_piv + size - 1) = max; + + ns->dst->offset += size; + size -= piv_overflow; + memcpy(d_piv, s_piv, size * sizeof(unsigned long)); + ns->dst->max = max; } } @@ -3800,13 +3908,14 @@ static void mas_wr_rebalance_nodes( * Returns: True on rebalance, false otherwise. */ static bool mas_wr_try_rebalance(struct ma_state *mas, - struct ma_node_state *src, unsigned char new_end, - struct ma_node_state *left, struct ma_node_state *right, + struct ma_node_info *src, unsigned char new_end, + struct ma_node_info *left, struct ma_node_info *right, struct ma_node_part *ma_part) { struct ma_state tmp_mas; - struct ma_node_state src2, parent, new_parent; - struct ma_node_state *l_src, *r_src; + struct ma_node_info src2_info, parent, new_parent; + struct ma_node_info *l_src, *r_src; + struct ma_node_state src2; unsigned char split, max; unsigned char p_end, p_off; bool left_store = false; @@ -3831,12 +3940,10 @@ static bool mas_wr_try_rebalance(struct ma_state *mas, tmp_mas.offset--; mas_descend(&tmp_mas); - mns_mas_init(&src2, &tmp_mas); - src2.max = tmp_mas.max; - src2.min = tmp_mas.min; - src2.insert = 255; - src2.end = ma_data_end(src2.node, src2.type, src2.pivots, - src2.max); + mns_mas_init(&src2, &src2_info, &tmp_mas); + src2_info.insert_off = 255; + src2_info.end = ma_data_end(src2.node, src2.type, src2.pivots, + src2.max); split = mas_wr_rebalance_calc(src2.end + new_end, src2.type); if (split) { p_off--; @@ -3852,8 +3959,6 @@ try_right: tmp_mas.offset = p_off + 1; mas_descend(&tmp_mas); mns_mas_init(&src2, &tmp_mas); - src2.min = tmp_mas.min; - src2.max = tmp_mas.max; src2.insert = 255; src2.end = ma_data_end(src2.node, src2.type, src2.pivots, src2.max); @@ -3876,12 +3981,12 @@ try_right: mas_wr_rebalance_nodes(l_src, r_src, left, right, left_store, split, ma_part, mas->offset, new_end); - mns_finalise(left); - mns_finalise(right); + mni_finalise(left); + mni_finalise(right); mas_ascend(mas); mas->end = p_end; mas->offset = p_off; - mns_node_part_init(ma_part, left, right); + mni_node_part_init(ma_part, left, right); ma_part->skip = 2; mas_wr_converged(&parent, &new_parent, ma_part, mas); src->enode = parent.enode; @@ -3902,6 +4007,7 @@ static void mas_wr_rebalance(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; struct ma_node_state src, parent, l_src, r_src; + struct ma_node_info src_info; struct ma_node_state left, right; struct ma_node_part ma_part; unsigned char total, split, height; @@ -3910,10 +4016,7 @@ static void mas_wr_rebalance(struct ma_wr_state *wr_mas) trace_ma_op(__func__, mas); mt_dump(mas->tree, mt_dump_dec); height = mas_mt_height(mas); - mns_mas_init(&src, mas); - src.min = mas->min; - src.max = mas->max; - src.end = mas->end; + mns_mas_init(&src, &src_info, mas); mns_node_part_leaf_init(&ma_part, wr_mas, &src); /* Total will lack sibling data until the sibling is known */ printk("end %p %u\n", mas->node, mas->end); @@ -3961,7 +4064,7 @@ static void mas_wr_rebalance(struct ma_wr_state *wr_mas) } printk("\tNew total will be %u\n", total); - mns_node_init(&left, mas_pop_node(mas), l_src.type); + mni_node_init(&left, mas_pop_node(mas), l_src.type); /* * Two possibilities: * 1. keep two nodes if possible and limit ripple @@ -3977,7 +4080,7 @@ static void mas_wr_rebalance(struct ma_wr_state *wr_mas) * operation stops early. */ - mns_node_init(&right, mas_pop_node(mas), r_src.type); + mni_node_init(&right, mas_pop_node(mas), r_src.type); printk("new right will be %p\n", right.node); split = mas_wr_rebalance_calc(total, l_src.type); printk("split is %u\n", split); @@ -3986,9 +4089,9 @@ static void mas_wr_rebalance(struct ma_wr_state *wr_mas) l_store, split, &ma_part, insert, total); - mns_finalise(&left); - mns_finalise(&right); - mns_node_part_init(&ma_part, &left, &right); + mni_finalise(&left); + mni_finalise(&right); + mni_node_part_init(&ma_part, &left, &right); ma_part.skip = 2; mas_wr_converged(&parent, &new_parent, &ma_part, mas); src.enode = parent.enode; @@ -4003,26 +4106,26 @@ static void mas_wr_rebalance(struct ma_wr_state *wr_mas) /* Reduce two nodes into one */ if (l_store) { if (l_src.insert) - mns_cp(&l_src, &left, l_src.insert); - mns_insert_part(&ma_part, &left); + mni_cp(&l_src, &left, l_src.insert); + mni_insert_part(&ma_part, &left); l_src.offset += ma_part.skip; if (l_src.offset <= l_src.end) - mns_cp(&l_src, &left, + mni_cp(&l_src, &left, l_src.end - l_src.offset + 1); - mns_cp(&r_src, &left, r_src.end); + mni_cp(&r_src, &left, r_src.end); } else { - mns_cp(&l_src, &left, l_src.end); + mni_cp(&l_src, &left, l_src.end); if (r_src.insert) - mns_cp(&r_src, &left, r_src.insert); - mns_insert_part(&ma_part, &left); + mni_cp(&r_src, &left, r_src.insert); + mni_insert_part(&ma_part, &left); r_src.offset += ma_part.skip; if (r_src.offset <= r_src.end) - mns_cp(&r_src, &left, + mni_cp(&r_src, &left, r_src.end - r_src.offset + 1); } left.node->parent = l_src.node->parent; - mns_finalise(&left); + mni_finalise(&left); if (mte_is_root(parent.enode)) { /* Height reduction */ if (mas->depth) @@ -4047,7 +4150,10 @@ static void mas_wr_rebalance(struct ma_wr_state *wr_mas) static void mas_wr_split(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; - struct ma_node_state src, parent, left, right; + struct ma_node_state state[4]; + unsigned char i = 0; + struct ma_node_info src_info, parent, left, right; + struct ma_node_state src; struct ma_node_part ma_part; int height; unsigned char split, total; @@ -4060,12 +4166,10 @@ static void mas_wr_split(struct ma_wr_state *wr_mas) mas->depth = height; /* First split the leaves */ - mns_node_init(&left, mas_pop_node(mas), wr_mas->type); - mns_node_init(&right, mas_pop_node(mas), wr_mas->type); - mns_mas_init(&src, mas); - src.max = mas->max; - src.min = mas->min; - mns_node_part_leaf_init(&ma_part, wr_mas, &src); + mni_node_init(&left, mas_pop_node(mas), wr_mas->type); + mni_node_init(&right, mas_pop_node(mas), wr_mas->type); + mns_mas_init(&src, &src_info, mas); + mns_node_part_leaf_init(&ma_part, wr_mas, &src_info); total = mas->end + ma_part.size - 1; split = (total + 1) / 2; @@ -4073,19 +4177,73 @@ static void mas_wr_split(struct ma_wr_state *wr_mas) right.alloc = left.alloc = true; if (height > 1 && - mas_wr_try_rebalance(mas, &src, total, &left, &right, &ma_part)) + mas_wr_try_rebalance(mas, &src, total, &left, &right, &ma_part)) goto rebalanced; left.min = mas->min; right.max = mas->max; - if (split >= mas->offset) - mns_in_left(&src, &left, &right, mas, split, total, &ma_part); - else - mns_in_right(&src, &left, &right, mas, split, total, &ma_part); + if (split >= mas->offset) { + unsigned char space; /* size remaining in left */ + + if (mas->offset) { + state[i]->info = &src; + mns_mni_init(state[i++], &left, 0, mas->offset); + } - mns_finalise(&left); - mns_finalise(&right); - mns_node_part_init(&ma_part, &left, &right); + state[i]->part = &ma_part; + state[i]->part = true; + space = split - mas->offset + 1; + if (space >= ma_part.size) + mns_mni_init(state[i++], &left, 0, ma_part.size); + } else { + mns_mni_init(state[i++], &left, 0, space); + state[i]->part = &ma_part; + state[i]->part = true; + mns_mni_init(state[i++], &right, space - 1, ma_part.size); + + } + + if (split > wr_mas->offset_end) { + state[i]->info = &src; + mns_mni_init(state[i++], &left, wr_mas->offset_end + 1, + split - wr_mas->offset_end + 1); + } + + if (mns_ends_in_null(state[i - 1])) { + state[i - 1]->size--; + split--; + } + + state[i]->info = &src; + mns_min_init(state[i++], &right, split + 1, mas-end - split + 1); + } else { + state[i]->info = &src; + mns_mni_init(state[i++], &left, 0, split); + if (mns_ends_in_null(state[i - 1])) { + state[i - 1]->size--; + split--; + } + + if (mas->offset > split + 1) { + state[i]->info = &src; + mns_mni_init(state[i++], &right, split + 1, + mas->offset - split + 1); + } + state[i]->part = &ma_part; + mns_mni_init(state[i], &right, 0, ma_part.size); + state[i++]->part = true; + if (wr_mas->offset_end > mas->end) { + state[i]->info = &src; + mns_mni_init(state[i++], &right, wr_mas->offset_end + 1, + mas->end - wr_mas->offset_end + 1); + } + } + + mns_assemble(&state, i); + + mni_finalise(&left); + mni_finalise(&right); + mni_node_part_init(&ma_part, &left, &right); if (height == 1) { if (mt_is_alloc(mas->tree)) @@ -4098,32 +4256,45 @@ static void mas_wr_split(struct ma_wr_state *wr_mas) //printk("%d height is %d\n", __LINE__, height); while (--height) { + i = 0; mas_wr_ascend_init(mas, &src); mas->end = ma_data_end(src.node, src.type, src.pivots, - src.max); + src.max); total = mas->end + 1; if (mas->end + 1 < mt_slots[src.type]) goto converged; //printk("\tConsume %p type %u\n", src.node, src.type); - mns_node_init(&left, mas_pop_node(mas), src.type); - mns_node_init(&right, mas_pop_node(mas), src.type); + mni_node_init(&left, mas_pop_node(mas), src.type); + mni_node_init(&right, mas_pop_node(mas), src.type); if ((height > 1) && - (mas_wr_try_rebalance(mas, &src, mas->end + 1, &left, - &right, &ma_part))) + (mas_wr_try_rebalance(mas, &src, mas->end + 1, &left, + &right, &ma_part))) goto rebalanced; left.min = src.min; right.max = src.max; split = (total + 1) / 2; + + if (split >= mas->offset) { + if (mas->offset) { + state[i]->info = &src; + msn_mni_init(state[i++], &left, 0, mas->offset); + } + } + + + /* Old if (split >= mas->offset) - mns_in_left(&src, &left, &right, mas, split, total, &ma_part); + mni_in_left(&src, &left, &right, mas, split, total, &ma_part); else - mns_in_right(&src, &left, &right, mas, split, total, &ma_part); + mni_in_right(&src, &left, &right, mas, split, total, &ma_part); + */ - mns_finalise(&left); - mns_finalise(&right); - mns_node_part_init(&ma_part, &left, &right); + mns_assemble(&state, i); + mni_finalise(&left); + mni_finalise(&right); + mni_node_part_init(&ma_part, &left, &right); } new_root: