From: Liam R. Howlett Date: Mon, 13 Jan 2020 15:35:30 +0000 (-0500) Subject: maple_tree: WIP, almost there. maple_tree: 653130 of 653130 tests passed X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=67411d57e2f3390058b23b842dcc5fdb0572d455;p=users%2Fjedix%2Flinux-maple.git maple_tree: WIP, almost there. maple_tree: 653130 of 653130 tests passed Signed-off-by: Liam R. Howlett --- diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 5640c8b3309c1..367fc5b0cccb6 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -87,10 +87,10 @@ unsigned char mt_min_slots[] = { [maple_sparse_64] = MAPLE_SPARSE64_SLOTS / 2, [maple_leaf_16] = MAPLE_RANGE16_SLOTS / 2, [maple_leaf_32] = MAPLE_RANGE32_SLOTS / 2, - [maple_leaf_64] = MAPLE_RANGE64_SLOTS / 2, + [maple_leaf_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, [maple_range_16] = MAPLE_RANGE16_SLOTS / 2, [maple_range_32] = MAPLE_RANGE32_SLOTS / 2, - [maple_range_64] = MAPLE_RANGE64_SLOTS / 2, + [maple_range_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, [maple_arange_64] = (MAPLE_ARANGE64_SLOTS + 1) / 2, }; #define mt_min_slot_cnt(x) mt_min_slots[mte_node_type(x)] @@ -569,40 +569,46 @@ static inline void mas_set_safe_pivot(struct ma_state *mas, unsigned char slot, } mte_set_pivot(mas->node, slot, val); } -static inline struct maple_enode *_mte_get_rcu_slot( - const struct maple_enode *mn, unsigned char slot, +static inline struct maple_enode *ma_get_rcu_slot( + const struct maple_node *mn, unsigned char slot, enum maple_type type) { switch (type) { case maple_range_64: case maple_leaf_64: - return rcu_dereference(mte_to_node(mn)->mr64.slot[slot]); + return rcu_dereference(mn->mr64.slot[slot]); default: case maple_dense: - return rcu_dereference(mte_to_node(mn)->slot[slot]); + return rcu_dereference(mn->slot[slot]); case maple_arange_64: - return rcu_dereference(mte_to_node(mn)->ma64.slot[slot]); + return rcu_dereference(mn->ma64.slot[slot]); case maple_sparse_6: - return rcu_dereference(mte_to_node(mn)->ms6.slot[slot]); + return rcu_dereference(mn->ms6.slot[slot]); case maple_sparse_9: - return rcu_dereference(mte_to_node(mn)->ms9.slot[slot]); + return rcu_dereference(mn->ms9.slot[slot]); case maple_sparse_16: - return rcu_dereference(mte_to_node(mn)->ms16.slot[slot]); + return rcu_dereference(mn->ms16.slot[slot]); case maple_sparse_21: - return rcu_dereference(mte_to_node(mn)->ms21.slot[slot]); + return rcu_dereference(mn->ms21.slot[slot]); case maple_sparse_32: - return rcu_dereference(mte_to_node(mn)->ms32.slot[slot]); + return rcu_dereference(mn->ms32.slot[slot]); case maple_sparse_64: - return rcu_dereference(mte_to_node(mn)->ms64.slot[slot]); + return rcu_dereference(mn->ms64.slot[slot]); case maple_range_16: case maple_leaf_16: - return rcu_dereference(mte_to_node(mn)->mr16.slot[slot]); + return rcu_dereference(mn->mr16.slot[slot]); case maple_range_32: case maple_leaf_32: - return rcu_dereference(mte_to_node(mn)->mr32.slot[slot]); + return rcu_dereference(mn->mr32.slot[slot]); } } +static inline struct maple_enode *_mte_get_rcu_slot( + const struct maple_enode *mn, unsigned char slot, + enum maple_type type) +{ + return ma_get_rcu_slot(mte_to_node(mn), slot, type); +} static inline struct maple_enode *mte_get_rcu_slot(const struct maple_enode *mn, unsigned char slot) { @@ -813,32 +819,73 @@ bool mas_retry(struct ma_state *mas, const void *entry) static inline void mas_ascend(struct ma_state *mas) { - struct maple_enode *p_enode; - struct maple_node *p_node; - unsigned char p_slot; - enum maple_type p_type; + struct maple_enode *p_enode; // parent enode. + struct maple_enode *a_enode = mas->node; // ancestor enode. + struct maple_node *a_node = mas_mn(mas); // ancestor node. + unsigned char a_slot = 0; + enum maple_type a_type = mte_node_type(mas->node); + unsigned long max = 0, min = ULONG_MAX; + bool set_max = false, set_min = false; + + //mt_dump(mas->tree); + //printk("ascending from %p\n", mas_mn(mas)); + p_enode = mt_mk_node(mte_parent(mas->node), + mas_parent_enum(mas, mas->node)); - p_type = mas_parent_enum(mas, mas->node); - p_node = mte_parent(mas->node); - p_slot = mte_parent_slot(mas->node); - p_enode = mt_mk_node(p_node, p_type); - if (_ma_is_root(p_node)) { - mas->min = 0; - mas->max = mt_max[p_type]; - } else if (!p_slot || p_slot >= mt_pivots[p_type]) { - mas->node = p_enode; - mas_ascend(mas); - mas_set_slot(mas, p_slot); - mas_descend(mas); - } else { - struct maple_enode *gp_enode = mt_mk_node(mte_parent(p_enode), - mas_parent_enum(mas, p_enode)); - unsigned char gp_slot = mte_parent_slot(p_enode); - mas->node = gp_enode; - mas_set_slot(mas, gp_slot); - mas_update_limits(mas, gp_slot, mas_parent_enum(mas, p_enode)); + + if (_ma_is_root(a_node)) + goto no_parent; + + a_type = mas_parent_enum(mas, mas->node); + a_enode = p_enode; + + if (mte_is_root(a_enode)) { + a_node = mte_to_node(a_enode); + //printk("parent is root\n"); + goto no_parent; + } + + mas->node = p_enode; +ascend: + a_type = mas_parent_enum(mas, mas->node); + a_node = mte_parent(mas->node); + a_slot = mte_parent_slot(mas->node); + a_enode = mt_mk_node(a_node, a_type); + + //printk("a_slot = %u %s %s %u\n", a_slot, set_min ? "yes" : "no", + // set_max ? "yes" : "no", mt_pivots[a_type]); + if (!set_min && a_slot) { + set_min = true; + min = mte_get_pivot(a_enode, a_slot - 1) + 1; + //printk("Using min from %p[%u] %lu\n", a_node, a_slot-1, min); + } + + if (!set_max && a_slot < mt_pivots[a_type]) { + set_max = true; + //printk("Getting pivot %u from %p\n", a_slot, a_enode); + max = mte_get_pivot(a_enode, a_slot); + //printk("Using max from %p[%u] %lu\n", a_node, a_slot, max); } + +no_parent: + if (_ma_is_root(a_node)) { + //printk("%s: hit root %p\n", __func__, a_node); + if (!set_min) + min = 0; + if (!set_max) + max = mt_max[a_type]; + } + + if (!max || min == ULONG_MAX) { + mas->node = a_enode; + goto ascend; + } + + mas->max = max; + mas->min = min; mas->node = p_enode; + //printk("%p %lu-%lu\n", mas_mn(mas), mas->min, mas->max); + return; } /* mas_get_prev_pivot() - Return the previous pivot. @@ -1099,7 +1146,6 @@ static inline unsigned char mas_data_end(const struct ma_state *mas, void *entry; piv = _mas_get_safe_pivot(mas, slot, type); - printk("%p[%u] piv %lu\n", mas_mn(mas), slot, piv); if (!piv && slot) { // Past the end of data. slot--; piv = prev_piv; @@ -1111,8 +1157,8 @@ static inline unsigned char mas_data_end(const struct ma_state *mas, (*coalesce) = 0; break; } - printk("data end with counted nulls %p\n", mas_mn(mas)); - printk("%u %u %u\n", slot, (*coalesce), counted_null); +// printk("data end with counted nulls %p\n", mas_mn(mas)); +// printk("%u %u %u\n", slot, (*coalesce), counted_null); (*coalesce) = (*coalesce) - counted_null + 1; slot -= counted_null; } @@ -1135,14 +1181,14 @@ static inline unsigned char mas_data_end(const struct ma_state *mas, } if (piv == mas->max) { - printk("max hit %lu\n", mas->max); +// printk("max hit %lu\n", mas->max); break; } prev_piv = piv; } - printk("last piv %lu slot %u coalesce %u\n", piv, slot, *coalesce); +// printk("last piv %lu slot %u coalesce %u\n", piv, slot, *coalesce); *last_piv = piv; return slot; } @@ -1168,7 +1214,7 @@ static inline struct maple_node *mas_switch_nodes(struct ma_state *cp, unsigned long piv) { left->max = piv; - printk("going to use %lu as max\n", left->max); +// printk("going to use %lu as max\n", left->max); right->min = piv + 1; mas_dup_state(cp, right); return mas_mn(right); @@ -1178,13 +1224,15 @@ static inline void mas_ma_cp_store(struct maple_node *mn, enum maple_type type, unsigned char slot, unsigned long piv, void *entry, bool flush) { - //printk("set %p[%u] -> %p\n", mn, slot, entry); + printk("%s: set %p[%u] -> %p\n", __func__, mn, slot, entry); ma_set_rcu_slot(mn, slot, type, entry); if (flush) wmb(); // data needs to exist before pivot for readers - if (slot < mt_pivots[type]) + if (slot < mt_pivots[type]) { + printk("%s: set %p[%u] piv %lu\n", __func__, mn, slot, piv); ma_set_pivot(mn, slot, type, piv); + } } static inline unsigned char mas_cp_calc_split(struct ma_state *mas, @@ -1258,9 +1306,19 @@ static inline void mas_ma_update_gap(struct ma_state *mas, * mas_ma_cp() - copy mas->node to mn (or left->node and right->node for * splits). * + * @mas - src + * @p_slot - parent slot - used to update gaps on split. + * @left - destination 1 mas + * @right - destination 2 mas - if not null, treat this as a split call. + * @mn - destination 1 node - if null, taken from @left->node - exists for + * inactive (unallocated) nodes. + * @mn_type - destination 1 node type - same as @mn, taken from @left->node. + * @entry - entry to insert at mas.index/mas.last + * @dst_start- destination start slot. If non-zero, implies append mode. + * * Trying to calculate a split is a rather difficult task. One has to * remember: - * 1. Keep gaps at the end of a node + * 1. Keep gaps at the end of a node - might not be done right now? * 2. Coalescing may affect the middle position of data * 3. Use the target slot and the new count to better get a 50/50 split. * @@ -1268,8 +1326,8 @@ static inline void mas_ma_update_gap(struct ma_state *mas, static inline unsigned char mas_ma_cp(struct ma_state *mas, unsigned char p_slot, struct ma_state *left, struct ma_state *right, struct maple_node *mn, - enum maple_type mn_type, int entry_cnt, void *entry, - bool append) + enum maple_type mn_type, unsigned long mn_min, + int entry_cnt, void *entry, unsigned char dst_start) { enum maple_type mas_type = mte_node_type(mas->node); struct maple_node *parent = mte_to_node(mas->node); @@ -1280,8 +1338,10 @@ static inline unsigned char mas_ma_cp(struct ma_state *mas, unsigned long max_gap = 0; bool attempt_insert = false; bool appending = false; + bool append = false; void *existing = NULL; bool null_run = false; + bool prev_null = false; bool update_gaps = mt_is_alloc(mas->tree); unsigned long piv[3] = { mas->index - 1, @@ -1291,13 +1351,16 @@ static inline unsigned char mas_ma_cp(struct ma_state *mas, MA_STATE(cp, mas->tree, mas->index, mas->last); + if (dst_start) + append = true; + if (ma_is_leaf(mas_type)) attempt_insert = true; if (right) { split = mas_cp_calc_split(mas, mas_type, append); - // printk("Using split of %d\n", split); + printk("Using split of %d\n", split); } if (left) { @@ -1309,32 +1372,48 @@ static inline unsigned char mas_ma_cp(struct ma_state *mas, mn_type = mte_node_type(cp.node); p_slot = mte_parent_slot(left->node); parent = mte_parent(left->node); + printk("parent %p of %p\n", parent, mas->node); mas_type = mas_parent_enum(mas, left->node); - // printk("parent is %p\n", parent); } if (!entry_cnt) piv[2] = mas->max; if (append) { // start at a given slot, so append there. - // printk("append\n"); + printk("append\n"); //mt_dump(mas->tree); mas_dup_state(&cp, mas); src_slot = entry_cnt; - existing = mte_get_rcu_slot(mas->node, src_slot); - written_piv = mas_get_safe_pivot(mas, src_slot); - if (!written_piv) // empty node. - written_piv = mas->min; - - // printk("src_slot %u written %lu\n", src_slot, written_piv); - slot = entry_cnt; - if (existing) { + slot = dst_start; + if (mn == mas_mn(&cp)) // src and dst are the same. + slot = entry_cnt; + + existing = ma_get_rcu_slot(mn, slot, mn_type); + written_piv = mn_min; + if (slot) + written_piv = ma_get_pivot(mn, slot, mn_type); + + printk("slot %u written %lu src_slot %u\n", slot, written_piv, src_slot); + if (mn == mas_mn(&cp)) { // src and dst are the same. + printk("src == dst\n"); + if (existing || (!slot && written_piv < cp.index)) { + src_slot++; + slot++; + existing = NULL; + } + entry_cnt = 0; + } else if (existing && written_piv != cp.index) { slot++; + entry_cnt = -1; + printk("Existing\n"); + } else { src_slot++; - // printk("there is an existing\n"); + entry_cnt = 0; } - entry_cnt = 0; + + printk("slot %u written %lu src_slot %u\n", slot, written_piv, src_slot); + appending = true; prev_piv = written_piv; attempt_insert = true; @@ -1342,13 +1421,14 @@ static inline unsigned char mas_ma_cp(struct ma_state *mas, max_gap = mas_leaf_max_gap(mas); } + printk("slot is %u\n", slot); do { int i = 2; // Default to just writing the pivot. int j = 3; // Loop limit. - // printk("src slot %u entry_cnt %d slot %u\n", src_slot, entry_cnt, slot); + printk("src slot %u entry_cnt %d slot %u\n", src_slot, entry_cnt, slot); if (entry_cnt < 0) { - // printk("Setting to appending mode\n"); + printk("Setting to appending mode\n"); appending = true; existing = NULL; piv[2] = mas->max; @@ -1357,8 +1437,8 @@ static inline unsigned char mas_ma_cp(struct ma_state *mas, if (!piv[2] && src_slot) piv[2] = mas->max; existing = mte_get_rcu_slot(mas->node, src_slot); - // printk("getting %u (%lu->%p)\n", src_slot, piv[2], - // existing); + printk("getting %p[%u] (%lu->%p)\n", mas_mn(mas), + src_slot, piv[2], existing); } @@ -1366,7 +1446,20 @@ static inline unsigned char mas_ma_cp(struct ma_state *mas, if (!appending) { if (mt_will_coalesce(existing)) { - // printk("coalesce\n"); + printk("coalesce\n"); + if (prev_null) { + if (slot && slot - 1 < mt_pivots[mas_type]) { + printk("Overwrite piv %u\n", slot - 1); + ma_set_pivot(mn, slot - 1, mas_type, + piv[2]); + written_piv = piv[2]; + } + if (update_gaps) { + mas_ma_update_gap(mas, mas_type, mn, mn_type, + piv[2], written_piv, src_slot, + slot, &max_gap, NULL); + } + } goto skip_src_slot; } @@ -1378,20 +1471,22 @@ static inline unsigned char mas_ma_cp(struct ma_state *mas, goto skip_src_slot; } - if (append || (attempt_insert && (cp.index <= piv[2]))) { + if (ma_is_leaf(mn_type) && + (append || (attempt_insert && cp.index <= piv[2]))) { i = 0; - // printk("Written piv %lu\n", written_piv); - if (written_piv == cp.index - 1) + printk("Written piv %lu\n", written_piv); + if (written_piv == cp.index - 1) { i = 1; // previous pivot matches exactly. + } else if (!src_slot && written_piv == cp.index) - i = 1; // mas->min matches exactly. + i = 1; - // printk("piv2 %lu <= last %lu\n", piv[2], cp.last); + printk("piv2 %lu <= last %lu\n", piv[2], cp.last); if (appending) j--; else if (!appending && (piv[2] <= cp.last)) j--; // skip last pivot. - // printk("piv array %u-%u\n", i, j); + printk("piv array %u-%u\n", i, j); attempt_insert = false; } @@ -1402,16 +1497,17 @@ static inline unsigned char mas_ma_cp(struct ma_state *mas, if (!loop_entry && !ma_is_leaf(mn_type)) { - // printk("BUG Here\n"); + printk("BUG Here\n"); } - // printk("write array %d %lu to %u entry_cnt %d\n", i, piv[i], slot, entry_cnt); + printk("write array %d %lu to %u entry_cnt %d\n", i, piv[i], slot, entry_cnt); if (i == 1) loop_entry = entry; if (append) wmb = true; - if (mt_is_empty(loop_entry)) { + if (mt_is_empty(loop_entry) || + xa_is_retry(loop_entry)) { if (null_run) slot--; @@ -1429,15 +1525,20 @@ static inline unsigned char mas_ma_cp(struct ma_state *mas, slot, &max_gap, loop_entry); } + if (!loop_entry) + prev_null = true; + else + prev_null = false; + written_piv = piv[i]; slot++; // If this is a split operation, right will exist // and the target may need to be switched. if ((right) && (cp.node != right->node) && (slot > split)) { - ret = slot - 1; + ret = slot; if (update_gaps) { - // printk("Write gap %p[%u] -> %lu\n", parent, p_slot, max_gap); + printk("Write gap %p[%u] -> %lu\n", parent, p_slot, max_gap); parent->ma64.gap[p_slot] = max_gap; } @@ -1452,14 +1553,14 @@ static inline unsigned char mas_ma_cp(struct ma_state *mas, prev_piv = piv[2]; skip_src_slot: src_slot++; - } while (entry_cnt-- > 0 || attempt_insert); + } while ((entry_cnt-- > 0) || attempt_insert); if (right && update_gaps) parent->ma64.gap[p_slot] = max_gap; - else - ret = src_slot - 1; + else if (!right) + ret = slot - 1; - // printk("Done\n"); + printk("Done\n"); return ret; } @@ -1757,6 +1858,8 @@ static inline unsigned long mas_first_entry(struct ma_state *mas, // Get the leaf slot. mas_set_slot(mas, 0); mas_first_node(mas, limit); + if (mas_is_none(mas)) + return limit; return mas_get_safe_pivot(mas, mas_get_slot(mas)); } @@ -1824,19 +1927,26 @@ static inline unsigned long _mas_copy(struct ma_state *mas, struct ma_cp *cp, void *entry; printk("%u to %u\n", sloc, dloc); - if (prev_piv >= mas->max) + if (prev_piv >= mas->max) { + printk("%d %u %lu >= %lu\n", __LINE__, sloc, prev_piv, mas->max); break; + } - if (sloc < pivot_cnt) + if (sloc < pivot_cnt) { piv = mte_get_pivot(cp->src, sloc); - else + printk("%d\n", __LINE__); + } + else { + printk("%d\n", __LINE__); piv = mas->max; + } if (sloc && !piv) break; entry = mte_get_rcu_slot(cp->src, sloc); if (mt_will_coalesce(entry)) { + printk("%d\n", __LINE__); printk("slot %u will coalesce\n", sloc); if (!sloc) { printk("HERE!\n"); @@ -1851,6 +1961,8 @@ static inline unsigned long _mas_copy(struct ma_state *mas, struct ma_cp *cp, goto next_src_slot; } } + else + goto next_src_slot; } else goto next_src_slot; @@ -1859,28 +1971,33 @@ static inline unsigned long _mas_copy(struct ma_state *mas, struct ma_cp *cp, // Last entry. if (dloc && (piv == mas->max || !piv)) { + printk("%d\n", __LINE__); if (!mte_get_rcu_slot(cp->dst, dloc -1) && mt_is_empty(entry)) { mte_set_pivot(cp->dst, dloc -1, 0); + printk("%d\n", __LINE__); break; } } - if (piv < cp->start_piv) + if (piv < cp->start_piv) { + printk("start_piv ski\n"); goto next_src_slot; + } - if (sloc && piv == prev_piv) + if (sloc && piv == prev_piv) { + printk("skip slot\n"); goto next_src_slot; + } if (dloc < pivot_cnt) { - printk("piv %p[%u] -> %lu\n", cp->dst, dloc, piv); + printk("dloc\n"); mte_set_pivot(cp->dst, dloc, piv); } if (dtype == maple_arange_64) mte_cp_gap(cp->dst, dloc, cp->src, sloc); - printk("%p[%u] -> %p[%u]\n", cp->src, sloc, cp->dst, dloc); mte_cp_rcu_slot(cp->dst, dloc++, cp->src, sloc); prev_piv = piv; next_src_slot: @@ -1911,7 +2028,7 @@ static inline void mte_adopt_children(struct maple_enode *parent) break; child = _mte_get_rcu_slot(parent, slot, type); - if (child) + if (!mt_is_empty(child)) mte_set_parent(child, parent, slot); } } @@ -2062,7 +2179,7 @@ static inline int mas_split(struct ma_state *mas, unsigned char slot, MA_STATE(left, mas->tree, mas->index, mas->last); MA_STATE(right , mas->tree, mas->index, mas->last); - printk("%s\n", __func__); + printk("%s %p\n", __func__, mas_mn(mas)); type = mte_node_type(mas->node); if (mte_is_root(mas->node)) { old_parent = full; @@ -2079,6 +2196,8 @@ static inline int mas_split(struct ma_state *mas, unsigned char slot, mas_dup_state(&parent, mas); mas_ascend(&parent); old_parent = parent.node; + printk("%s mas %p parent %p max %lu\n", __func__, mas_mn(mas), + mas_mn(&parent), parent.max); ptype = mte_node_type(parent.node); p_end = mas_data_end(&parent, ptype, &last_pivot, &coalesce); @@ -2090,6 +2209,8 @@ static inline int mas_split(struct ma_state *mas, unsigned char slot, if (mas_is_err(mas)) return 0; + printk("Parent split returned\n"); + mt_dump(mas->tree); mas_dup_state(&parent, mas); if (split < p_slot) p_slot -= split; @@ -2098,12 +2219,20 @@ static inline int mas_split(struct ma_state *mas, unsigned char slot, mas_set_slot(&parent, p_slot); } ptype = mte_node_type(parent.node); + printk("end of node %p\n", mas_mn(&parent)); p_end = mas_data_end(&parent, ptype, &last_pivot, &coalesce); mas_dup_state(mas, &parent); mas_set_slot(mas, p_slot); + printk("%s mas %p parent %p max %lu\n", __func__, mas_mn(mas), + mas_mn(&parent), parent.max); mas_descend(mas); + printk("%s mas %p parent %p max %lu\n", __func__, mas_mn(mas), + mas_mn(&parent), parent.max); } + printk("%s mas %p parent %p max %lu\n", __func__, mas_mn(mas), + mas_mn(&parent), parent.max); + mas_node_cnt(mas, 3); if (mas_is_err(mas)) return 0; @@ -2119,9 +2248,8 @@ static inline int mas_split(struct ma_state *mas, unsigned char slot, mte_set_parent(left.node, new_parent, p_slot); mte_set_parent(right.node, new_parent, p_slot+1); // split the data into left & right and do the insert. - split = mas_ma_cp(mas, p_slot, &left, &right, NULL, type, entry_cnt, - entry, false); - right.max = mas->max; + split = mas_ma_cp(mas, p_slot, &left, &right, NULL, type, 0, entry_cnt, + entry, 0); // Copy the parents information if (!mte_is_root(full)) { @@ -2136,14 +2264,16 @@ static inline int mas_split(struct ma_state *mas, unsigned char slot, if (right.node) skip++; - // Copy from p_slot + @skip to the end. + // Copy from p_slot + skip to the end. if (p_slot < mt_slots[ptype] - 2) { cp.src_start = p_slot + 1; cp.src_end = p_end; cp.dst_start += skip; + printk("copy %p[%u-%u] up to %lu\n", mte_to_node(cp.src), cp.src_start,cp.src_end, right.max); _mas_copy(&parent, &cp, right.max); } } + right.max = mas->max; // left will be placed in p_slot mte_link(left.node, new_parent, p_slot, left.max, ptype); @@ -2336,7 +2466,7 @@ static inline int _mas_add_slot_cnt(struct ma_state *mas, // Check if it's safe to use the slot without a pivot. // max of the slot == max of the node. // max of the tree is being set. - if ((max == mas->max && mas->max != ULONG_MAX) || + if ((mas->last == mas->max && mas->max != ULONG_MAX) || (mas->max == ULONG_MAX && mas->last == ULONG_MAX)) return req_slots; @@ -2352,13 +2482,14 @@ static inline int __mas_add(struct ma_state *mas, void *entry, enum maple_type mas_type = mte_node_type(mas->node); struct maple_node space; struct maple_node* mn = NULL; + unsigned long mn_min = mas->min; int ret = 0; MA_STATE(cp, mas->tree, mas->index, mas->last); if (append) { printk("Appending.\n"); - mn = mas_mn(mas); + mn = mas_mn(mas); // Appending means writing to the source. } else if (active) { cp.node = mt_mk_node(ma_mnode_ptr(mas_next_alloc(mas)), mas_type); @@ -2369,8 +2500,8 @@ static inline int __mas_add(struct ma_state *mas, void *entry, cp.node = NULL; } - ret = mas_ma_cp(mas, 0, &cp, NULL, mn, mas_type, entry_cnt, entry, - append); + ret = mas_ma_cp(mas, 0, &cp, NULL, mn, mas_type, mn_min, entry_cnt, entry, + append ? entry_cnt : 0); mn->parent = mas_mn(mas)->parent; // FIXME: Propagate gaps. @@ -2424,33 +2555,60 @@ static inline int _mas_add(struct ma_state *mas, void *entry, bool overwrite, goto update_gap; } + + if (!ma_is_leaf(this_type)) { + + BUG_ON(1); + // Allocation failed, so rebuild the tree. + } + + printk("%s: here %d\n", __func__, __LINE__); old_end = mas_data_end(mas, this_type, &last_piv, &coalesce); if (slot > slot_cnt) // search returned MAPLE_NODE_SLOTS slot = old_end + 1; _mas_get_range(mas, slot, &min, &max); + if (mas_get_slot(mas) > slot_cnt) + max = mas->max; + printk("Landed %p[%u] max %lu mas_max %lu\n", mas_mn(mas), mas_get_slot(mas), max, + mas->max); if (slot <= old_end) contents = mte_get_rcu_slot(mas->node, slot); + printk("%s: here %d\n", __func__, __LINE__); // Check early failures. if (!overwrite) { if (mas->last > max) { // spans range. + printk("%s: here %d %lu %lu\n", __func__, __LINE__, mas->last, max); mas_set_err(mas, -ERANGE); return 0; } if (!mt_is_empty(contents)) { + printk("%s: here %d\n", __func__, __LINE__); mas_set_err(mas, -EBUSY); return 0; } } + printk("%s: here %d\n", __func__, __LINE__); // At this point, the we can perform the add. // spans node means we will replace the tree. + printk("mas: l %lu max %lu\n", mas->last, mas->max); if (mas->last > mas->max) return mas_replace_tree(mas, entry); + if (!mte_is_leaf(mas->node)) { + // An allocation failed previously during a rebalance. There + // is no way to know how broken things are, so try to rebuild + // the tree. + mas_reset(mas); + mas_first_node(mas, ULONG_MAX); + printk("Using first node %p\n", mas_mn(mas)); + return mas_replace_tree(mas, entry); + } + // Fits neatly into a slot. if (mas->index == min && mas->last == max) { mte_set_rcu_slot(mas->node, slot, entry); @@ -2472,11 +2630,6 @@ static inline int _mas_add(struct ma_state *mas, void *entry, bool overwrite, return old_end - new_end; } - if(!mte_is_leaf(mas->node)) { - printk("trying to add %lu-%lu to %p\n", mas->index, mas->last, mas_mn(mas)); - } - BUG_ON(!mte_is_leaf(mas->node)); - if (active) { printk("%s %p\n", __func__, mas); printk("Active and needs a node\n"); @@ -2490,8 +2643,7 @@ static inline int _mas_add(struct ma_state *mas, void *entry, bool overwrite, printk("slot %u old_end %u\n", slot, old_end); if (slot > old_end && !coalesce) append = true; - // FIXME: append (slot > old_end) may cause coalescing which wouldn't - // be + __mas_add(mas, entry, old_end, active, append); @@ -2610,7 +2762,7 @@ walk_down: do { void *entry = NULL; if (slot) - mas->min = mas_get_safe_pivot(mas, slot - 1); + mas->min = mas_get_safe_pivot(mas, slot - 1) + 1; mas->max = mas_get_safe_pivot(mas, slot); entry = mte_get_rcu_slot(mas->node, slot); if (xa_is_skip(entry)) { @@ -2630,8 +2782,9 @@ walk_down: if (mt_is_empty(mas->node)) goto no_entry; - if (mte_is_leaf(mas->node)) + if (mte_is_leaf(mas->node)) { goto done; + } slot = 0; } while (1); @@ -2831,7 +2984,7 @@ restart_next_node: break; mn = mte_get_rcu_slot(mas->node, slot); - if (!mn) { + if (mt_is_empty(mn)) { prev_piv = pivot; continue; } @@ -2855,6 +3008,7 @@ restart_next_node: if (mte_is_root(mas->node)) goto no_entry; + mas_set_slot(mas, mte_parent_slot(mas->node)); } no_entry: @@ -2968,6 +3122,7 @@ static inline void* mas_last_entry(struct ma_state *mas, void *entry = mte_get_rcu_slot(mas->node, slot - 1); if (mte_is_leaf(mas->node)) { mas->index = range_start - 1; + mas->index = mte_get_pivot(mas->node, slot - 1); return entry; } mas->max = prev_max; @@ -3202,464 +3357,6 @@ static inline void mas_coalesce_empty(struct ma_state *mas, mas_coalesce_pivots(mas, p_slot, p_type, false); } -/** Private - * - * mas_rebalance() - Rebalance mas->node by acquiring other pivots from the - * node to the right or rebalance the left node. - * - * - * - * Try to allocate 1-2 nodes, depending on if the right node will be - * completely consumed. - * - * returns 0 on failure or the number of slots that were not filled. - * - */ -/* - * - * FIXME: Test spanning multiple levels of the tree. - * - */ -static inline void mas_coalesce(struct ma_state *mas); -#if 0 -static inline int mas_rebalance(struct ma_state *mas, unsigned char end, - unsigned char coalesce) -{ - struct maple_enode *r_enode, *this_enode = mas->node; - unsigned long ret = 0; - unsigned char this_p_slot, r_p_slot; // Parent slots (this one and right) - unsigned char all_slots; // Total slots needed for this and right node. - unsigned char l_slot_cnt, r_slot_cnt; // left and right slot counts - unsigned long l_piv, r_piv = 0; // left and right pivots - unsigned char r_end, r_coalesce; // right node end and values that can be coalesced. - unsigned char node_cnt = 0; // Number of nodes to allocate - unsigned char r_take; - enum maple_type new_type; - bool p_coalesce = false; // coalesce parent. - bool l_null_end = false, r_null_start = false; - struct maple_node new_node; - struct ma_state *src = mas; - int i = 0; - - MA_STATE(p_state, mas->tree, mas->index, mas->last); - MA_STATE(r_state, mas->tree, mas->index, mas->last); - MA_STATE(cp_state, mas->tree, mas->index, mas->last); - MA_CP(cp, this_enode, NULL, 0, end); - - - printk("%s: %p\n", __func__, mas_mn(mas)); - mt_dump(mas->tree); - l_slot_cnt = ma_hard_data(end, coalesce); - if (mte_is_root(this_enode)) - return mas_coalesce_node(mas, end, coalesce, true); - - this_p_slot = mte_parent_slot(this_enode); - mas_dup_state(&p_state, mas); // set the parent node, etc. - mas_ascend(&p_state); // Actually make it the parent. - - // Get the next entry. - mas_set_slot(&p_state, this_p_slot + 1); - if (!mas_next_nentry(&p_state, ULONG_MAX, &r_piv)) { - // this_enode is the right-most node of the parent. - // Reset parent info. - mas_dup_state(&p_state, mas); - mas_ascend(&p_state); - mas_set_slot(&p_state, this_p_slot); - if (mas_prev_nentry(&p_state, 0, &r_piv)) { - // If there is a node to the left, rebalance the left - mas_descend(&p_state); - end = mas_data_end(&p_state, - mte_node_type(p_state.node), &l_piv, &coalesce); - return mas_rebalance(&p_state, end, coalesce); - } - // No left or right, rebalance the parent. - // But first, remove this entry if it's empty. - if (end < coalesce) { - printk("%d empty\n", __LINE__); - mas_coalesce_empty(&p_state, p_state.node, this_p_slot, - mte_node_type(p_state.node)); - } - mas_coalesce(&p_state); - if (mas_is_err(&p_state)) { - mas->node = p_state.node; - return 0; - } - return 1; - } - - // If we reached here, then the node to the right exists. - // set the ma_state information and save a copy for this slot. - mas_dup_state(&r_state, &p_state); - mas_descend(&r_state); - r_enode = r_state.node; - - r_end = mas_data_end(&r_state, mte_node_type(r_enode), &l_piv, &r_coalesce); - r_slot_cnt = ma_hard_data(r_end, r_coalesce); - printk("r_slot_cnt %u\n", r_slot_cnt); - if (r_end < r_coalesce && r_state.max != ULONG_MAX) { - printk("skip entry?\n"); - // This node needs to be a skip entry. - all_slots = l_slot_cnt; - new_type = mte_node_type(this_enode); - l_piv = r_state.max; - goto right_empty; - } - // Add 1 for each slot 0. - all_slots = r_slot_cnt + 2 + l_slot_cnt; - r_take = r_slot_cnt; - node_cnt = 1; - if (all_slots > mt_slot_count(this_enode) - 1) { - node_cnt++; - r_take = mt_slot_count(this_enode) / 2; - } - - if (r_take > r_slot_cnt) - r_take = r_slot_cnt; - - printk("%d l_slot_cnt %d\n", __LINE__, l_slot_cnt); - // check if left ends in NULL - if (l_slot_cnt && !mte_get_rcu_slot(this_enode, l_slot_cnt)) - l_null_end = true; - - printk("r_end and r_coalesce %u %u\n", r_end, r_coalesce); - if ((r_end == r_coalesce) && l_null_end) { // no space needed.. - printk("ends in null at %p[%u] => %ld\n", this_enode, l_slot_cnt, r_state.max); - mte_set_pivot(this_enode, l_slot_cnt, r_state.max); - do { // FIXME: What about non-end nodes? - mas_ascend(mas); - mte_set_pivot(mas->node, this_p_slot, r_state.max); - this_p_slot = mte_parent_slot(mas->node); - } while (!mte_is_root(mas->node)); - return 0; - } - - - if (end < coalesce) { // Left is empty, just copy right. - l_slot_cnt = -1; - ret = r_slot_cnt; - goto left_empty; - } - - mas_node_cnt(mas, node_cnt); - if (mas_is_err(mas)) - return 0; - printk("%d: nodes %u\n", __LINE__, node_cnt); - - mas_dup_state(&cp_state, mas); - // for each entry from right to copy, - cp_state.index = r_state.min; - cp_state.node = mt_mk_node(mas_next_alloc(mas), - mte_node_type(this_enode)); - - - l_slot_cnt = end; - - for (i = 0; i <= r_take; i++) { - // set cp_state->index, cp_state->last, entry - void *entry = mte_get_rcu_slot(r_enode, i); - if (!entry || mt_will_coalesce(entry)) { - cp_state.index = mte_get_pivot(r_enode, i)+1; - continue; - } - cp_state.last = mte_get_pivot(r_enode, i); - // copy node on first run, append after. - // - // call mas_ma_cp(&cp_state, 0, &l_state, type, l_slot_cnt + 1?, - // entry, i == 0 ? false : true) - printk("\tma_cp with %lu-%lu -> %p\n", cp_state.index, cp_state.last, - entry); - l_slot_cnt = mas_ma_cp(mas, 0, &cp_state, NULL, NULL, - mte_node_type(this_enode), l_slot_cnt, entry, - i == 0 ? false : true); - cp_state.index = cp_state.last + 1; - cp_state.max = cp_state.last; // update cp state max. - cp.src_start = i; - } - // - - mte_to_node(cp_state.node)->parent = mte_to_node(this_enode)->parent; - mas->node = cp_state.node; - printk("mas node is %p\n", mas_mn(mas)); - new_type = mte_node_type(cp_state.node); - mas_replace(mas); - mas->max = cp_state.max; - ret = l_slot_cnt; - mte_set_pivot(p_state.node, this_p_slot, mas->max); - -right_empty: - printk("Right empty goto\n"); - if (node_cnt < 2) { - printk("Right is empty, skip\n"); - // Right was entirely consumed. - void *entry = XA_SKIP_ENTRY; - - ret = mt_slots[new_type] - all_slots; - if (!ret) - ret = 1; - - r_p_slot = mte_parent_slot(r_enode); - mte_set_rcu_slot(p_state.node, r_p_slot, entry); - if (r_p_slot < mt_pivot_count(mas->node)) - mte_set_pivot(p_state.node, r_p_slot, l_piv); - mte_free(r_enode); - p_coalesce = true; - goto right_done; - } -left_empty: - - cp.src = r_state.node; - cp.src_end = r_end; - cp.dst = NULL; - cp.dst_start = l_slot_cnt + 1; - cp.dst_end = r_end; - printk("Copy %p to ?[%u-%u]\n", cp.src, cp.dst_start, cp.dst_end); - mas_copy(&r_state, &cp); // cp.dst is coalesced remainder of r_enode. - mte_to_node(cp.dst)->parent = mte_to_node(r_enode)->parent; - r_state.node = cp.dst; - r_p_slot = mte_parent_slot(r_enode); - mas_replace(&r_state); - if (r_p_slot < mt_pivot_count(p_state.node) - 1) - mte_set_pivot(p_state.node, r_p_slot, r_state.max); - - -right_done: - // Set the parent slots to l_piv for all skip slots.. - while (r_p_slot-- > this_p_slot) - mte_set_pivot(p_state.node, r_p_slot, l_piv); - - /* If there is a freed node, then mas->node must point to the parent - * which contained the freed node so it can also be checked for - * coalescing. - */ - if (p_coalesce) - mas_coalesce(&p_state); - return ret; -} -#else -static inline int mas_rebalance(struct ma_state *mas, unsigned char end, - unsigned char coalesce) -{ - struct maple_enode *r_enode, *this_enode = mas->node; - unsigned long ret = 0; - unsigned char this_p_slot, r_p_slot; // Parent slots (this one and right) - unsigned char all_slots; // Total slots needed for this and right node. - unsigned char l_slot_cnt, r_slot_cnt; // left and right slot counts - unsigned long l_piv, r_piv = 0; // left and right pivots - unsigned char r_end, r_coalesce; // right node end and values that can be coalesced. - unsigned char node_cnt; // Number of nodes to allocate - enum maple_type new_type; - bool p_coalesce = false; // coalesce parent. - bool l_null_end = false, r_null_start = false; - - MA_STATE(p_state, mas->tree, mas->index, mas->last); - MA_STATE(r_state, mas->tree, mas->index, mas->last); - MA_CP(cp, this_enode, NULL, 0, end); - - printk("%s: %p\n", __func__, mas_mn(mas)); - l_slot_cnt = ma_hard_data(end, coalesce); - if (mte_is_root(this_enode)) - return mas_coalesce_node(mas, end, coalesce, true); - - this_p_slot = mte_parent_slot(this_enode); - mas_dup_state(&p_state, mas); // set the parent node, etc. - mas_ascend(&p_state); // Actually make it the parent. - - // Get the next entry. - mas_set_slot(&p_state, this_p_slot + 1); - if (!mas_next_nentry(&p_state, ULONG_MAX, &r_piv)) { - // this_enode is the right-most node of the parent. - // Reset parent info. - mas_dup_state(&p_state, mas); - mas_ascend(&p_state); - mas_set_slot(&p_state, this_p_slot); - if (mas_prev_nentry(&p_state, 0, &r_piv)) { - // If there is a node to the left, rebalance the left - mas_descend(&p_state); - end = mas_data_end(&p_state, - mte_node_type(p_state.node), &l_piv, &coalesce); - return mas_rebalance(&p_state, end, coalesce); - } - // No left or right, rebalance the parent. - // But first, remove this entry if it's empty. - if (end < coalesce) { - printk("%d empty %d %d\n", __LINE__, end, coalesce); - // FIXME: deleted? - // deleted.. AND FREE! - mas_coalesce_empty(&p_state, p_state.node, this_p_slot, - mte_node_type(p_state.node)); - } - mas_coalesce(&p_state); - if (mas_is_err(&p_state)) { - mas->node = p_state.node; - return 0; - } - return 1; - } - - printk("left and right\n"); - // If we reached here, then the node to the right exists. - // set the ma_state information and save a copy for this slot. - mas_dup_state(&r_state, &p_state); - mas_descend(&r_state); - r_enode = r_state.node; - - r_end = mas_data_end(&r_state, mte_node_type(r_enode), &l_piv, &r_coalesce); - r_slot_cnt = ma_hard_data(r_end, r_coalesce); - printk("r_slot_cnt %u\n", r_slot_cnt); - if (r_end < r_coalesce && r_state.max != ULONG_MAX) { - printk("skip entry?\n"); - // This node needs to be a deleted entry. - all_slots = l_slot_cnt; - new_type = mte_node_type(this_enode); - l_piv = r_state.max; - goto right_empty; - } - // Add 1 for each slot 0. - all_slots = r_slot_cnt + 2 + l_slot_cnt; - - node_cnt = 1; - if (all_slots > mt_slot_count(this_enode) - 1) - node_cnt++; - - printk("%d l_slot_cnt %d %u %u\n", __LINE__, l_slot_cnt, end, coalesce); - // check if left ends in NULL - if (l_slot_cnt && !mte_get_rcu_slot(this_enode, l_slot_cnt)) - l_null_end = true; - - printk("r_end and r_coalesce %u %u\n", r_end, r_coalesce); - if ((r_end == r_coalesce) && l_null_end) { // no space needed.. - printk("ends in null at %p[%u] => %ld\n", this_enode, l_slot_cnt, r_state.max); - mte_set_pivot(this_enode, l_slot_cnt, r_state.max); - do { - unsigned old_piv, old_mas_max = mas->max; - mas_ascend(mas); - old_piv = mte_get_pivot(mas->node, this_p_slot); - if (old_piv != old_mas_max) - break; - mte_set_pivot(mas->node, this_p_slot, r_state.max); - this_p_slot = mte_parent_slot(mas->node); - } while (!mte_is_root(mas->node)); - return 1; - } - mas_node_cnt(mas, node_cnt); - if (mas_is_err(mas)) { - // Special case. If we are failing to allocate but the tree - // should be collapsed back, then do the right thing (tm). - if ((r_end == r_coalesce) && - (r_state.max == ULONG_MAX) && - (end >= coalesce)) { - mte_set_pivot(this_enode, l_slot_cnt, r_state.max); - do { - unsigned old_piv, old_mas_max = mas->max; - mas_ascend(mas); - old_piv = mte_get_pivot(mas->node, this_p_slot); - if (old_piv != old_mas_max) - break; - mte_set_pivot(mas->node, this_p_slot, r_state.max); - this_p_slot = mte_parent_slot(mas->node); - } while (!mte_is_root(mas->node)); - r_p_slot = mte_parent_slot(r_enode); - mas_coalesce_empty(&r_state, p_state.node, r_p_slot, - mte_node_type(p_state.node)); - mt_dump(mas->tree); - } - return 0; - } - printk("here %d\n", __LINE__); - - // Coalesce this_enode into a new node. - cp.dst_end = l_slot_cnt; - printk("%d Copy %p[%u-%u] to %p[%u-%u]\n", __LINE__, mas_mn(mas), cp.src_start, cp.src_end, "??", cp.dst_start, cp.dst_end); - mas_copy(mas, &cp); // cp.dst now has coalesced this_enode. - - - // check if right start in NULL.. - if (mt_will_coalesce(mte_get_rcu_slot(r_enode, 0)) || - !mte_get_rcu_slot(r_enode, 0)) - r_null_start = true; - - printk("r first is %p\n", mte_get_rcu_slot(r_enode, 0)); - if (l_null_end && r_null_start) - { - all_slots--; - } else if (!l_null_end && !r_null_start && - r_state.min != mas->max + 1) { - // Need a null if the pivots don't line up here. - mte_set_pivot(cp.dst, cp.dst_start++, r_state.min); - all_slots++; - } - - // Copy from the right node. - cp.src = r_enode; - cp.src_start = 0; - cp.dst_end = mt_slot_count(cp.dst) - 1; - if (all_slots <= mt_slot_count(mas->node) - 1) - cp.src_end = r_end; - else - cp.dst_end = (all_slots + 1)/ 2; // Take 1/2 the entries. - printk("Copy %p[%u-%u] to %p[%u-%u]\n", mas_mn(&r_state), cp.src_start, cp.src_end, mte_to_node(cp.dst), cp.dst_start, cp.dst_end); - - _mas_copy(&r_state, &cp, - mte_get_pivot(cp.dst, cp.dst_start-1)); // cp.dst is now complete, place it in the tree. - mte_to_node(cp.dst)->parent = mte_to_node(this_enode)->parent; - new_type = mte_node_type(cp.dst); - mas->node = cp.dst; - mas_replace(mas); - mas->max = r_state.max; - if (all_slots > mt_slots[new_type] - 1) { - mas_data_end(mas, mte_node_type(mas->node), &l_piv, &coalesce); - mas->max = l_piv; - } - l_piv = mas->max; - ret = mt_slot_count(mas->node) - cp.dst_start + 1; - mte_set_pivot(p_state.node, this_p_slot, l_piv); - -right_empty: - if (all_slots <= mt_slots[new_type] - 1) { - // Right was entirely consumed. - void *entry = XA_SKIP_ENTRY; - - ret = mt_slots[new_type] - all_slots; - if (!ret) - ret = 1; - - r_p_slot = mte_parent_slot(r_enode); - mte_set_rcu_slot(p_state.node, r_p_slot, entry); - if (r_p_slot < mt_pivot_count(mas->node)) - mte_set_pivot(p_state.node, r_p_slot, l_piv); - mte_free(r_enode); - p_coalesce = true; - goto right_done; - } - - cp.src_end = r_end; - cp.dst = NULL; - cp.dst_start = 0; - cp.dst_end = r_end; - mas_copy(&r_state, &cp); // cp.dst is coalesced remainder of r_enode. - mte_to_node(cp.dst)->parent = mte_to_node(r_enode)->parent; - r_state.node = cp.dst; - r_p_slot = mte_parent_slot(r_enode); - mas_replace(&r_state); - if (r_p_slot < mt_pivot_count(p_state.node) - 1) - mte_set_pivot(p_state.node, r_p_slot, r_state.max); - - -right_done: - // Set the parent slots to l_piv for all skip slots.. - while (r_p_slot-- > this_p_slot) - mte_set_pivot(p_state.node, r_p_slot, l_piv); - - /* If there is a freed node, then mas->node must point to the parent - * which contained the freed node so it can also be checked for - * coalescing. - */ - if (p_coalesce) - mas_coalesce(&p_state); - return ret; -} -#endif - /** Private * */ @@ -3733,162 +3430,269 @@ remove_level: mas_replace(mas); } +/* Append from src to mas */ +static inline int mas_cp_append(struct ma_state *mas, struct ma_state *src, + struct ma_state *p_mas, unsigned char dst_cnt, + unsigned char count, bool retry) +{ + enum maple_type mas_type = mte_node_type(mas->node); + int slot = 0; + printk("count = %u\n", count); + MA_STATE(dst, mas->tree, mas->index, mas->last); + + mas_dup_state(&dst, mas); + + dst.index = src->min; + dst.max = src->max; + while (slot <= count ) { + void *entry = mte_get_rcu_slot(src->node, slot); + dst.last = mas_get_safe_pivot(src, slot); + printk("range %lu-%lu\n", dst.index, dst.last); + if (mt_is_empty(entry) && dst.last != ULONG_MAX) + goto next; + src->index = dst.index; + src->last = dst.last; + + printk("Appending %p[%u] to %p[%u]\n", mas_mn(src), slot, + mas_mn(&dst), dst_cnt); + + dst_cnt = mas_ma_cp(src, 0, &dst, NULL, mas_mn(&dst), mas_type, + dst.min, slot, + entry, dst_cnt); +next: + dst.index = dst.last+1; + printk("update dst to %lu\n", dst.index); + slot++; + } + printk("Just left while with %u\n", slot); + if (retry) { + mte_set_pivot(p_mas->node, mte_parent_slot(mas->node), + mas_get_safe_pivot(src, slot)); + wmb(); + do { + mte_set_rcu_slot(src->node, slot, XA_RETRY_ENTRY); // relocated. + } while(slot-- > 0); + } + return dst_cnt; + // FIXME: Check mas->node's gap. +} /** Private - * mas_coalesce() - Check prev/next node for end/start NULLs and adjust the - * gaps and parent pivot accordingly. - * - * - * Attempt to move things left with a rebalance. Upon failure, check if there - * is a contiguous gap from the end of this node to the start of the next. - * - * Notes: - * - Entries move Left, appending data at the end of the leaf - * - Holes move Right. - * - Either way, parent pivots need to be changed. - * - empty nodes will be replaced by skip entries on allocation failure. + * mas_coalesce() - * + * coalesce completely consumes the right node into this node. * */ -static inline void mas_coalesce(struct ma_state *mas) -{ - unsigned char end, p_slot, coalesce; - struct maple_enode *this_enode, *eparent; - enum maple_type this_type, p_type; - bool check_parent = false; - unsigned long piv, this_piv; - int h_data; - void *entry; - -start: - mt_dump(mas->tree); - if (mte_is_root(mas->node)) - return mas_coalesce_root(mas); - this_enode = mas->node; - this_type = mte_node_type(this_enode); - end = mas_data_end(mas, this_type, &this_piv, &coalesce); - printk("data end of %p = %u\n", mas_mn(mas), end); - p_type = mas_parent_enum(mas, this_enode); - p_slot = mte_parent_slot(this_enode); - eparent = mt_mk_node(mte_parent(this_enode), p_type); - - - /* If there is any space to save, try to reallocate */ - h_data = ma_hard_data(end, coalesce); - if (h_data < mt_min_slots[this_type] - 1) { - printk("rebalance?\n"); - if (mas_rebalance(mas, end, coalesce)) - goto done; - - if (mas_is_err(mas)) { - if (end >= coalesce) - return; - if (mas->max == ULONG_MAX) - return; - // Single entry and allocation failed, free this_enode - printk("failure path? %u %u\n", end, coalesce); - // FIXME: Deleted? - mas->node = this_enode; - mas_coalesce_empty(mas, eparent, p_slot, p_type); - check_parent = true; - goto done; - } +static inline bool mas_coalesce(struct ma_state *mas, unsigned char l_end_slot, + unsigned char l_coalesce, enum maple_type l_type, + struct ma_state *r_mas, unsigned char r_end_slot, + struct ma_state *p_mas, unsigned long total_slots, + struct ma_cp *cp) +{ + struct maple_node *mn; + bool ret = false; + printk("%s: %p + %p\n", __func__, mas_mn(mas), mas_mn(r_mas)); + + MA_STATE(dst, mas->tree, mas->index, mas->last); + // it is possible that all of the right node can be appended to the + // left + if (total_slots + 1 - l_end_slot < mt_slots[l_type] - l_end_slot) { + printk("append to %p[%u] from %p\n", mas_mn(mas), l_end_slot, + mas_mn(r_mas)); +// mas->max = r_mas->max; + mas_cp_append(mas, r_mas, p_mas, l_end_slot, r_end_slot, false); + mte_set_rcu_slot(p_mas->node, mte_parent_slot(r_mas->node), + XA_SKIP_ENTRY); + mte_free(r_mas->node); + printk("coalesce complete\n"); + goto done; } - /* Group the gaps together. Acquire any data from the next node, if - * necessary - */ - - entry = mte_get_rcu_slot(this_enode, end); - if (!mt_is_empty(entry)) - goto check_start; - - - // Check if next node starts with null. - mas_next_slot(mas, ULONG_MAX); - if (mas->node == MAS_NONE) - goto check_start; - - if (mt_is_empty(mas->node) || - mt_is_empty(mte_get_rcu_slot(mas->node, 0))) { - unsigned char slot = end; - - if (check_parent) - slot = 0; + mas_node_cnt(mas, 1); // Try to allocate. + if (mas_is_err(mas)) { + printk("%s: Allocation failure\n", __func__); + // Allocation failed, we could try to append as much + // as possible here? + return false; + } - while (--slot > 0) { - if (!mt_is_empty(mte_get_rcu_slot(this_enode, slot))) - break; - } - mas->node = eparent; - mas_update_limits(mas, p_slot, p_type); - mas->node = this_enode; + printk("New node\n"); + mn = mas_next_alloc(mas); + mas_dup_state(&dst, mas); + mn->parent = mas_mn(mas)->parent; + dst.node = mt_mk_node(mn, l_type); +// cp->dst = mas->node; +// mas_copy(mas, cp); + l_end_slot = mas_cp_append(&dst, mas, p_mas, 0, l_end_slot, false); + printk("l_node_end is now %u\n", l_end_slot); + if (!l_end_slot && !mte_get_pivot(dst.node, 0)) { + mte_set_pivot(dst.node, 0, mas->max); + printk("set first pivot to %lu\n", mas->max); + l_end_slot++; + } + mas->node = dst.node; + // FIXME: Broken. + mas_cp_append(mas, r_mas, p_mas, l_end_slot, r_end_slot, true); + ret = true; +done: + if (!mte_is_leaf(mas->node)) + mte_adopt_children(mas->node); - if (!slot) { - // Empty node... - // deleted.. AND FREE! - mas_coalesce_empty(mas, eparent, p_slot, p_type); - check_parent = true; - piv = mas->min; - } else { - piv = mas_get_safe_pivot(mas, slot); - } + mte_set_pivot(p_mas->node, mte_parent_slot(mas->node), r_mas->max); + return ret; +} - if (p_slot <= mt_pivot_count(eparent)) - mte_set_pivot(eparent, p_slot, piv); - // Walk up checking for the old pivot and set to piv. +/** Private + * mas_rebalance() - + * + * rebalance moves data from the node to the right to this node if the + * low watermark of data is not met. It also calls coalesce if the right data + * can fully be moved to the left. + * + */ +static inline void mas_rebalance(struct ma_state *mas) +{ + struct maple_enode *this_enode; + unsigned char l_end_slot, l_coalesce, r_end_slot, r_coalesce; + unsigned char p_slot; // parent slot. + unsigned char total_slots, copy_count; + unsigned long l_end_piv, r_end_piv; + enum maple_type l_type, r_type; + bool try_anyways = false; + bool free; + + MA_STATE(r_mas, mas->tree, mas->index, mas->last); // right state + MA_STATE(p_mas, mas->tree, mas->index, mas->last); // parent state + MA_CP(cp, mas->node, NULL, 0, 0); + mt_dump(mas->tree); +start: + printk("%s: Starting with %p\n", __func__, mas_mn(mas)); + free = false; + this_enode = mas->node; + l_type = mte_node_type(mas->node); + if (mte_is_root(mas->node)) + return mas_coalesce_root(mas); // height reduction and such. - if (!slot) + mas_dup_state(&p_mas, mas); + mas_ascend(&p_mas); + p_slot = mte_parent_slot(mas->node); + l_end_slot = mas_data_end(mas, l_type, &l_end_piv, &l_coalesce); + printk("hard data is %u\n", ma_hard_data(l_end_slot, l_coalesce)); + if (!try_anyways && + (ma_hard_data(l_end_slot, l_coalesce) >= mt_min_slots[l_type])) { + printk("Done this\n"); + return; // Everything's perfectly all right now. + } +#if 0 + else if (ma_hard_data(l_end_slot, l_coalesce == 1)) { + printk("Empty node %p\n", mas_mn(mas)); + mte_set_rcu_slot(p_mas.node, p_slot, XA_DELETED_ENTRY); + mte_free(mas->node); + goto done; + } +#endif + try_anyways = false; + + // Make sure there is a right node. + mas_dup_state(&r_mas, &p_mas); + mas_set_slot(&r_mas, p_slot + 1); + if (!mas_next_nentry(&r_mas, ULONG_MAX, &r_end_piv)) { + // Right-most node coalescing. + printk("Right most\n"); + mas_dup_state(&r_mas, &p_mas); + printk("reset to node is %p\n", mas_mn(&r_mas)); + mas_set_slot(&r_mas, p_slot); + if (!mas_prev_nentry(&r_mas, 0, &r_end_piv)) { + // Single entry in the parent. + printk("Single entry?\n"); + if (l_end_slot < l_coalesce) { // Single entry is empty. + free = true; + } else if (l_end_slot == l_coalesce && mas->max == ULONG_MAX) { + // Possible single entry of null for + // ULONG_MAX. + printk("ulong max check needed\n"); + BUG_ON(1); + free = true; + } + if (free) { + mte_set_rcu_slot(p_mas.node, p_slot, + XA_DELETED_ENTRY); + } goto done; + } + mas_descend(&r_mas); + printk("prev slot is %p\n", mas_mn(&r_mas)); + mas_dup_state(mas, &r_mas); + printk("Forcing next one, coalesce prev.\n"); + try_anyways = true; // Force a rebalance/coalesce of these nodes. + goto start; // restart with new left. + } + mas_descend(&r_mas); + printk("Right node is %p\n", mas_mn(&r_mas)); + + // We have a left and a right, check if they can be coalesced. + r_type = mte_node_type(r_mas.node); // not for racing. + r_end_slot = mas_data_end(&r_mas, r_type, &r_end_piv, &r_coalesce); + + // end_slot values don't count slot 0, so add one. + total_slots = l_end_slot + 1 - l_coalesce; + total_slots += r_end_slot + 1 - r_coalesce; + if (l_end_piv + 1 != r_mas.min) + total_slots++; // will need a null entry between the two. + + mas_dup_state(&p_mas, mas); + mas_ascend(&p_mas); + cp.src_end = l_end_slot; + if (total_slots <= mt_slots[l_type]) { + // Totally consume the right node. + free = mas_coalesce(mas, l_end_slot, l_coalesce, l_type, + &r_mas, r_end_slot, &p_mas, total_slots, &cp); + if (mas_is_err(mas)) + return; - // Indicate value has been moved. - while (++slot <= end) - mte_set_rcu_slot(this_enode, slot, XA_RETRY_ENTRY); + goto done; } -check_start: - mas->node = this_enode; - entry = mte_get_rcu_slot(this_enode, 0); - if (mt_is_empty(entry)) { - unsigned char prev_end; - // Check the previous node. - mas_prev_slot(mas, 0); - if (mas_is_none(mas)) - goto done; + // Rebalance between mas and r_mas nodes. + p_slot = mte_parent_slot(mas->node); - if (!mt_is_empty(mas->node)) { - prev_end = mas_data_end(mas, mte_node_type(mas->node), - &piv, &coalesce); + mas_node_cnt(mas, 1); // Try to allocate. + if (mas_is_err(mas)) { + printk("%s: Allocation failure\n", __func__); + // Allocation failed, we could try to append as much + // as possible here? + return; + } + printk("new node\n"); - if (!mt_is_empty(mte_get_rcu_slot(mas->node, prev_end))) - goto done; - } else { - piv = mte_get_pivot(this_enode, 0); - } + free = true; // free this_enode after the operation. + mas->node = mt_mk_node(mas_next_alloc(mas), l_type); + mas_mn(mas)->parent = mte_to_node(this_enode)->parent; + cp.dst = mas->node; + mas_copy(mas, &cp); - if (p_slot) - mte_set_pivot(eparent, p_slot - 1, piv); + // Put 1/2 of the contents into the left if not all of them can fit. + copy_count = (total_slots / 2) - (l_end_slot + 1 - l_coalesce); - // Walk up and set all the pivots to piv - } -done: - mas->node = this_enode; - if (check_parent) { - check_parent = false; - mas_ascend(mas); - printk(" -> max is %lu\n", mas->max); - mte_free(this_enode); - printk("going to start with %p\n", mas_mn(mas)); - goto start; + mas_cp_append(mas, &r_mas, &p_mas, l_end_slot, copy_count, true); + if (total_slots <= mt_slots[l_type]) { + printk("Use skip entry\n"); + mte_set_rcu_slot(p_mas.node, mte_parent_slot(r_mas.node), + XA_SKIP_ENTRY); + mte_free(r_mas.node); } - if (mt_is_alloc(mas->tree)) - mas_update_gap(mas); - - return; +done: + if (!mte_is_leaf(mas->node)) + mte_adopt_children(mas->node); + if (free) + mas_replace(mas); + mt_dump(mas->tree); + // Check parent for rebalancing. + mas_dup_state(mas, &p_mas); + goto start; } - static inline bool _mas_rev_awalk(struct ma_state *mas, unsigned long size) { enum maple_type type; @@ -4179,16 +3983,19 @@ skip_entry: default: for (i = mas_get_slot(mas); i < mt_slots[type]; i++) { pivot = _mas_get_safe_pivot(mas, i, type); + if (i != 0 && pivot == 0) { i = MAPLE_NODE_SLOTS; goto done; } + if (min > pivot) // coalescing value was in the last slot. + min = pivot; + if (mas->index <= pivot) { max = pivot; break; } - min = pivot + 1; } @@ -4406,13 +4213,143 @@ void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max, } EXPORT_SYMBOL(mt_find); +static inline int mas_build_replacement(struct ma_state *mas, void *new_entry, + long node_cnt) + +{ + struct maple_enode *last = NULL; + unsigned long new_index, new_last; + unsigned long r_index, r_last; + void *entry; + + DEFINE_MTREE(new_tree); + MA_STATE(new_mas, &new_tree, 0, 0); + + + if (!node_cnt) + return 0; + node_cnt += 3; // Room for an extra split. + + //printk("Using node count %d for %lu-%lu\n", node_cnt, mas->index, mas->last); + mas_node_cnt(mas, node_cnt); + if (mas_is_err(mas)) + return 0; + + new_index = mas->index; + new_last = mas->last; + + /* Move allocations from mas to new_mas. + * NOTE: This is necessary as mas will pass back errors and will retry + * the allocation, so it has to be done in mas and has to be moved for + * below. + */ + new_mas.alloc = mas->alloc; + mas->alloc = NULL; + + //printk("\tCopy left\n"); + // Copy left side + mas_reset(mas); + mas->index = 0; + mas->last = 0; + mas_for_each(mas, entry, new_index - 1) { + //printk("\tLeft for each\n"); + new_mas.index = mas->index; + new_mas.last = mas_get_safe_pivot(mas, mas_get_slot(mas)); + if (entry == XA_DELETED_ENTRY) + BUG_ON(1); + ma_inactive_insert(&new_mas, entry); + if (mas_is_err(&new_mas)) { + goto error; + } + //printk("node count is %u\n", mas_get_alloc_cnt(&new_mas)); + //mt_dump(new_mas.tree); + } + + //printk("Done\n"); + //printk("new_mas node %p\n", mas_mn(&new_mas)); + // Insert the new value. + new_mas.index = new_index; + new_mas.last = new_last; + //printk("\tinsert new entry.. %p %lu %lu\n", mas_mn(&new_mas), new_mas.index, new_mas.last); + ma_inactive_insert(&new_mas, new_entry); + if (mas_is_err(&new_mas)) + goto error; + + + /* + * We need to run through a few things: + * - new_mas.last goes beyond anything right now (no entries) + * - new_mas.last cuts a range + * - new_mas.last ends in a null + * - new_mas.last has a sequentially next value (48) + */ + + mas_reset(mas); + mas->index = new_last+1; + mas->last = new_last+1; + _mas_range_walk(mas, &r_index, &r_last); + + if (mas_get_slot(mas) == MAPLE_NODE_SLOTS) + goto skip_right; + + + if (mte_is_leaf(mas->node)) { + entry = mte_get_rcu_slot(mas->node, mas_get_slot(mas)); + if (!mt_is_empty(entry)) + { + new_mas.index = r_index; + new_mas.last = r_last; + printk("!!Inserting %lu-%lu -> %p\n", r_index, r_last, entry); + ma_inactive_insert(&new_mas, entry); + if (mas_is_err(&new_mas)) + goto error; + } + } + + printk("\tCopy right from %p\n", mas->node); + //mt_dump(mas->tree); + mas_for_each(mas, entry, ULONG_MAX) { + if (mas->index < new_index) + continue; + + new_mas.index = mas->index; + new_mas.last = mas_get_safe_pivot(mas, mas_get_slot(mas)); + // printk("\tInsert %lu-%lu => %p\n", new_mas.index, new_mas.last, entry); + ma_inactive_insert(&new_mas, entry); + if (mas_is_err(&new_mas)) + goto error; + //mt_dump(new_mas.tree); + } + +skip_right: + + mt_dump(mas->tree); + printk("Done rebuild due to %lu-%lu\n", new_index, new_last); + mt_dump(new_mas.tree); + last = mas->tree->ma_root; + mas->node = new_tree.ma_root; + _mas_replace(mas, false, false); + mas->node = MAS_START; + mas->alloc = new_mas.alloc; + mte_destroy_walk(last); + + return node_cnt; + +error: + printk("Error on adding %lu\n",new_mas.index); + if (new_mas.tree) + mte_destroy_walk(new_mas.tree->ma_root); + mt_dump(mas->tree); + BUG_ON(1); + return 0; +} + /* Private * mas_replace_tree() - Build a new tree and replace the entire structure. * */ static inline int mas_replace_tree(struct ma_state *mas, void *new_entry) { - void *entry; unsigned long piv; unsigned char coalesce; unsigned int slot_cnt = 0; @@ -4421,20 +4358,20 @@ static inline int mas_replace_tree(struct ma_state *mas, void *new_entry) enum maple_type p_type = mas_parent_enum(mas, mas->node); // Create a new tree. - DEFINE_MTREE(new_tree); - MA_STATE(new_mas, &new_tree, 0, 0); MA_STATE(r_mas, mas->tree, mas->last + 1, mas->last + 1); - MA_STATE(l_mas, mas->tree, 0, 0); - + printk("%s\n", __func__); + mt_dump(mas->tree); // Count the slots that will be used in the node we landed. slot_cnt = 3 + mas_get_slot(mas); // 3 is the max a new entry can create. // Count the nodes that are currently used to the left. mas_set_slot(mas, mte_parent_slot(mas->node)); + printk("Starting left at %p[%u]\n", mas_mn(mas), mas_get_slot(mas)); while (!mas_is_none(mas)) { last = mas->node; mas_prev_node(mas, 0); + printk("prev node %p[%u]\n", mas_mn(mas), mas_get_slot(mas)); leaves++; } // Set mas->node to a valid node. @@ -4442,9 +4379,11 @@ static inline int mas_replace_tree(struct ma_state *mas, void *new_entry) // Walk down to the right side of the tree. _mas_walk(&r_mas); + printk("Starting right at %p[%u]\n", mas_mn(&r_mas), mas_get_slot(&r_mas)); // Add the slots to the right of where the search landed. if (mas_get_slot(&r_mas) == MAPLE_NODE_SLOTS) { r_mas.node = MAS_NONE; + slot_cnt++; //entry for oo goto skip_r_count; } slot_cnt -= mas_get_slot(&r_mas); @@ -4458,7 +4397,6 @@ static inline int mas_replace_tree(struct ma_state *mas, void *new_entry) mas_next_node(&r_mas, ULONG_MAX); leaves++; } - r_mas.node = last; skip_r_count: // Calculate all the nodes needed for a new tree. @@ -4466,78 +4404,14 @@ skip_r_count: leaves++; printk("%d leaves\n", leaves); - node_cnt = 2; // Root node. + node_cnt = 1; // Root node. and room to split. while (leaves) { // add the number of nodes at each level. node_cnt += leaves; leaves /= mt_slots[p_type]; } - - printk("Using node count %d for %lu-%lu\n", node_cnt, mas->index, mas->last); - mas_node_cnt(mas, node_cnt); - if (mas_is_err(mas)) - return 0; - - /* Move allocations from mas to new_mas. - * NOTE: This is necessary as mas will pass back errors and will retry - * the allocation, so it has to be done in mas and has to be moved for - * below. - */ - new_mas.alloc = mas->alloc; - mas->alloc = NULL; - - // Copy left side - mas_reset(&l_mas); - mas_for_each(&l_mas, entry, mas->index - 1) { - new_mas.index = l_mas.index; - new_mas.last = l_mas.index; - ma_inactive_insert(&new_mas, entry); - if (mas_is_err(&new_mas)) { - mt_dump(new_mas.tree); - BUG_ON(1); - } - } - - // Insert the new value. - new_mas.index = mas->index; - new_mas.last = mas->last; - ma_inactive_insert(&new_mas, new_entry); - if (mas_is_err(&new_mas)) - BUG_ON(1); - - - /* - * We need to run through a few things: - * - new_mas.last goes beyond anything right now (no entries) - * - new_mas.last cuts a range - * - new_mas.last ends in a null - * - new_mas.last has a sequentially next value (48) - */ - if (mas_is_none(&r_mas)) // No entry beyond new_mas.last - goto skip_right; - - printk("node %p\n", new_mas.node); - mas_for_each(&l_mas, entry, ULONG_MAX) { - if (l_mas.index < r_mas.index) - continue; - - new_mas.index = l_mas.index; - new_mas.last = mas_get_safe_pivot(&l_mas, mas_get_slot(&l_mas)); - ma_inactive_insert(&new_mas, entry); - if (mas_is_err(&new_mas)) - BUG_ON(1); - } - -skip_right: - - last = mas->tree->ma_root; - mas->node = new_tree.ma_root; - _mas_replace(mas, false, false); - mas->node = MAS_START; - mas->alloc = new_mas.alloc; - mte_destroy_walk(last); - - return node_cnt; + return mas_build_replacement(mas, new_entry, node_cnt); } + static inline bool mas_rewind_node(struct ma_state *mas); static inline void mas_rev_awalk(struct ma_state *mas, unsigned long size) { @@ -4994,12 +4868,16 @@ static inline void *mas_erase(struct ma_state *mas) type = mte_node_type(mas->node); entry = mte_get_rcu_slot(mas->node, slot); + printk("%s: %lu return %p[%u] -> %p now to zero.\n", __func__, + mas->index, mas_mn(mas), slot, entry); mte_update_rcu_slot(mas->node, slot, XA_DELETED_ENTRY); // dense nodes only need to set a single value. - if (!ma_is_dense(type)) - mas_coalesce_pivots(mas, slot, type, false); +// if (!ma_is_dense(type)) +// mas_coalesce_pivots(mas, slot, type, false); - mas_coalesce(mas); + mas_rebalance(mas); + printk("%s: %lu return %p[%u] -> %p\n", __func__, + mas->index, mas_mn(mas), slot, entry); return entry; } @@ -5185,6 +5063,7 @@ void *mtree_erase(struct maple_tree *mt, unsigned long index) MA_STATE(mas, mt, index, index); + printk("%s: erase %lu\n", __func__, index); mtree_lock(mt); entry = mas_erase(&mas); mtree_unlock(mt); @@ -5427,8 +5306,11 @@ void mas_validate_gaps(struct ma_state *mas) gap = mte_get_gap(mte, i); if (mt_is_empty(mte_get_rcu_slot(mas->node, i))) BUG_ON(gap != p_end - p_start + 1); - else + else { + printk("gap %lu p_end %lu p_start %lu\n", gap, p_end, p_start); + printk("Gap %p[%u] != %lu\n", mas_mn(mas), i, p_end - p_start+1); BUG_ON(gap >= p_end - p_start + 1); + } } if (gap > max_gap) @@ -5470,6 +5352,11 @@ void mas_validate_limits(struct ma_state *mas) unsigned long piv = mas_get_safe_pivot(mas, i); if (!piv) break; + printk("Checking %p[%u] %lu %lu\n", mas_mn(mas), i, mas->min, mas->max); + if (piv > mas->max) { + printk("piv fails %lu\n", piv); + mt_dump(mas->tree); + } BUG_ON(piv < mas->min); BUG_ON(piv > mas->max); } diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c index ade52143846ac..4f58df2410751 100644 --- a/lib/test_maple_tree.c +++ b/lib/test_maple_tree.c @@ -17,9 +17,9 @@ int mtree_insert_index(struct maple_tree *mt, unsigned long index, gfp_t gfp) static void mtree_erase_index(struct maple_tree *mt, unsigned long index) { - printk("erase %lu\n", index); + printk("\t\terase %lu\n", index); MT_BUG_ON(mt, mtree_erase(mt, index) != xa_mk_value(index & LONG_MAX)); - printk("load %lu\n", index); + printk("\t\tload %lu\n", index); MT_BUG_ON(mt, mtree_load(mt, index) != NULL); } @@ -91,6 +91,7 @@ static noinline void check_load(struct maple_tree *mt, unsigned long index, { void *ret = mtree_test_load(mt, index); + printk("load %lu ret %p != %p\n", index, ret, ptr); MT_BUG_ON(mt, ret != ptr); } @@ -558,6 +559,7 @@ static noinline void check_find(struct maple_tree *mt) entry = mas_prev(&mas, 0); index = mas.index; last = mas.last; + printk("Expected %lu-%lu=> %p\n", index, last, entry); mt_dump(mt); // Erase the last entry. @@ -574,6 +576,7 @@ static noinline void check_find(struct maple_tree *mt) printk("index set to %lu\n", mas.index); // Check results. + printk("Obtained %lu-%lu=> %p\n", mas.index, mas.last, entry2); printk("entry %p %p\n" ,entry, entry2); MT_BUG_ON(mt, entry != entry2); printk("index %lu != %lu\n", index, mas.index); @@ -615,6 +618,7 @@ static noinline void check_find_2(struct maple_tree *mt) mt_dump(mt); printk("\t\tErase %lu\n", i); mtree_erase_index(mt, i); + printk("\t\tDONE %lu\n\n", i); j = i + 1; mas_set(&mas, 0); rcu_read_lock(); @@ -859,12 +863,20 @@ static noinline void check_erase_testset(struct maple_tree *mt) // Shrinking tree test. // + printk("\tStarting Shrinking with this:\n"); + mt_dump(mt); for (int i = 13; i < ARRAY_SIZE(set); i++) { + printk("\tInserting %lu\n", set[i]); erase_check_insert(mt, i); + mt_dump(mt); } + printk("\tStarting shrink with this:\n"); + mt_dump(mt); mt_set_non_kernel(99); for (int i = 18; i < ARRAY_SIZE(set); i++) { + printk("\tErase %lu\n", set[i]); + mt_dump(mt); erase_check_erase(mt, i); for (int j = 0; j < ARRAY_SIZE(set); j++) { if (j < 18 || j > i) @@ -979,13 +991,13 @@ static noinline void check_erase2_testset(struct maple_tree *mt, check = 0; addr = 0; mt_for_each(mt, foo, addr, ULONG_MAX) { + printk("%d %ld\n", check, addr - 1); check++; if (check > entry_cnt) break; - printk("%d %ld\n", check, addr - 1); } - printk("mt_for_each %d == %d\n", check, entry_cnt); + printk("mt_for_each %d == entry_cnt %d\n", check, entry_cnt); MT_BUG_ON(mt, check != entry_cnt); check = 0; @@ -999,10 +1011,13 @@ static noinline void check_erase2_testset(struct maple_tree *mt, if (check > entry_cnt) break; } - printk("mas_for_each check = %d\n", check); + printk("mas_for_each check = %d entry_cnt %d\n", check, + entry_cnt); mt_dump(mas.tree); MT_BUG_ON(mt, check != entry_cnt); + + MT_BUG_ON(mt, mtree_load(mas.tree, 0)); } } @@ -2126,9 +2141,15 @@ static int maple_tree_seed(void) mtree_init(&tree, 0); /* Test inserting into a NULL hole. */ + mt_dump(&tree); check_insert(&tree, set[5], ptr); // insert 1001 -> ptr + mt_dump(&tree); + printk("insert 1003\n"); check_insert(&tree, set[7], &tree); // insert 1003 -> &tree + mt_dump(&tree); check_insert(&tree, set[6], ptr); // insert 1002 -> ptr + mt_dump(&tree); + printk("Loading\n"); check_load(&tree, set[5], ptr); // See if 1001 -> ptr check_load(&tree, set[6], ptr); // See if 1002 -> ptr check_load(&tree, set[7], &tree); // See if 1003 -> &tree