From: Liam R. Howlett Date: Wed, 16 Oct 2019 19:31:37 +0000 (-0400) Subject: maple_tree: Fix coalesce/rebalance when erasing from the front of the X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=16d03b26b87b1a8b6ade7581a287be4824768ab9;p=users%2Fjedix%2Flinux-maple.git maple_tree: Fix coalesce/rebalance when erasing from the front of the tree. mas_coalesce_pivots added to set all null entries to the same pivot. mas_coalesce_empty added to coalesce pivots after setting the entry to NULL. mas_rebalance reworked to free empty & coalesce_pivots on allocation failure. Parent nodes are then checked. coalesce_root has a special case for when end == coalesce for handling allocation failures. mas_coalesce reworked to better perform when checking parent nodes & works when allocations fail. Gaps are now pushed to the right to avoid pulling the 0 pivot to the left. Also included: Add check for dense nodes. Fix mas_prev_slot called with root node. typo fix for rebalance comment Signed-off-by: Liam R. Howlett --- diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 9d949eaa6cf8..4f31c361eff9 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -115,6 +115,12 @@ static inline enum maple_type mte_node_type(const struct maple_enode *entry) return ((unsigned long)entry >> 3) & 15; } + +static inline bool ma_is_dense(const enum maple_type type) +{ + return type < maple_sparse_6; +} + static inline bool ma_is_leaf(const enum maple_type type) { return type < maple_range_16; @@ -2262,6 +2268,9 @@ static inline void mas_prev_slot(struct ma_state *mas, unsigned long min) { unsigned char slot, coalesce; + if (mte_is_root(mas->node)) + goto no_entry; + // Walk up. while (1) { slot = mte_parent_slot(mas->node); @@ -2598,12 +2607,57 @@ static inline int mas_coalesce_node(struct ma_state *mas, unsigned char end, return ma_hard_data(end, coalesce); } -static inline void mas_coalesce_empty(struct ma_state *mas, - struct maple_enode *eparent, unsigned char p_slot) +static inline void mas_coalesce_pivots(struct ma_state *mas, int slot, + enum maple_type type, bool skip) { - mte_set_rcu_slot(eparent, p_slot, XA_DELETED_ENTRY); + unsigned long piv_val = _mas_get_safe_pivot(mas, slot, type); + unsigned char pivot_cnt = mt_pivots[type]; + int last_null = slot; + + while ((slot < pivot_cnt - 1)) { + unsigned long this_pivot = mte_get_pivot(mas->node, slot + 1); + + if (!this_pivot) // end of node. + break; + + // There is data for this pivot. + if (!mt_is_empty(mte_get_rcu_slot(mas->node, slot + 1))) + break; + + // empty slot above the erase. + piv_val = this_pivot; + slot++; + } + + + /* Walk down and set all the previous pivots with NULLs to piv_val */ + while (--slot >= 0) { + void *entry = mte_get_rcu_slot(mas->node, slot); + + if (!mt_is_empty(entry)) { + last_null = slot + 1; + break; + } + if (!slot) + last_null = slot; + + if (skip) + mte_set_rcu_slot(mas->node, slot, XA_SKIP_ENTRY); + + mte_set_pivot(mas->node, slot, piv_val); + } + if (skip) + mte_set_rcu_slot(mas->node, last_null, NULL); + +} +static inline void mas_coalesce_empty(struct ma_state *mas, + struct maple_enode *eparent, unsigned char p_slot, + enum maple_type p_type) +{ + mte_set_rcu_slot(eparent, p_slot, NULL); mas_set_slot(mas, p_slot); + mas_coalesce_pivots(mas, p_slot, p_type, true); } /** Private @@ -2644,36 +2698,48 @@ static inline int mas_rebalance(struct ma_state *mas, unsigned char end, if (coalesce) coalesce--; + l_slot_cnt = ma_hard_data(end, coalesce); if (mte_is_root(this_enode)) return mas_coalesce_node(mas, end, coalesce, true); - + this_p_slot = mte_parent_slot(this_enode); mas_encoded_parent(mas); mas_set_slot(mas, this_p_slot + 1); if (!mas_next_nentry(mas, ULONG_MAX, &r_piv)) { - // this_enode is the right-most node. - BUG_ON(!l_slot_cnt); + // this_enode is the right-most node of the parent. mas->node = this_enode; mas->max = this_max; mas->min = this_min; - mas_set_slot(mas, this_p_slot); + //mas_set_slot(mas, this_p_slot); mas_encoded_parent(mas); mas_set_slot(mas, this_p_slot); - // If there is a single entry, rebalance the parent. if (mas_prev_nentry(mas, 0, &r_piv)) { + // If there is a node to the left, rebalance the left mas->node = mte_get_rcu_slot(mas->node, mas_get_slot(mas)); end = mas_data_end(mas, mte_node_type(mas->node), &l_piv, &coalesce); return mas_rebalance(mas, end, coalesce); } + // No left or right, rebalance the parent. + // But first, remove this entry if it's empty. mas->node = this_enode; mas->max = this_max; mas->min = this_min; mas_set_slot(mas, this_p_slot); + if (l_slot_cnt <= 1) { + enum maple_type p_type = + mas_parent_enum(mas, this_enode); + struct maple_enode *eparent = + mt_mk_node(mte_parent(this_enode), p_type); + mas_coalesce_empty(mas, eparent, this_p_slot, p_type); + } mas_encoded_parent(mas); mas_coalesce(mas); + if (mas_is_err(mas)) + return 0; + return 1; } // If we reached here, then the node to the right exists. @@ -2798,11 +2864,24 @@ static inline void mas_coalesce_root(struct ma_state *mas) this_type = mte_node_type(this_enode); end = mas_data_end(mas, this_type, &piv, &coalesce); hard_data = ma_hard_data(end, coalesce); + if (end == 1 && coalesce == 1) { + mas_first_node(mas, ULONG_MAX); + if (mas_is_none(mas)) { + // Allocation failures + + mas->tree->ma_root = NULL; + mte_free(this_enode); + return; + } + mas->node = this_enode; + } + if ((end <= coalesce)) { if(!mte_is_leaf(this_enode)) { // Remove level in tree. mas_set_slot(mas, 0); mas_first_node(mas, ULONG_MAX); + mte_to_node(mas->node)->parent = mte_to_node(this_enode)->parent; mas->node = mte_mk_root(mas->node); @@ -2849,12 +2928,13 @@ static inline void mas_coalesce_root(struct ma_state *mas) * * * Attempt to move things left with a rebalance. Upon failure, check if there - * is a contigeous gap from the end of this node to the start of the next. + * is a contiguous gap from the end of this node to the start of the next. * * Notes: * - Entries move Left, appending data at the end of the leaf * - Holes move Right. * - Either way, parent pivots need to be changed. + * - empty nodes will be replaced by skip entries on allocation failure. * * */ @@ -2862,41 +2942,52 @@ static inline void mas_coalesce(struct ma_state *mas) { unsigned char end, p_slot, coalesce; struct maple_enode *this_enode, *eparent; - enum maple_type this_type; - unsigned long piv; - void *entry; + enum maple_type this_type, p_type; bool check_parent = false; + unsigned long piv, this_piv; + int h_data; + void *entry; +start: if (mte_is_root(mas->node)) return mas_coalesce_root(mas); -start: this_enode = mas->node; this_type = mte_node_type(this_enode); - end = mas_data_end(mas, this_type, &piv, &coalesce); + end = mas_data_end(mas, this_type, &this_piv, &coalesce); + p_type = mas_parent_enum(mas, this_enode); + p_slot = mte_parent_slot(this_enode); + eparent = mt_mk_node(mte_parent(this_enode), p_type); /* If there is any space to save, try to reallocate */ - if (ma_hard_data(end, coalesce) < mt_min_slots[this_type] - 1) { + h_data = ma_hard_data(end, coalesce); + if (h_data < mt_min_slots[this_type] - 1) { if (mas_rebalance(mas, end, coalesce)) goto done; - if (mas_is_err(mas)) - return; + if (mas_is_err(mas)) { + if (h_data >= 1) + return; + // Single entry and allocation failed, free this_enode + mas->node = this_enode; + mas_encoded_parent(mas); + mas_coalesce_empty(mas, eparent, p_slot, p_type); + check_parent = true; + mas->node = this_enode; + goto done; + } } /* Group the gaps together. Acquire any data from the next node, if * necessary */ - p_slot = mte_parent_slot(this_enode); - eparent = mt_mk_node(mte_parent(this_enode), - mas_parent_enum(mas, this_enode)); entry = mte_get_rcu_slot(this_enode, end); if (!mt_is_empty(entry)) goto check_start; if (!end || end + 1 <= coalesce) { - mas_coalesce_empty(mas, eparent, p_slot); + mas_coalesce_empty(mas, eparent, p_slot, p_type); check_parent = true; } @@ -2919,12 +3010,12 @@ start: } mas->node = eparent; - mas_update_limits(mas, p_slot, mte_node_type(mas->node)); + mas_update_limits(mas, p_slot, p_type); mas->node = this_enode; if (!slot) { // Empty node... - mas_coalesce_empty(mas, eparent, p_slot); + mas_coalesce_empty(mas, eparent, p_slot, p_type); check_parent = true; piv = mas->min; } else { @@ -2950,7 +3041,7 @@ check_start: unsigned char prev_end; // Check the previous node. mas_prev_slot(mas, 0); - if (mas->node == MAS_NONE) + if (mas_is_none(mas)) goto done; if (!mt_is_empty(mas->node)) { @@ -2960,14 +3051,13 @@ check_start: if (!mt_is_empty(mte_get_rcu_slot(mas->node, prev_end))) goto done; } else { - piv = mas->min; + piv = mte_get_pivot(this_enode, 0); } if (p_slot) mte_set_pivot(eparent, p_slot - 1, piv); // Walk up and set all the pivots to piv - } done: mas->node = this_enode; @@ -4069,9 +4159,7 @@ static inline bool mas_skip_node(struct ma_state *mas) static inline void *mas_erase(struct ma_state *mas) { enum maple_type type; - unsigned char pivot_cnt; - unsigned long piv_val; - int slot, ret = 1; + int slot; void *entry = NULL; _mas_walk(mas); @@ -4086,41 +4174,12 @@ static inline void *mas_erase(struct ma_state *mas) return NULL; type = mte_node_type(mas->node); - pivot_cnt = mt_pivots[type]; entry = mte_get_rcu_slot(mas->node, slot); mte_update_rcu_slot(mas->node, slot, XA_DELETED_ENTRY); // dense nodes only need to set a single value. - if (!pivot_cnt) - goto done; + if (!ma_is_dense(type)) + mas_coalesce_pivots(mas, slot, type, false); - piv_val = _mas_get_safe_pivot(mas, slot, type); - while ((slot < pivot_cnt - 1)) { - unsigned long this_pivot = mte_get_pivot(mas->node, slot + 1); - - if (!this_pivot) // end of node. - break; - - // There is data for this pivot. - if (!mt_is_empty(mte_get_rcu_slot(mas->node, slot + 1))) - break; - - // empty slot above the erase. - piv_val = this_pivot; - slot++; - } - - /* Walk down and set all the previous pivots with NULLs to piv_val */ - while (--slot >= 0) { - void *entry = mte_get_rcu_slot(mas->node, slot); - - if (!mt_is_empty(entry)) - break; - - mte_set_pivot(mas->node, slot, piv_val); - ret++; - } - -done: mas_coalesce(mas); return entry; }