From: Liam R. Howlett Date: Wed, 23 Sep 2020 02:43:38 +0000 (-0400) Subject: maple_tree: Add CONFIG_MAPLE_RCU_DISABLED, drop full_cnt, optimize _mas_store X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=ceb5227dec1d9dbc38e3125981a65bf56575b5fe;p=users%2Fjedix%2Flinux-maple.git maple_tree: Add CONFIG_MAPLE_RCU_DISABLED, drop full_cnt, optimize _mas_store Signed-off-by: Liam R. Howlett --- diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 5239bd1469e7..e4e8a24b9911 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -191,7 +191,6 @@ struct ma_state { unsigned long max; /* The maximum index of this node - implied pivot max */ struct maple_node *alloc; /* Allocated nodes for this operation */ struct maple_enode *span_enode; /* Pointer to maple parent/slot that set the max */ - int full_cnt; /* (+ full nodes) / (- almost empty) nodes above */ unsigned char depth; /* depth of tree descent during write */ }; @@ -362,6 +361,9 @@ static inline void mt_init(struct maple_tree *mt) static inline bool mt_in_rcu(struct maple_tree *mt) { +#ifdef CONFIG_MAPLE_RCU_DISABLED + return false; +#endif return !!(mt->ma_flags & MAPLE_USE_RCU); } /** diff --git a/lib/maple_tree.c b/lib/maple_tree.c index feb164fc770b..905bc28f7fb6 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -1560,14 +1560,14 @@ static inline void mas_descend_adopt(struct ma_state *mas) */ static inline unsigned char mas_store_b_node(struct ma_state *mas, struct maple_big_node *b_node, - void *entry) + void *entry, unsigned char end) { unsigned char slot = mas_offset(mas); - unsigned char end = mas_data_end(mas); void *contents = mas_get_slot(mas, slot); unsigned char b_end = 0; // Possible underflow of piv will wrap back to 0 before use. unsigned long piv = mas->min - 1; + unsigned long *pivots = ma_pivots(mas_mn(mas), b_node->type); // Copy start data up to insert. if (slot) { @@ -1589,7 +1589,7 @@ static inline unsigned char mas_store_b_node(struct ma_state *mas, b_node->pivot[b_end] = mas->last; // Handle range overlap end. - piv = mas_safe_pivot(mas, slot); + piv = _mas_safe_pivot(mas, pivots, slot, b_node->type); if (piv > mas->last) { b_node->slot[++b_end] = contents; if (!contents) @@ -1604,7 +1604,7 @@ static inline unsigned char mas_store_b_node(struct ma_state *mas, // Handle range overwrites do { - piv = mas_safe_pivot(mas, ++slot); + piv = _mas_safe_pivot(mas, pivots, ++slot, b_node->type); } while ((piv <= mas->last) && (slot <= end)); // Copy end data to the end of the node. @@ -2292,13 +2292,6 @@ new_root: return mast->bn->b_end; } -static inline int mas_cnt_positive(struct ma_state *mas) -{ - if (mas->full_cnt < 0) - return mas->full_cnt * -1; - return mas->full_cnt; -} - /* * mas_rebalance() - Rebalance a given node. * @@ -2313,7 +2306,7 @@ static inline int mas_cnt_positive(struct ma_state *mas) static inline int mas_rebalance(struct ma_state *mas, struct maple_big_node *b_node) { - char empty_cnt = mas_cnt_positive(mas); + char empty_cnt = mas_mt_height(mas); struct maple_subtree_state mast; unsigned char shift, b_end = ++b_node->b_end; @@ -2533,8 +2526,9 @@ static inline int mas_split(struct ma_state *mas, MA_TOPIARY(mat, mas->tree); trace_mas_split(mas); + mas->depth = mas_mt_height(mas); // Allocation failures will happen early. - mas_node_cnt(mas, 1 + mas->full_cnt * 2); + mas_node_cnt(mas, 1 + mas->depth * 2); if (mas_is_err(mas)) return 0; @@ -2545,8 +2539,7 @@ static inline int mas_split(struct ma_state *mas, mast.free = &mat; mast.bn = b_node; - mas->depth = mas_mt_height(mas); - while (height++ <= mas->full_cnt) { + while (height++ <= mas->depth) { if (mas_split_final_node(&mast, mas, height)) break; @@ -2708,12 +2701,9 @@ exists: * @entry - the entry that is going to be written. * */ -static inline bool mas_is_span_wr(struct ma_state *mas, unsigned long piv, +bool mas_is_span_wr(struct ma_state *mas, unsigned long piv, enum maple_type type, void *entry) { - if (mas->span_enode) // Already a spanning store. - return true; - if (piv > mas->last) // Contained in this pivot return false; @@ -2724,7 +2714,6 @@ static inline bool mas_is_span_wr(struct ma_state *mas, unsigned long piv, return false; if (ma_is_leaf(type)) { - trace_mas_is_span_wr(mas, piv, entry); if (mas->last < mas->max) // Fits in the node, but may span slots. return false; if (entry && (mas->last == mas->max)) // Writes to the end of the node but not null. @@ -2782,22 +2771,6 @@ dense: return ret; } -static inline void mas_cnt_full(struct ma_state *mas) -{ - if (mas->full_cnt < 0) - mas->full_cnt = 1; - else - mas->full_cnt++; -} - -static inline void mas_cnt_empty(struct ma_state *mas) -{ - if (mas->full_cnt > 0) - mas->full_cnt = -1; - else - mas->full_cnt--; -} - /* * mas_wr_walk(): Walk the tree for a write. * @range_min - pointer that will be set to the minimum of the slot range @@ -2807,18 +2780,14 @@ static inline void mas_cnt_empty(struct ma_state *mas) * * Tracks extra information which is used in special cases of a write. */ -static inline bool mas_wr_walk(struct ma_state *mas, unsigned long *range_min, +bool mas_wr_walk(struct ma_state *mas, unsigned long *range_min, unsigned long *range_max, void *entry) { enum maple_type type; - unsigned char end; bool ret = false; - mas->span_enode = NULL; - mas->full_cnt = 0; mas->depth = 0; - while (true) { type = mte_node_type(mas->node); mas->depth++; @@ -2832,14 +2801,6 @@ static inline bool mas_wr_walk(struct ma_state *mas, unsigned long *range_min, if (ma_is_leaf(type)) return true; - end = mas_data_end(mas); - if (end < mt_min_slots[type]) - mas_cnt_empty(mas); - else if (end >= mt_slots[type] - 1) - mas_cnt_full(mas); - else - mas->full_cnt = 0; - // Traverse. mas->max = *range_max; mas->min = *range_min; @@ -2950,22 +2911,19 @@ static inline bool __mas_walk(struct ma_state *mas, unsigned long *range_min, static inline int mas_spanning_store(struct ma_state *mas, void *entry) { unsigned long range_min, range_max; - unsigned char count = 0; struct maple_big_node b_node; struct maple_subtree_state mast; - int node_cnt = 0; + unsigned char height = mas_mt_height(mas); + int node_cnt = 1 + height * 3; // Holds new left and right sub-tree MA_STATE(l_mas, mas->tree, mas->index, mas->index); MA_STATE(r_mas, mas->tree, mas->index, mas->index); trace_mas_spanning_store(mas); - // Leaf nodes - node_cnt = mas_cnt_positive(mas); // For rebalance upwards. /* Node rebalancing may occur due to this store, so there may be two new * entries per level plus a new root. */ - node_cnt += 1 + mas_mt_height(mas) * 3; mas_node_cnt(mas, node_cnt); if (mas_is_err(mas)) return 0; @@ -3003,18 +2961,16 @@ static inline int mas_spanning_store(struct ma_state *mas, void *entry) // Copy l_mas and store the value in b_node. - b_node.b_end = mas_store_b_node(&l_mas, &b_node, entry); + b_node.b_end = mas_store_b_node(&l_mas, &b_node, entry, + mas_data_end(&l_mas)); // Copy r_mas into b_node. mas_mab_cp(&r_mas, mas_offset(&r_mas), mt_slot_count(r_mas.node), &b_node, b_node.b_end + 1); // Stop spanning searches by searching for just index. l_mas.index = l_mas.last = mas->index; - // Calc the number of iterations of combining and splitting that will - // need to occur. - count = mas_cnt_positive(mas) + mas_mt_height(mas) - mas->depth + 1; // Combine l_mas and r_mas and split them up evenly again. - return mas_spanning_rebalance(mas, &mast, count); + return mas_spanning_rebalance(mas, &mast, height + 1); } static inline bool mas_can_append(struct ma_state *mas, @@ -3082,13 +3038,13 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite if (!entry) mas_extend_null(mas, mas); + end = mas_data_end(mas); memset(&b_node, 0, sizeof(struct maple_big_node)); - b_node.b_end = mas_store_b_node(mas, &b_node, entry); - b_node.min = mas->min; b_node.type = mte_node_type(mas->node); + b_node.b_end = mas_store_b_node(mas, &b_node, entry, end); + b_node.min = mas->min; // Check if this is an append operation. - end = mas_data_end(mas); slot_cnt = mt_slot_count(mas->node); if (mas_can_append(mas, &b_node, slot_cnt, end)) { offset = b_node.b_end; @@ -3101,12 +3057,6 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite goto append; } - // count the node as full if it has not already been counted. - if (b_node.b_end >= slot_cnt && end < slot_cnt) - mas_cnt_full(mas); - else if (b_node.b_end < mt_min_slot_cnt(mas->node)) - mas_cnt_empty(mas); - if (!mas_commit_b_node(mas, &b_node, end)) return NULL;