From 157a32329a7b87bfbac0e19b611d48e2ad015fd2 Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Tue, 19 Jan 2021 10:14:18 -0500 Subject: [PATCH] maple_tree: Comments, drop unused, name cleaning Signed-off-by: Liam R. Howlett --- include/linux/maple_tree.h | 1 + lib/maple_tree.c | 889 +++++++++++++++++++++++++++++-------- 2 files changed, 700 insertions(+), 190 deletions(-) diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 56a34309c219..7810321a35d8 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -30,6 +30,7 @@ #define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ +#define MAPLE_ARANGE64_META_MAX 15 /* Out of range for metadata */ #define MAPLE_NODE_MASK 255UL #else /* Need to do corresponding calculations for 32-bit kernels */ diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 86624da32c8c..9185f6c42b4b 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -108,11 +108,10 @@ static void mt_free_rcu(struct rcu_head *head) } /* ma_free_rcu() - Use rcu callback to free a maple node - * node: The node to free + * @node: The node to free * * The maple tree uses the parent pointer to indicate this node is no longer in * use and will be freed. - * Internal only. */ static void ma_free_rcu(struct maple_node *node) { @@ -205,25 +204,30 @@ static inline struct maple_node *mte_to_node(const struct maple_enode *entry) return (struct maple_node *)((unsigned long)entry & ~127); } -/* mte_to_mat() - Convert a maple encoded node to a maple topiary node. +/* + * mte_to_mat() - Convert a maple encoded node to a maple topiary node. * @entry: The maple encoded node - * Returns a maple topiary pointer + * + * Return: a maple topiary pointer */ static inline struct maple_topiary *mte_to_mat(const struct maple_enode *entry) { return (struct maple_topiary *)((unsigned long)entry & ~127); } -/* mas_mn() - Get the maple state node. +/* + * mas_mn() - Get the maple state node. * @mas: The maple state - * Returns the maple node (not encoded - bare pointer). + * + * Return: the maple node (not encoded - bare pointer). */ static inline struct maple_node *mas_mn(const struct ma_state *mas) { return mte_to_node(mas->node); } -/* mte_set_node_dead() - Set a maple encoded node as dead. +/* + * mte_set_node_dead() - Set a maple encoded node as dead. * @mn: The maple encoded node. */ static inline void mte_set_node_dead(struct maple_enode *mn) @@ -347,7 +351,7 @@ static inline void mte_set_parent(struct maple_enode *enode, * mte_parent_slot() - get the parent slot of @enode. * @enode: The encoded maple node. * - * Returns: The slot in the parent node where @enode resides. + * Return: The slot in the parent node where @enode resides. */ static inline unsigned int mte_parent_slot(const struct maple_enode *enode) { @@ -364,7 +368,7 @@ static inline unsigned int mte_parent_slot(const struct maple_enode *enode) * mte_parent() - Get the parent of @node. * @node: The encoded maple node. * - * Returns: The parent maple node. + * Return: The parent maple node. */ static inline struct maple_node *mte_parent(const struct maple_enode *enode) { @@ -374,7 +378,7 @@ static inline struct maple_node *mte_parent(const struct maple_enode *enode) /* * mte_dead_node() - check if the @enode is dead. - * Returns: true if dead, false otherwise. + * Return: true if dead, false otherwise. */ static inline bool mte_dead_node(const struct maple_enode *enode) { @@ -388,9 +392,11 @@ static inline bool mte_dead_node(const struct maple_enode *enode) * mas_allocated() - Get the number of nodes allocated in a maple state. * @mas: The maple state * - * Walks through the allocated nodes and returns the number allocated. + * If @mas->alloc has bit 1 set (0x1), then there is a request for + * (@mas->alloc >> 1) nodes. See mas_alloc_req(). Otherwise, there is a total + * of @mas->alloc->total nodes allocated. * - * Returns: The total number of nodes allocated + * Return: The total number of nodes allocated * */ static inline unsigned long mas_allocated(const struct ma_state *mas) @@ -405,6 +411,10 @@ static inline unsigned long mas_allocated(const struct ma_state *mas) * mas_set_alloc_req() - Set the requested number of allocations. * @mas: the maple state * @count: the number of allocations. + * + * If @mas->alloc has bit 1 set (0x1) or @mas->alloc is %NULL, then there are no + * nodes allocated and @mas->alloc should be set to count << 1 | 1. If there is + * already nodes allocated, then @mas->alloc->request_count stores the request. */ static inline void mas_set_alloc_req(struct ma_state *mas, unsigned long count) { @@ -421,9 +431,12 @@ static inline void mas_set_alloc_req(struct ma_state *mas, unsigned long count) /* * mas_alloc_req() - get the requested number of allocations. - * @mas: The maple state. + * @mas: The maple state + * + * The alloc count is either stored directly in @mas, or in + * @mas->alloc->request_count if there is at least one node allocated. * - * Returns: The allocation request count. + * Return: The allocation request count. */ static inline unsigned int mas_alloc_req(const struct ma_state *mas) { @@ -435,11 +448,11 @@ static inline unsigned int mas_alloc_req(const struct ma_state *mas) } /* - * ma_pivots() - Get the pivot of a node. - * @node - the maple node. - * @type - the node type. + * ma_pivots() - Get a pointer to the maple node pivots. + * @node - the maple node + * @type - the node type * - * Returns: The value of the @piv in the @node. + * Return: A pointer to the maple node pivots */ static inline unsigned long *ma_pivots(struct maple_node *node, enum maple_type type) @@ -456,6 +469,13 @@ static inline unsigned long *ma_pivots(struct maple_node *node, } } +/* + * ma_gaps() - Get a pointer to the maple node gaps. + * @node - the maple node + * @type - the node type + * + * Return: A pointer to the maple node gaps + */ static inline unsigned long *ma_gaps(struct maple_node *node, enum maple_type type) { @@ -470,6 +490,13 @@ static inline unsigned long *ma_gaps(struct maple_node *node, } } +/* + * mte_pivot() - Get the pivot at @piv of the maple encoded node. + * @mn: The maple encoded node. + * @piv: The pivot. + * + * Return: the pivot at @piv of @mn. + */ static inline unsigned long mte_pivot(const struct maple_enode *mn, unsigned char piv) { @@ -487,6 +514,16 @@ static inline unsigned long mte_pivot(const struct maple_enode *mn, } } +/* + * _mas_safe_pivot() - get the pivot at @piv or mas->max. + * @mas: The maple state + * @pivots: The pointer to the maple node pivots + * @piv: The pivot to fetch + * @type: The maple node type + * + * Return: The pivot at @piv within the limit of the @pivots array, @mas->max + * otherwise. + */ static inline unsigned long _mas_safe_pivot(const struct ma_state *mas, unsigned long *pivots, unsigned char piv, enum maple_type type) @@ -499,8 +536,8 @@ _mas_safe_pivot(const struct ma_state *mas, unsigned long *pivots, /* * mas_safe_pivot() - Return pivot, implied or otherwise. - * @mas: The maple state. - * @piv: the pivot location. + * @mas: The maple state + * @piv: the pivot location * * Return: The pivot (including mas->max for the final piv) */ @@ -515,29 +552,51 @@ mas_safe_pivot(const struct ma_state *mas, unsigned char piv) /* * mas_safe_min() - Return the minimum for a given offset. + * @mas: The maple state + * @pivots: The pointer to the maple node pivots + * @offset: The offset into the pivot array * + * Returns: The minimum range value that is contained in @offset. */ static inline unsigned long -mas_safe_min(struct ma_state *mas, unsigned long *pivots, unsigned char piv) +mas_safe_min(struct ma_state *mas, unsigned long *pivots, unsigned char offset) { - if (unlikely(!piv)) + if (unlikely(!offset)) return mas->min; - return pivots[piv - 1] + 1; + return pivots[offset - 1] + 1; } - -// Check what the maximum value the pivot could represent. +/* + * mas_logical_pivot() - Get the logical pivot of a given offset. + * @mas: The maple state + * @pivots: The pointer to the maple node pivots + * @offset: The offset into the pivot array + * @type: The maple node type + * + * When there is no value at a pivot (beyond the end of the data), then the + * pivot is actually @mas->max. + * + * Return: the logical pivot of a given @offset. + */ static inline unsigned long mas_logical_pivot(struct ma_state *mas, unsigned long *pivots, - unsigned char piv, enum maple_type type) + unsigned char offset, enum maple_type type) { - unsigned long lpiv = _mas_safe_pivot(mas, pivots, piv, type); + unsigned long lpiv = _mas_safe_pivot(mas, pivots, offset, type); - if (!lpiv && piv) + if (!lpiv && offset) return mas->max; return lpiv; } + +/* + * ma_set_pivot() - Set a pivot to a value. + * @mn: The maple node + * @piv: The pivot offset + * @type: The maple node type + * @val: The value of the pivot + */ static inline void ma_set_pivot(struct maple_node *mn, unsigned char piv, enum maple_type type, unsigned long val) { @@ -556,12 +615,25 @@ static inline void ma_set_pivot(struct maple_node *mn, unsigned char piv, } } +/* + * mte_set_pivot() - Set a pivot to a value in an encoded maple node. + * @mn: The encoded maple node + * @piv: The pivot offset + * @val: The value of the pivot + */ static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv, unsigned long val) { return ma_set_pivot(mte_to_node(mn), piv, mte_node_type(mn), val); } +/* + * ma_slots() - Get a pointer to the maple node slots. + * @mn: The maple node + * @mt: The maple node type + * + * Return: A pointer to the maple node slots + */ static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt) { switch (mt) { @@ -576,6 +648,14 @@ static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt) } } +/* + * mas_slot_locked() - Get the slot value when holding the maple tree lock. + * @mas: The maple state + * @slots: The pointer to the slots + * @offset: The offset into the slots array to fetch + * + * Return: The entry stored in @slots at the @offset. + */ static inline void *mas_slot_locked(struct ma_state *mas, void **slots, unsigned char offset) { @@ -585,6 +665,14 @@ static inline void *mas_slot_locked(struct ma_state *mas, void **slots, return slots[offset]; } +/* + * mas_slot() - Get the slot value when not holding the maple tree lock. + * @mas: The maple state + * @slots: The pointer to the slots + * @offset: The offset into the slots array to fetch + * + * Return: The entry stored in @slots at the @offset + */ static inline void *mas_slot(struct ma_state *mas, void **slots, unsigned char offset) { @@ -594,6 +682,13 @@ static inline void *mas_slot(struct ma_state *mas, void **slots, return slots[offset]; } +/* + * mas_get_slot() - Get the entry in the maple state node stored at @offset. + * @mas: The maple state + * @offset: The offset into the slot array to fetch. + * + * Return: The entry stored at @offset. + */ static inline struct maple_enode *mas_get_slot(struct ma_state *mas, unsigned char offset) { @@ -601,6 +696,12 @@ static inline struct maple_enode *mas_get_slot(struct ma_state *mas, offset); } +/* + * mas_root() - Get the maple tree root. + * @mas: The maple state. + * + * Return: The pointer to the root of the tree + */ static inline void *mas_root(struct ma_state *mas) { if (mt_in_rcu(mas->tree)) @@ -609,6 +710,12 @@ static inline void *mas_root(struct ma_state *mas) return mas->tree->ma_root; } +/* + * mas_root_locked() - Get the maple tree root when holding the maple tree lock. + * @mas: The maple state. + * + * Return: The pointer to the root of the tree + */ static inline void *mas_root_locked(struct ma_state *mas) { if (mt_in_rcu(mas->tree)) @@ -617,6 +724,7 @@ static inline void *mas_root_locked(struct ma_state *mas) return mas->tree->ma_root; } + /* * ma_set_slot() - Set a nodes rcu slot. * @@ -647,6 +755,9 @@ static inline void ma_set_slot(struct maple_node *mn, /* * mte_set_slot() - Set an encoded nodes rcu slot. + * @mn: The encoded maple node + * @slot: The offset into the slots array + * @val: The entry to store into the slot. */ static inline void mte_set_slot(const struct maple_enode *mn, unsigned char slot, void *val) @@ -654,26 +765,53 @@ static inline void mte_set_slot(const struct maple_enode *mn, ma_set_slot(mte_to_node(mn), slot, mte_node_type(mn), val); } + #define MA_META_END_MASK 0b1111 #define MA_META_GAP_SHIFT 4 +/* + * ma_set_meta() - Set the metadata information of a node. + * @mn: The maple node + * @mt: The maple node type + * @offset: The offset of the highest sub-gap in this node. + * @end: The end of the data in this node. + */ static inline void ma_set_meta(struct maple_node *mn, enum maple_type mt, unsigned char offset, unsigned char end) { mn->ma64.meta = (offset << MA_META_GAP_SHIFT) | end; } + +/* + * ma_meta_end() - Get the data end of a node from the metadata + * @mn: The maple node + * @mt: The maple node type + */ static inline unsigned char ma_meta_end(struct maple_node *mn, enum maple_type mt) { return mn->ma64.meta & MA_META_END_MASK; } + +/* + * ma_meta_gap() - Get the largest gap location of a node from the metadat + * @mn: The maple node + * @mt: The maple node type + */ static inline unsigned char ma_meta_gap(struct maple_node *mn, enum maple_type mt) { return mn->ma64.meta >> MA_META_GAP_SHIFT; } + +/* + * ma_set_meta_gap() - Set the largest gap location in a nodes metadata + * @mn: The maple node + * @mn: The maple node type + * @offset: The location of the largest gap. + */ static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, unsigned char offset) { @@ -681,13 +819,13 @@ static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, mn->ma64.meta = (offset << MA_META_GAP_SHIFT) | (mn->ma64.meta & MA_META_END_MASK); } + /* * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. - * - * Add the %dead_enode to the linked list in %mat. - * * @mat - the ma_topiary, a linked list of dead nodes. * @dead_enode - the node to be marked as dead and added to the tail of the list + * + * Add the @dead_enode to the linked list in @mat. */ static inline void mat_add(struct ma_topiary *mat, struct maple_enode *dead_enode) @@ -704,16 +842,16 @@ static inline void mat_add(struct ma_topiary *mat, } static void mte_destroy_walk(struct maple_enode *, struct maple_tree *); +static inline void mas_free(struct ma_state *mas, struct maple_enode *used); + /* * mat_free() - Free all nodes in a dead list. - * - * Free or destroy walk a dead list. - * * @mat - the ma_topiary linked list of dead nodes to free. * @recursive - specifies if this sub-tree is to be freed or just the single * node. + * + * Free or destroy walk a dead list. */ -static inline void mas_free(struct ma_state *mas, struct maple_enode *used); static void mas_mat_free(struct ma_state *mas, struct ma_topiary *mat, bool recursive) { @@ -731,7 +869,6 @@ static void mas_mat_free(struct ma_state *mas, struct ma_topiary *mat, /* * mas_dup_state() - duplicate the internal state of a ma_state. - * * @dst - the destination to store the state information * @src - the source of the state information */ @@ -749,7 +886,6 @@ static inline void mas_dup_state(struct ma_state *dst, struct ma_state *src) /* * mas_descend() - Descend into the slot stored in the ma_state. - * * @mas - the maple state. */ static inline void mas_descend(struct ma_state *mas) @@ -770,6 +906,12 @@ static inline void mas_descend(struct ma_state *mas) mas->node = mas_slot(mas, slots, mas->offset); } +/* + * mte_set_gap() - Set a maple node gap. + * @mn: The encoded maple node + * @gap: The offset of the gap to set + * @val: The gap value + */ static inline void mte_set_gap(const struct maple_enode *mn, unsigned char gap, unsigned long val) { @@ -782,6 +924,14 @@ static inline void mte_set_gap(const struct maple_enode *mn, } } +/* + * mas_ascend() - Walk up a level of the tree. + * @mas: The maple state + * + * Sets the @mas->max and @mas->min to the correct values when walking up. This + * may cause several levels of walking up to find the correct min and max. + * May find a dead node which will cause a premature return. + */ static void mas_ascend(struct ma_state *mas) { struct maple_enode *p_enode; // parent enode. @@ -840,6 +990,12 @@ static void mas_ascend(struct ma_state *mas) return; } +/* + * mas_pop_node() - Get a previously allocated maple node from the maple state. + * @mas: The maple state + * + * Return: A pointer to a maple node. + */ static inline struct maple_node *mas_pop_node(struct ma_state *mas) { struct maple_alloc *ret, *node = mas->alloc; @@ -875,6 +1031,15 @@ new_head: } return (struct maple_node *)ret; } + +/* + * mas_push_node() - Push a node back on the maple state allocation. + * @mas: The maple state + * @used: The used encoded maple node + * + * Stores the maple node back into @mas->alloc for reuse. Updates allocated and + * requested node count as necessary. + */ static inline void mas_push_node(struct ma_state *mas, struct maple_enode *used) { struct maple_alloc *reuse = (struct maple_alloc *)mte_to_node(used); @@ -907,6 +1072,11 @@ done: mas_set_alloc_req(mas, requested - 1); } +/* + * mas_alloc_nodes() - Allocate nodes into a maple state + * @mas: The maple state + * @gfp: The GFP Flags + */ static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp) { struct maple_alloc *node; @@ -971,6 +1141,14 @@ nomem: } +/* + * mas_free() - Free an encoded maple node + * @mas: The maple state + * @used: The encoded maple node to free. + * + * Uses rcu free if necessary, pushes @used back on the maple state allocations + * otherwise. + */ static inline void mas_free(struct ma_state *mas, struct maple_enode *used) { if (mt_in_rcu(mas->tree)) @@ -979,6 +1157,12 @@ static inline void mas_free(struct ma_state *mas, struct maple_enode *used) mas_push_node(mas, used); } +/* + * mas_node_count() - Check if enough nodes are allocated and request more if + * there is not enough nodes. + * @mas: The maple state + * @count: The number of nodes needed + */ static void mas_node_count(struct ma_state *mas, int count) { unsigned long allocated = mas_allocated(mas); @@ -990,9 +1174,13 @@ static void mas_node_count(struct ma_state *mas, int count) } /* - * Sets up maple state for operations by setting mas->min = 0 & mas->node to - * certain values. - * returns: + * mas_start() - Sets up maple state for operations. + * @mas: The maple state. + * + * If mas->node == MAS_START, then set the min, max, depth, and offset to + * defaults. + * + * Return: * - If mas->node is an error or MAS_START, return NULL. * - If it's an empty tree: NULL & mas->node == MAS_NONE * - If it's a single entry: The entry & mas->node == MAS_ROOT @@ -1037,7 +1225,10 @@ done: * @mas: the maple state * @type: the type of maple node * - * Returns: The zero indexed last slot with data (may be null). + * This method is optimized to check the metadata of a node if the node type + * supports data end metadata. + * + * Return: The zero indexed last slot with data (may be null). */ static inline unsigned char mas_data_end(struct ma_state *mas) { @@ -1076,8 +1267,9 @@ decrement: /* * mas_leaf_max_gap() - Returns the largest gap in a leaf node - * * @mas - the maple state + * + * Return: The maximum gap in the leaf. */ static unsigned long mas_leaf_max_gap(struct ma_state *mas) { @@ -1148,6 +1340,18 @@ static unsigned long mas_leaf_max_gap(struct ma_state *mas) } return max_gap; } + +/* + * ma_max_gap() - Get the maximum gap in a maple node (non-leaf) + * @node: The maple node + * @gaps: The pointer to the gaps + * @mt: The maple node type + * @*off: Pointer to store the offset location of the gap. + * + * Uses the metadata data end to scan backwards across set gaps. + * + * Return: The maximum gap value + */ static inline unsigned long ma_max_gap(struct maple_node *node, unsigned long *gaps, enum maple_type mt, unsigned char *off) @@ -1169,6 +1373,11 @@ ma_max_gap(struct maple_node *node, unsigned long *gaps, enum maple_type mt, /* * mas_max_gap() - find the largest gap in a non-leaf node and set the slot. + * @mas: The maple state. + * + * If the metadata gap is set to MAPLE_ARANGE64_META_MAX, there is no gap. + * + * Return: The gap value. */ static inline unsigned long mas_max_gap(struct ma_state *mas) { @@ -1183,13 +1392,22 @@ static inline unsigned long mas_max_gap(struct ma_state *mas) node = mas_mn(mas); offset = ma_meta_gap(node, mt); - if (offset == 15) + if (offset == MAPLE_ARANGE64_META_MAX) return 0; gaps = ma_gaps(node, mt); return gaps[offset]; } +/* + * mas_parent_gap() - Set the parent gap and any gaps above, as needed + * @mas: The maple state + * @offset: The gap offset in the parent to set + * @new: The new gap value. + * + * Set the parent gap then continue to set the gap upwards, using the metadata + * of the parent to see if it is necessary to check the node above. + */ static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset, unsigned long new) { @@ -1207,7 +1425,7 @@ static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset, ascend: meta_offset = ma_meta_gap(pnode, pmt); - if (meta_offset == 15) + if (meta_offset == MAPLE_ARANGE64_META_MAX) meta_gap = 0; else meta_gap = pgaps[meta_offset]; @@ -1242,7 +1460,6 @@ ascend: /* * mas_update_gap() - Update a nodes gaps and propagate up if necessary. - * * @mas - the maple state. */ static inline void mas_update_gap(struct ma_state *mas) @@ -1270,7 +1487,6 @@ static inline void mas_update_gap(struct ma_state *mas) /* * mas_adopt_children() - Set the parent pointer of all nodes in @parent to * @parent with the slot encoded. - * * @mas - the maple state (for the tree) * @parent - the maple encoded node containing the children. */ @@ -1294,7 +1510,6 @@ static inline void mas_adopt_children(struct ma_state *mas, /* * mas_replace() - Replace a maple node in the tree with mas->node. Uses the * parent encoding to locate the maple node in the tree. - * * @mas - the ma_state to use for operations. * @advanced - boolean to adopt the child nodes and free the old node (false) or * leave the node (true) and handle the adoption and free elsewhere. @@ -1342,7 +1557,6 @@ static inline void mas_replace(struct ma_state *mas, bool advanced) * mas_new_child() - Find the new child of a node. * @mas: the maple state * @child: the maple state to store the child. - * */ static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child) __must_hold(mas->tree->lock) @@ -1377,7 +1591,6 @@ static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child) /* * mab_shift_right() - Shift the data in mab right. Note, does not clean out the * old data or set b_node->b_end. - * * @b_node: the maple_big_node * @shift: the shift count */ @@ -1393,12 +1606,12 @@ static inline void mab_shift_right(struct maple_big_node *b_node, /* * mab_middle_node() - Check if a middle node is needed (unlikely) - * * @b_node: the maple_big_node that contains the data. * @size: the amount of data in the b_node * @split: the potential split location * @slot_count: the size that can be stored in a single node being considered. - * Returns: true if a middle node is required. + * + * Return: true if a middle node is required. */ static inline bool mab_middle_node(struct maple_big_node *b_node, int split, unsigned char slot_count) @@ -1416,11 +1629,11 @@ static inline bool mab_middle_node(struct maple_big_node *b_node, int split, /* * mab_no_null_split() - ensure the split doesn't fall on a NULL - * * @b_node: the maple_big_node with the data * @split: the suggested split location * @slot_count: the number of slots in the node being considered. - * Returns the split location. + * + * Return: the split location. */ static inline int mab_no_null_split(struct maple_big_node *b_node, unsigned char split, unsigned char slot_count) @@ -1441,10 +1654,10 @@ static inline int mab_no_null_split(struct maple_big_node *b_node, /* * mab_calc_split() - Calculate the split location and if there needs to be two * splits. - * * @b_node: The maple_big_node with the data * @mid_split: The second split, if required. 0 otherwise. - * Returns: The first split location. + * + * Return: The first split location. */ static inline int mab_calc_split(struct ma_state *mas, struct maple_big_node *b_node, @@ -1497,7 +1710,6 @@ static inline int mab_calc_split(struct ma_state *mas, /* * mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node * and set @b_node->b_end to the next free slot. - * * @mas: The maple state * @mas_start: The starting slot to copy * @mas_end: The end slot to copy (inclusively) @@ -1599,12 +1811,13 @@ static inline void mab_mas_cp(struct maple_big_node *b_node, } /* - * mas_descend_adopt() - Descend through a sub-tree and adopt children who do - * not have the correct parents set. Follow the parents which have the correct - * parents as they are the new entries which need to be followed to find other - * incorrectly set parents. - * + * mas_descend_adopt() - Descend through a sub-tree and adopt children. * @mas: the maple state with the maple encoded node of the sub-tree. + * + * Descend through a sub-tree and adopt children who do not have the correct + * parents set. Follow the parents which have the correct parents as they are + * the new entries which need to be followed to find other incorrectly set + * parents. */ static inline void mas_descend_adopt(struct ma_state *mas) { @@ -1642,6 +1855,12 @@ static inline void mas_descend_adopt(struct ma_state *mas) } } +/* + * mas_bulk_rebalance() - Rebalance the end of a tree after a bulk insert. + * @mas: The maple state + * @end: The maple node end + * @mt: The maple node type + */ static inline void mas_bulk_rebalance(struct ma_state *mas, unsigned char end, enum maple_type mt) { @@ -1656,14 +1875,15 @@ static inline void mas_bulk_rebalance(struct ma_state *mas, unsigned char end, return; } } + /* * mas_store_b_node() - Store an @entry into the b_node while also copying the * data from a maple encoded node. - * * @mas: the maple state * @b_node: the maple_big_node to fill with data * @entry: the data to store. - * Returns: The actual end of the data stored in @b_node + * + * Return: The actual end of the data stored in @b_node */ static inline unsigned char mas_store_b_node(struct ma_state *mas, struct maple_big_node *b_node, @@ -1740,9 +1960,9 @@ static inline void mas_node_walk(struct ma_state *mas, enum maple_type type, /* * mas_prev_sibling() - Find the previous node with the same parent. - * * @mas: the maple state - * Returns: True if there is a previous sibling, false otherwise. + * + * Return: True if there is a previous sibling, false otherwise. */ static inline bool mas_prev_sibling(struct ma_state *mas) { @@ -1761,10 +1981,10 @@ static inline bool mas_prev_sibling(struct ma_state *mas) } /* - *mas_next_sibling() - Find the next node with the same parent. - * + * mas_next_sibling() - Find the next node with the same parent. * @mas: the maple state - * Returns true if there is a next sibling, false otherwise. + * + * Return: true if there is a next sibling, false otherwise. */ static inline bool mas_next_sibling(struct ma_state *mas) { @@ -1789,6 +2009,14 @@ static inline bool mas_next_sibling(struct ma_state *mas) return true; } +/* + * mte_node_or_node() - Return the encoded node or MAS_NONE. + * @enode: The encoded maple node. + * + * Shorthand to avoid setting %NULLs in the tree or maple_subtree_state. + * + * Return: @enode or MAS_NONE + */ static inline struct maple_enode *mte_node_or_none(struct maple_enode *enode) { if (enode) @@ -1800,7 +2028,6 @@ static inline struct maple_enode *mte_node_or_none(struct maple_enode *enode) /* * mast_topiary() - Add the portions of the tree to the removal list; either to * be freed or discarded (destroy walk). - * * @mast: The maple_subtree_state. */ static inline void mast_topiary(struct maple_subtree_state *mast) @@ -1845,6 +2072,11 @@ static inline void mast_topiary(struct maple_subtree_state *mast) mat_add(mast->destroy, slots[offset]); } +/* + * mast_rebalance_next() - Rebalance against the next node + * @mast: The maple subtree state + * @old_r: The encoded maple node to the right (next node). + */ static inline void mast_rebalance_next(struct maple_subtree_state *mast, struct maple_enode *old_r) { @@ -1858,6 +2090,11 @@ static inline void mast_rebalance_next(struct maple_subtree_state *mast, mast->orig_l->node = mast->orig_r->node; } +/* + * mast_rebalace_prev() - Rebalance against the previous node + * @mast: The maple subtree state + * @old_l: The encoded maple node to the left (previous node) + */ static inline void mast_rebalance_prev(struct maple_subtree_state *mast, struct maple_enode *old_l) { @@ -1875,26 +2112,9 @@ static inline void mast_rebalance_prev(struct maple_subtree_state *mast, mast->l->offset += end + 1; } -static inline bool mast_sibling_rebalance_left(struct maple_subtree_state *mast) -{ - struct maple_enode *old_r = mast->orig_r->node; - struct maple_enode *old_l = mast->orig_l->node; - - if (mas_prev_sibling(mast->orig_l)) { - mast_rebalance_prev(mast, old_l); - return true; - } - if (mas_next_sibling(mast->orig_r)) { - mast_rebalance_next(mast, old_r); - return true; - } - return false; -} - /* * mast_sibling_rebalance_right() - Rebalance from nodes with the same parents. * Check the right side, then the left. Data is copied into the @mast->bn. - * * @mast: The maple_subtree_state. */ static inline bool mast_sibling_rebalance_right(struct maple_subtree_state *mast) @@ -1923,7 +2143,6 @@ static inline unsigned long mas_next_node(struct ma_state *mas, /* * mast_cousin_rebalance_right() - Rebalance from nodes with different parents. * Check the right side, then the left. Data is copied into the @mast->bn. - * * @mast: The maple_subtree_state. */ static inline bool mast_cousin_rebalance_right(struct maple_subtree_state *mast) @@ -2004,7 +2223,7 @@ mast_ascend_free(struct maple_subtree_state *mast) * Use the node type from the maple_big_node to allocate a new node from the * maple_state. This function exists mainly for code readability. * - * Returns: A new maple encoded node + * Return: A new maple encoded node */ static inline struct maple_enode *mas_new_ma_node(struct ma_state *mas, struct maple_big_node *b_node) @@ -2022,7 +2241,7 @@ static inline struct maple_enode * @middle: the pointer which may have the middle node (rare) * @mid_split: the split location for the middle node * - * Returns: the split of left. + * Return: the split of left. */ static inline unsigned char mas_mab_to_node(struct ma_state *mas, struct maple_big_node *b_node, @@ -2101,6 +2320,12 @@ static inline void mas_set_split_parent(struct ma_state *mas, /* * mte_mid_split_check() - Check if the next node passes the mid-split + * @**l: Pointer to left encoded maple node. + * @**m: Pointer to middle encoded maple node. + * @**r: Pointer to right encoded maple node. + * @slot: The offset + * @*split: The split location. + * @mid_split: The middle split. */ static inline void mte_mid_split_check(struct maple_enode **l, struct maple_enode **r, @@ -2123,7 +2348,6 @@ static inline void mte_mid_split_check(struct maple_enode **l, /* * mast_set_split_parents() - Helper function to set three nodes parents. Slot * is taken from @mast->l. - * * @mast - the maple subtree state * @left - the left node * @right - the right node @@ -2161,6 +2385,14 @@ static inline void mast_set_split_parents(struct maple_subtree_state *mast, mas_set_split_parent(mast->r, l, r, &slot, split); } +/* + * mas_wmb_replace() - Write memory barrier and replace + * @mas: The maple state + * @free: the maple topiary list of nodes to free + * @destroy: The maple topiary list of nodes to destroy (walk and free) + * + * Updates gap as necessary. + */ static inline void mas_wmb_replace(struct ma_state *mas, struct ma_topiary *free, struct ma_topiary *destroy) @@ -2185,6 +2417,11 @@ static inline void mas_wmb_replace(struct ma_state *mas, mas_update_gap(mas); } +/* + * mast_new_root() - Set a new tree root during subtree creation + * @mast: The maple subtree state + * @mas: The maple state + * */ static inline void mast_new_root(struct maple_subtree_state *mast, struct ma_state *mas) { @@ -2203,6 +2440,15 @@ static inline void mast_new_root(struct maple_subtree_state *mast, } } +/* + * mast_cp_to_nodes() - Copy data out to nodes. + * @mast: The maple subtree state + * @left: The left encoded maple node + * @middle: The middle encoded maple node + * @right: The right encoded maple node + * @split: The location to split between left and (middle ? middle : right) + * @mid_split: The location to split between middle and right. + */ static inline void mast_cp_to_nodes(struct maple_subtree_state *mast, struct maple_enode *left, struct maple_enode *middle, @@ -2234,8 +2480,9 @@ static inline void mast_cp_to_nodes(struct maple_subtree_state *mast, } /* - * Copy in the original left side of the tree into the combined data set in the - * maple subtree state big node. + * mast_combine_cp_left - Copy in the original left side of the tree into the + * combined data set in the maple subtree state big node. + * @mast: The maple subtree state */ static inline void mast_combine_cp_left(struct maple_subtree_state *mast) { @@ -2248,8 +2495,9 @@ static inline void mast_combine_cp_left(struct maple_subtree_state *mast) } /* - * Copy in the original right side of the tree into the combined data set in - * the maple subtree state big node. + * mast_combine_cp_right: Copy in the original right side of the tree into the + * combined data set in the maple subtree state big node. + * @mast: The maple subtree state */ static inline void mast_combine_cp_right(struct maple_subtree_state *mast) { @@ -2262,8 +2510,10 @@ static inline void mast_combine_cp_right(struct maple_subtree_state *mast) mast->orig_r->last = mast->orig_r->max; } -/* Check if the maple subtree state has enough data in the big node to create at - * least one sufficient node +/* + * mast_sufficient: Check if the maple subtree state has enough data in the big + * node to create at least one sufficient node + * @mast: the maple subtree state */ static inline bool mast_sufficient(struct maple_subtree_state *mast) { @@ -2273,6 +2523,11 @@ static inline bool mast_sufficient(struct maple_subtree_state *mast) return false; } +/* + * mast_overflow: Check if there is too much data in the subtree state for a + * single node. + * @mast: The maple subtree state + */ static inline bool mast_overflow(struct maple_subtree_state *mast) { if (mast->bn->b_end >= mt_slot_count(mast->orig_l->node)) @@ -2281,6 +2536,11 @@ static inline bool mast_overflow(struct maple_subtree_state *mast) return false; } +/* + * mast_setup_bnode_for_split() - Prepare the subtree state big node for + * splitting + * @mast: The maple subtree state + */ static inline void mast_setup_bnode_for_split(struct maple_subtree_state *mast) { mast->bn->b_end--; @@ -2305,7 +2565,7 @@ static inline void mast_setup_bnode_for_split(struct maple_subtree_state *mast) * be either freed or destroyed. orig_l_mas->depth keeps track of the height of * the new sub-tree in case the sub-tree becomes the full tree. * - * Returns the number of elements in b_node during the last loop. + * Return: the number of elements in b_node during the last loop. */ static int mas_spanning_rebalance(struct ma_state *mas, struct maple_subtree_state *mast, @@ -2417,7 +2677,7 @@ new_root: * Rebalance two nodes into a single node or two new nodes that are sufficient. * Continue upwards until tree is sufficient. * - * Returns the number of elements in b_node during the last loop. + * Return: the number of elements in b_node during the last loop. */ static inline int mas_rebalance(struct ma_state *mas, struct maple_big_node *b_node) { @@ -2458,6 +2718,15 @@ static inline int mas_rebalance(struct ma_state *mas, struct maple_big_node *b_n return mas_spanning_rebalance(mas, &mast, empty_count); } +/* + * mas_destroy_rebalance() - Rebalance left-most node while destroying the maple + * state. + * @mas: The maple state + * @end: The end of the left-most node. + * + * During a mass-insert event (such as forking), it may be necessary to + * rebalance the left-most node when it is not sufficient. + */ static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end) { enum maple_type mt = mte_node_type(mas->node); @@ -2570,6 +2839,12 @@ done: mas_update_gap(mas); } +/* + * _mas_split_final_node() - Split the final node in a subtree operation. + * @mast: the maple subtree state + * @mas: The maple state + * @height: The height of the tree in case it's a new root. + */ static inline bool _mas_split_final_node(struct maple_subtree_state *mast, struct ma_state *mas, int height) { @@ -2583,7 +2858,7 @@ static inline bool _mas_split_final_node(struct maple_subtree_state *mast, mas->depth = height; } /* Only a single node is used here, could be root. - * Big_node should just fit in a single node. + * The Big_node data should just fit in a single node. */ ancestor = mas_new_ma_node(mas, mast->bn); mte_set_parent(mast->l->node, ancestor, mast->l->offset); @@ -2596,6 +2871,16 @@ static inline bool _mas_split_final_node(struct maple_subtree_state *mast, return true; } +/* + * mas_split_final_node() - Check if a subtree state can be contained within a + * single node and do so if possible. + * @mast: The maple subtree state + * @mas: The maple state + * @height: The height in case of a new root. + * + * Return: True if this was the final node and it has been handled, false + * otherwise. + */ static inline bool mas_split_final_node(struct maple_subtree_state *mast, struct ma_state *mas, int height) { @@ -2605,6 +2890,12 @@ static inline bool mas_split_final_node(struct maple_subtree_state *mast, return _mas_split_final_node(mast, mas, height); } +/* + * mast_fill_bnode() - Copy data into the big node in the subtree state + * @mast: The maple subtree state + * @mas: the maple state + * @skip: The number of entries to skip for new nodes insertion. + */ static inline void mast_fill_bnode(struct maple_subtree_state *mast, struct ma_state *mas, unsigned char skip) @@ -2647,6 +2938,13 @@ static inline void mast_fill_bnode(struct maple_subtree_state *mast, sizeof(unsigned long) * zero); } +/* + * mast_split_data() - Split the data in the subtree state big node into regular + * nodes. + * @mast: The maple subtree state + * @mas: The maple state + * @split: The location to split the big node + */ static inline void mast_split_data(struct maple_subtree_state *mast, struct ma_state *mas, unsigned char split) @@ -2668,6 +2966,18 @@ static inline void mast_split_data(struct maple_subtree_state *mast, } } +/* + * mas_push_data() - Instead of splitting a node, it is beneficial to push the + * data to the right or left node if there is room. + * @mas: The maple state + * @height: The current height of the maple state + * @mast: The maple subtree state + * @left: Push left or not. + * + * Keeping the height of the tree low means faster lookups. + * + * Return: True if pushed, false otherwise. + */ static inline bool mas_push_data(struct ma_state *mas, int height, struct maple_subtree_state *mast, bool left) { @@ -2732,18 +3042,12 @@ static inline bool mas_push_data(struct ma_state *mas, int height, return true; } -static inline bool mas_push_right(struct ma_state *mas, int height, - struct maple_subtree_state *mast) -{ - return mas_push_data(mas, height, mast, false); -} - -static inline bool mas_push_left(struct ma_state *mas, int height, - struct maple_subtree_state *mast) -{ - return mas_push_data(mas, height, mast, true); -} - +/* + * mas_split() - Split data that is too big for one node into two. + * @mas: The maple state + * @b_node: The maple big node + * Return: 1 on success, 0 on failure. + */ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) { @@ -2779,10 +3083,12 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) mas_dup_state(&r_mas, mas); l_mas.node = mas_new_ma_node(mas, b_node); r_mas.node = mas_new_ma_node(mas, b_node); - if (mas_push_left(mas, height, &mast)) + // Try to push left. + if (mas_push_data(mas, height, &mast, true)) break; - if (mas_push_right(mas, height, &mast)) + // Try to push right. + if (mas_push_data(mas, height, &mast, false)) break; split = mab_calc_split(mas, b_node, &mid_split); @@ -2801,6 +3107,16 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) return 1; } +/* + * mas_reuse_node() - Reuse the node to store the data. + * @mas: The maple state + * @bn: The maple big node + * @end: The end of the data. + * + * Will always return false in RCU mode. + * + * Return: True if node was reused, false otherwise. + */ static inline bool mas_reuse_node(struct ma_state *mas, struct maple_big_node *bn, unsigned char end) { @@ -2828,6 +3144,12 @@ static inline bool mas_reuse_node(struct ma_state *mas, } +/* + * mas_commit_b_node() - Commit the big node into the tree. + * @mas: The maple state + * @b_node: The maple big node + * @end: The end of the data. + */ static inline int mas_commit_b_node(struct ma_state *mas, struct maple_big_node *b_node, unsigned char end) { @@ -2861,6 +3183,11 @@ reused_node: return 2; } +/* + * mas_root_expand() - Expand a root to a node + * @mas: The maple state + * @entry: The entry to store into the tree + */ static inline int mas_root_expand(struct ma_state *mas, void *entry) { void *contents = mas_root_locked(mas); @@ -2899,7 +3226,15 @@ static inline int mas_root_expand(struct ma_state *mas, void *entry) return slot; } -static inline int ma_root_ptr(struct ma_state *mas, void *entry, bool overwrite) +/* + * mas_root_ptr() - Store entry into root. + * @mas: The maple state + * @entry: The entry to store + * @overwrite: If it is okay to overwrite data + * + * Return: 0 on success, 1 otherwise. + */ +static inline int mas_root_ptr(struct ma_state *mas, void *entry, bool overwrite) { int ret = 1; @@ -2925,13 +3260,17 @@ exists: } /* + * mas_is_span_wr() - Check if the write needs to be treated as a write that + * spans the node. + * @mas: The maple state + * @piv: The pivot value being written + * @type: The maple node type + * @entry: The data to write * - * mas_is_span_() - Check if the write spans the node. - * entry being written spans this nodes slot or touches the end of this slot and - * is NULL. - * @piv - the pivot of the slot in this node - * @entry - the entry that is going to be written. + * Spanning writes are writes that start in one node and end in another OR if + * the write of a %NULL will cause the node to end with a %NULL. * + * Return: True if this is a spanning write, false otherwise. */ bool mas_is_span_wr(struct ma_state *mas, unsigned long piv, enum maple_type type, void *entry) @@ -2964,6 +3303,16 @@ bool mas_is_span_wr(struct ma_state *mas, unsigned long piv, return true; } +/* + * mas_node_walk() - Walk a maple node to offset of the index. + * @mas: The maple state + * @type: The maple node type + * @*range_min: Pointer to store the minimum range of the offset + * @*range_max: Pointer to store the maximum range of the offset + * + * The offset will be stored in the maple state. + * + */ static inline void mas_node_walk(struct ma_state *mas, enum maple_type type, unsigned long *range_min, unsigned long *range_max) { @@ -3018,9 +3367,10 @@ done: * @range_min - pointer that will be set to the minimum of the slot range * @range_max - pointer that will be set to the maximum of the slot range * @entry - the value that will be written. - * Returns: True if found, false otherwise. * - * Tracks extra information which is used in special cases of a write. + * Uses mas_slot_locked() and does not need to worry about dead nodes. + * + * Return: True if it's contained in a node, false on spanning write. */ bool mas_wr_walk(struct ma_state *mas, unsigned long *range_min, unsigned long *range_max, void *entry) @@ -3048,6 +3398,11 @@ bool mas_wr_walk(struct ma_state *mas, unsigned long *range_min, return true; } +/* + * mas_extend_null() - Extend a store of a %NULL to include surrounding %NULLs. + * @l_mas: The left maple state + * @r_mas: The right maple state + */ static inline void mas_extend_null(struct ma_state *l_mas, struct ma_state *r_mas) { unsigned char l_slot = l_mas->offset; @@ -3101,11 +3456,13 @@ static inline void mas_extend_null(struct ma_state *l_mas, struct ma_state *r_ma * * __mas_walk(): Locates a value and sets the mas->node and slot accordingly. * range_min and range_max are set to the range which the entry is valid. - * Returns true if mas->node is a leaf. + * @mas: The maple state + * @*range_min: A pointer to store the minimum of the range + * @*range_max: A pointer to store the maximum of the range * - * Will not point to a skip entry. - * May point to a deleted or retry entry. + * Check mas->node is still valid on return of any value. * + * Return: true if pointing to a valid node and offset. False otherwise. */ static inline bool __mas_walk(struct ma_state *mas, unsigned long *range_min, unsigned long *range_max) @@ -3143,7 +3500,10 @@ static inline bool __mas_walk(struct ma_state *mas, unsigned long *range_min, * and new nodes where necessary, then place the sub-tree in the actual tree. * Note that mas is expected to point to the node which caused the store to * span. + * @mas: The maple state + * @entry: The entry to store. * + * Return: 0 on error, positive on success. */ static inline int mas_spanning_store(struct ma_state *mas, void *entry) { @@ -3208,6 +3568,19 @@ static inline int mas_spanning_store(struct ma_state *mas, void *entry) return mas_spanning_rebalance(mas, &mast, height + 1); } +/* + * mas_append() - Attempt to append data to the end of a node + * @mas: The maple state + * @entry: The entry to store + * @min: The minimum of the range + * @end: The end of the node + * @content: The contents of the slot currently + * @mt: The maple node type + * + * Appending never needs to allocate. + * + * Return: True if stored, false otherwise + */ static inline bool mas_append(struct ma_state *mas, void *entry, unsigned long min, unsigned char end, void *content, enum maple_type mt) @@ -3243,6 +3616,21 @@ static inline bool mas_append(struct ma_state *mas, void *entry, mas_update_gap(mas); return true; } + +/* + * mas_node_store() - Attempt to store the value in a node + * @mas: The maple state + * @entry: The value to store + * @min: The minimum of the range + * @max: The maximum of the range + * @mt: The maple node type + * @slots: Pointer to the slot array + * @pivots: Pointer to the pivot array + * + * Attempts to reuse the node, but may allocate. + * + * Return: True if stored, false otherwise + */ static inline bool mas_node_store(struct ma_state *mas, void *entry, unsigned long min, unsigned long max, unsigned char end, void *content, @@ -3348,6 +3736,19 @@ done: return true; } +/* + * mas_slot_store: Attempt to store a value in a slot. + * @mas: the maple state + * @entry: The entry to store + * @min: The range minimum + * @max: The range maximum + * @end: The end of the maple node + * @content: The current content + * @mt: The maple node type + * @slots: The pointer to the slots array + * + * Return: True if stored, false otherwise + */ static inline bool mas_slot_store(struct ma_state *mas, void *entry, unsigned long min, unsigned long max, unsigned char end, void *content, @@ -3409,6 +3810,14 @@ try_node_store: return mas_node_store(mas, entry, min, max, end, content, mt, slots, pivots); } +/* + * _mas_store() - Internal call to store a value + * @mas: The maple state + * @entry: The entry to store + * @overwrite: Allowed to overwrite entries or not + * + * Return: The contents that was stored at the index. + */ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite) { unsigned long r_max, r_min; @@ -3422,7 +3831,7 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite int ret = 0; if (mas_start(mas) || mas_is_none(mas) || mas->node == MAS_ROOT) { - ret = ma_root_ptr(mas, entry, overwrite); + ret = mas_root_ptr(mas, entry, overwrite); if (mas_is_err(mas)) return NULL; @@ -3523,6 +3932,11 @@ spanning_store: /* * mas_prev_node() - Find the prev non-null entry at the same level in the * tree. The prev value will be mas->node[mas->offset] or MAS_NONE. + * @mas: The maple state + * @limit: The lower limit to search + * + * Result needs to be checked if the node is dead. + * The prev node value will be mas->node[mas->offset] or MAS_NONE. */ static inline void mas_prev_node(struct ma_state *mas, unsigned long limit) { @@ -3600,12 +4014,10 @@ no_entry: * @mas: The maple state * @max: The maximum pivot value to check. * - * Returns: The next value will be mas->node[mas->offset] or MAS_NONE. + * Return needs to be checked for dead nodes. * - * Finds the next non-null entry at the same level in the tree. Slot is passed - * in the maple state offset, eg: mas->offset + * Return: The next value will be mas->node[mas->offset] or MAS_NONE. */ - static inline unsigned long mas_next_node(struct ma_state *mas, unsigned long max) { @@ -3623,7 +4035,6 @@ static inline unsigned long mas_next_node(struct ma_state *mas, if (mas->max >= max) goto no_entry; - // Save the location in case of dead node. level = 0; do { if (mte_is_root(mas->node)) @@ -3679,9 +4090,7 @@ no_entry: * @mas: The maple state. * @limit: The lower limit to check for a value. * - * Returns the entry, NULL otherwise. - * - * Internal only. + * Return: the entry, %NULL otherwise. */ static inline void *mas_prev_nentry(struct ma_state *mas, unsigned long limit) { @@ -3718,11 +4127,15 @@ static inline void *mas_prev_nentry(struct ma_state *mas, unsigned long limit) } /* - * mas_next_nentry() - Next node entry. Set the @mas slot to the next valid - * entry and range_start to the start value for that entry. If there is no - * entry, returns false. + * mas_next_nentry() - Get the next node entry + * @mas: The maple state + * @max: The maximum value to check + * @*range_start: Pointer to store the start of the range. + * + * Sets @mas->offset to the offset of the next node entry, @mas->last to the + * pivot of the entry. * - * Internal only. + * Return: The next entry, %NULL otherwise */ static inline void *mas_next_nentry(struct ma_state *mas, unsigned long max, unsigned long *range_start) @@ -3786,9 +4199,10 @@ found: * @mas: The maple state. * @*range_min: The minimum range to be set. * @*range_max: The maximum range to be set. - * Returns: True if a value exists, false otherwise. * * Ranges are only valid if there is a valid entry at @mas->index. + * + * Return: True if a value exists, false otherwise. */ static inline bool _mas_walk(struct ma_state *mas, unsigned long *range_min, unsigned long *range_max) @@ -3830,10 +4244,10 @@ not_found: /* * mas_dead_node() - Check if the maple state is pointing to a dead node. - * * @mas: The maple state * @index: The index to restore in @mas. - * Return 1 if @mas has been reset to MAS_START, 0 otherwise. + * + * Return: 1 if @mas has been reset to MAS_START, 0 otherwise. */ static inline int mas_dead_node(struct ma_state *mas, unsigned long index) { @@ -3853,10 +4267,13 @@ static inline int mas_dead_node(struct ma_state *mas, unsigned long index) /* * mas_first_entry() - Go the first leaf and find the first entry. - * * @mas: the maple state. * @limit: the maximum index to check. - * Returns: The start of the range. + * @*r_start: Pointer to set to the range start. + * + * Sets mas->offset to the offset of the entry, r_start to the range minimum. + * + * Return: The first entry or MAS_NONE. */ static inline void *mas_first_entry(struct ma_state *mas, unsigned long limit, unsigned long *r_start) @@ -3914,13 +4331,18 @@ done: *r_start = range_start; return entry; } + /* + * __mas_next() - Internal function to get the next entry. + * @mas: The maple state + * @limit: The maximum range start. * - * __mas_next() Set the @mas->node to the next entry and the range_start to + * Set the @mas->node to the next entry and the range_start to * the beginning value for the entry. Does not check beyond @limit. + * Sets @mas->index and @mas->last to the limit if it is hit. + * Restarts on dead nodes. * - * May return NULL. - * + * Return: the next entry or %NULL. */ static inline void *__mas_next(struct ma_state *mas, unsigned long limit) { @@ -3972,17 +4394,15 @@ next_node: } /* - * _mas_prev() - Return the previous entry + * _mas_prev() - Internal function. Return the previous entry * @mas: The maple state. * @limit: The lower limit to check. - * Returns the previous entry or NULL. * - * Internal only. + * Return: the previous entry or %NULL. */ static inline void *_mas_prev(struct ma_state *mas, unsigned long limit) { void *entry; - unsigned long index = mas->index; retry: @@ -4006,11 +4426,13 @@ retry: * mas_prev() - Get the previous entry * @mas: The maple state * @min: The minimum value to check. - * Returns the previous value or NULL. * + * Must hold rcu_read_lock or the write lock. * Will reset mas to MAS_START if the node is MAS_NONE. Will stop on not * searchable nodes. If mas->node is MAS_START, it will first look up the * index, then get the previous entry. + * + * Return: the previous value or %NULL. */ void *mas_prev(struct ma_state *mas, unsigned long min) { @@ -4041,6 +4463,15 @@ void *mas_prev(struct ma_state *mas, unsigned long min) } EXPORT_SYMBOL_GPL(mas_prev); +/* + * _mas_rev_awalk() - Internal function. Reverse allocation walk. Find the + * highest gap address of a given size in a given node and descend. + * @mas: The maple state + * @size: The needed size. + * + * Return: True if found in a leaf, false otherwise. + * + */ static bool _mas_rev_awalk(struct ma_state *mas, unsigned long size) { enum maple_type type = mte_node_type(mas->node); @@ -4196,10 +4627,11 @@ done: /* mas_walk() - Search for @mas->index in the tree. * @mas - the maple state. - * Returns the entry at the location or NULL. * * mas->index and mas->last will be set to the range if there is a value. If * mas->node is MAS_NONE, reset to MAS_START. + * + * Return: the entry at the location or %NULL. */ void *mas_walk(struct ma_state *mas) { @@ -4294,6 +4726,15 @@ static inline bool mas_rewind_node(struct ma_state *mas) return true; } +/* + * mas_rev_awalk() - Reverse allocation walk for a given size. + * @mas: The maple state + * @size: The size of the gap + * + * Reverse search from @mas->last to @mas->index for a gap of @size. + * + * Return: true if found, false otherwise. + */ static bool mas_rev_awalk(struct ma_state *mas, unsigned long size) { struct maple_enode *last = NULL; @@ -4317,7 +4758,12 @@ static bool mas_rev_awalk(struct ma_state *mas, unsigned long size) return true; } -/* Skip this slot in the parent. */ +/* + * mas_skip_node() - Internal function. Skip over a node. + * @mas: The maple state. + * + * Return: true if there is another node, false otherwise. + */ static inline bool mas_skip_node(struct ma_state *mas) { unsigned char slot, slot_count; @@ -4352,6 +4798,14 @@ static inline bool mas_skip_node(struct ma_state *mas) return true; } +/* + * mas_awalk() - Allocation walk. Search from low address to high, for a gap of + * @size + * @mas: The maple state + * @size: The size of the gap required + * + * Search between @mas->index and @mas->last for a gap of @size. + */ static inline void mas_awalk(struct ma_state *mas, unsigned long size) { struct maple_enode *last = NULL; @@ -4377,8 +4831,16 @@ static inline void mas_awalk(struct ma_state *mas, unsigned long size) } } -static int mas_fill_gap(struct ma_state *mas, void *entry, unsigned char slot, - unsigned long size, unsigned long *index) +/* + * mas_fill_gap() - Fill a located gap with @entry. + * @mas: The maple state + * @entry: The value to store + * @slot: The offset into the node to store the @entry + * @size: The size of the entry + * @index: The start location + */ +static inline void mas_fill_gap(struct ma_state *mas, void *entry, + unsigned char slot, unsigned long size, unsigned long *index) { unsigned char pslot = mte_parent_slot(mas->node); struct maple_enode *mn = mas->node; @@ -4404,10 +4866,18 @@ static int mas_fill_gap(struct ma_state *mas, void *entry, unsigned char slot, mas->node = mn; mas->offset = slot; _mas_store(mas, entry, false); - return 0; } -static inline void _mas_sparse_area(struct ma_state *mas, unsigned long min, +/* + * mas_sparse_area() - Internal function. Return upper or lower limit when + * searching for a gap in an empty tree. + * @mas: The maple state + * @min: the minimum range + * @max: The maximum range + * @size: The size of the gap + * @fwd: Searching forward or back + */ +static inline void mas_sparse_area(struct ma_state *mas, unsigned long min, unsigned long max, unsigned long size, bool fwd) { unsigned long start = 0; @@ -4427,6 +4897,14 @@ static inline void _mas_sparse_area(struct ma_state *mas, unsigned long min, mas->index = max; } +/* + * mas_get_empty_area() - Get the lowest address within the range that is + * sufficient for the size requested. + * @mas: The maple state + * @min: The lowest value of the range + * @max: The highest value of the range + * @size: The size needed + */ int mas_get_empty_area(struct ma_state *mas, unsigned long min, unsigned long max, unsigned long size) { @@ -4437,7 +4915,7 @@ int mas_get_empty_area(struct ma_state *mas, unsigned long min, // Empty set. if (mas_is_none(mas) || mas_is_ptr(mas)) { - _mas_sparse_area(mas, min, max, size, true); + mas_sparse_area(mas, min, max, size, true); return 0; } @@ -4468,6 +4946,14 @@ int mas_get_empty_area(struct ma_state *mas, unsigned long min, return 0; } +/* + * mas_get_empty_area_rev() - Get the highest address within the range that is + * sufficient for the size requested. + * @mas: The maple state + * @min: The lowest value of the range + * @max: The highest value of the range + * @size: The size needed + */ int mas_get_empty_area_rev(struct ma_state *mas, unsigned long min, unsigned long max, unsigned long size) { @@ -4475,7 +4961,7 @@ int mas_get_empty_area_rev(struct ma_state *mas, unsigned long min, // Empty set. if (mas_is_none(mas) || mas_is_ptr(mas)) { - _mas_sparse_area(mas, min, max, size, false); + mas_sparse_area(mas, min, max, size, false); return 0; } @@ -4502,15 +4988,6 @@ int mas_get_empty_area_rev(struct ma_state *mas, unsigned long min, return 0; } -/* - * mas_alloc() - Allocate a range. - * - * Give a size, a minimum starting point (mas->index), a maximum (mas->last), - * and a size (size), find the lowest location in the min-max window in the - * tree which this allocation fits and set index to that value. - * - * Returns: 0 on success, -ENOMEM if allocation fails, -EBUSY otherwise. - */ static inline int mas_alloc(struct ma_state *mas, void *entry, unsigned long size, unsigned long *index) { @@ -4543,21 +5020,13 @@ static inline int mas_alloc(struct ma_state *mas, void *entry, if (mas->index < min) mas->index = min; - return mas_fill_gap(mas, entry, mas->offset, size, index); + mas_fill_gap(mas, entry, mas->offset, size, index); + return 0; no_gap: return -EBUSY; } -/* - * mas_rev_alloc() - Reverse allocate a range. - * - * Give a size, a minimum value (mas->index), a maximum starting point - * (mas->last), and a size (size), find the largest location in the min-max - * window in tree which this allocation fits and set index to that value. - * - * Returns: 0 on success, -EBUSY otherwise. - */ static inline int mas_rev_alloc(struct ma_state *mas, unsigned long min, unsigned long max, void *entry, unsigned long size, unsigned long *index) @@ -4574,21 +5043,25 @@ static inline int mas_rev_alloc(struct ma_state *mas, unsigned long min, if (mas->offset == MAPLE_NODE_SLOTS) goto no_gap; - return mas_fill_gap(mas, entry, mas->offset, size, index); + mas_fill_gap(mas, entry, mas->offset, size, index); + return 0; no_gap: return -EBUSY; } /* + * mas_range_load() - Load the entry at an index and get the range. + * @mas: The maple state + * @*range_min: Pointer to store the minimum range the entry is valid + * @*range_max: Pointer to store the maximum range the entry is valid * * Must hold rcu_read_lock or the write lock. - * - * Find where ms->index is located and return the entry. - * mas->node will point to the node containing the entry. - * + * Find where mas->index is located and return the entry. + * @mas->node will point to the node containing the entry. * range_min and range_max will be set accordingly. * + * Return: The entry at mas->index or %NULL */ static inline void *mas_range_load(struct ma_state *mas, unsigned long *range_min, unsigned long *range_max) @@ -4613,6 +5086,12 @@ retry: return entry; } +/* + * mas_load() - Load the value stored at @mas->index + * @mas: The maple state + * + * Must hold rcu_read_lock or the write lock. + */ void *mas_load(struct ma_state *mas) { unsigned long range_max, range_min; @@ -4621,8 +5100,14 @@ void *mas_load(struct ma_state *mas) } /* + * _mas_next() - Finds the next entry and sets @mas->index and @mas->last to the + * range. + * @mas: The maple state + * @limit: The maximum value to check. + * + * Internal function. * - * _mas_next() - Finds the next entry, sets index to the start of the range. + * Return: Point to the next entry or %NULL * */ static void *_mas_next(struct ma_state *mas, unsigned long limit) @@ -4653,10 +5138,14 @@ static void *_mas_next(struct ma_state *mas, unsigned long limit) * mas_find: If mas->node == MAS_START, find the first * non-NULL entry >= mas->index. * Otherwise, find the first non-NULL entry > mas->index + * @mas: The maple state + * @max: The maximum value to check. * + * Must hold rcu_read_lock or the write lock. * If an entry exists, last and index are updated accordingly. + * May set @mas->node to MAS_NONE. * - * returns entry or null and set mas->node to MAS_NONE. + * Return: The entry or %NULL. */ void *mas_find(struct ma_state *mas, unsigned long max) { @@ -4683,10 +5172,15 @@ retry: } EXPORT_SYMBOL_GPL(mas_find); -/* mt_find() - Search from start up until an entry is found. +/* + * _mt_find() - Search from start up until an entry is found. + * @mt: The maple tree + * @*index: Pointer which contains the start location of the search + * @max: The maximum value to check + * @start: If this is the first time being called or not. * - * Note: Does not return the zero entry. - * returns an entry. + * Internal function. Does not return the zero entry. Handles locking. + * Return: the entry or %NULL */ void *_mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max, bool start) @@ -4723,6 +5217,16 @@ void *_mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max, return entry; } +/* + * mt_find() - Search from the start up until an entry is found. + * @mt: The maple tree + * @*index: Pointer which contains the start location of the search + * @max: The maximum value to check + * + * Handles locking. + * + * Return: The entry at or after the @*index or %NULL + */ void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max) { return _mt_find(mt, index, max, true); @@ -4730,9 +5234,14 @@ void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max) EXPORT_SYMBOL(mt_find); /* - * mas_next() - Get the next entry. Can return the zero entry. mas->node - * must be a valid node and not a special value. Unsafe for single entry - * trees. + * mas_next() - Get the next entry. + * @mas: The maple state + * @max: The maximum index to check. + * + * Must hold rcu_read_lock or the write lock. + * Can return the zero entry. + * + * Return: The next entry or %NULL */ void *mas_next(struct ma_state *mas, unsigned long max) { -- 2.50.1