From: Liam R. Howlett Date: Sat, 11 Jul 2020 00:48:44 +0000 (-0400) Subject: maple_tree: mas_combine_separate changes. X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=a8217e12d2ef780913145d0c07d0bd28d58efb4f;p=users%2Fjedix%2Flinux-maple.git maple_tree: mas_combine_separate changes. Fix the calculation of node splitting when needing 3 nodes. Change allocation count, spanning store may cause more allocations if rebalancing across nodes occurs. Remove free/destroy lists and use a linked list. Weight reduction of the function. Signed-off-by: Liam R. Howlett --- diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index fd2397cdeac5..12f8c09ff031 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -36,6 +36,7 @@ #define MAPLE_SPARSE9_SLOTS 13 /* 127 bytes */ #define MAPLE_SPARSE6_SLOTS 14 /* 128 bytes */ +#define MAPLE_TOPIARY_SLOTS MAPLE_NODE_SLOTS - 1 #define MAPLE_NODE_COALESCE 2 /* Limit on coalesce count */ #else @@ -142,6 +143,12 @@ struct maple_sparse_6 { unsigned long pivot; /* Use a bitmap for pivots */ }; +struct maple_topiary { + struct maple_pnode *parent; + void __rcu *slot[MAPLE_TOPIARY_SLOTS]; + struct maple_enode *next; /* Overlaps the pivot */ +}; + struct maple_node { union { struct { @@ -207,6 +214,11 @@ struct maple_tree { #define mtree_lock(mt) spin_lock((&mt->ma_lock)) #define mtree_unlock(mt) spin_unlock((&mt->ma_lock)) +struct ma_topiary { + struct maple_enode *head; + struct maple_enode *tail; + struct maple_tree *mtree; +}; void mtree_init(struct maple_tree *mt, unsigned int ma_flags); void *mtree_load(struct maple_tree *mt, unsigned long index); @@ -281,6 +293,13 @@ struct ma_state { .max = ULONG_MAX, \ } +#define MA_TOPIARY(name, tree) \ + struct ma_topiary name = { \ + .head = NULL, \ + .tail = NULL, \ + .mtree = tree, \ + } + void *mas_walk(struct ma_state *mas); void *mas_store(struct ma_state *mas, void *entry); void *mas_find(struct ma_state *mas, unsigned long max); diff --git a/lib/maple_tree.c b/lib/maple_tree.c index b3a6f05d2124..5490d1e0acf4 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -200,14 +200,18 @@ static inline struct maple_node *mte_to_node(const struct maple_enode *entry) { return (struct maple_node *)((unsigned long)entry & ~127); } +static inline struct maple_topiary *mte_to_mat(const struct maple_enode *entry) +{ + return (struct maple_topiary *)((unsigned long)entry & ~127); +} static inline struct maple_node *mas_mn(const struct ma_state *mas) { return mte_to_node(mas->node); } -static inline void mas_set_node_dead(struct ma_state *mas) +static inline void mte_set_node_dead(struct maple_enode *mn) { - mas_mn(mas)->parent = ma_parent_ptr(mas_mn(mas)); + mte_to_node(mn)->parent = ma_parent_ptr(mte_to_node(mn)); } static void mte_free(struct maple_enode *enode) { @@ -612,16 +616,76 @@ static inline struct maple_enode *mas_get_rcu_slot(const struct ma_state *mas, { return mte_get_rcu_slot(mas->node, slot, mas->tree); } -static inline struct maple_enode *mas_get_rcu_sanitized( - struct ma_state *mas, unsigned char slot) +/* Private + * mte_destroy_walk: Free the sub-tree from @mn and below. + */ +void mte_destroy_walk(struct maple_enode *mn, struct maple_tree *mtree) { - void *entry = mte_get_rcu_slot(mas->node, slot, mas->tree); + struct maple_enode *node; + unsigned int type = mte_node_type(mn); + unsigned char slot_cnt = mt_slot_count(mn); + int i; - if (xa_is_deleted(entry)) - return NULL; + switch (type) { + case maple_range_16: + case maple_range_32: + case maple_range_64: + case maple_arange_64: + for (i = 0; i < slot_cnt; i++) { + node = mte_get_rcu_slot(mn, i, mtree); + if (node) + mte_destroy_walk(node, mtree); + } + break; + default: + break; + } + mte_free(mn); - return entry; } +/** Private + * mt_dead_list_add() - Add a @dead_node to the @tail of a list of dead nodes. + * @tail may be modified to point to @dead_node if @tail is full. + * + * @mtree - the tree which contains the node to be marked ad dead. + * @tail - the tail of the dead list + * @dead_node - the node to be marked as dead and added to the tail of the list + * (or to become the tail node of the list) + */ +static inline void mat_add(struct ma_topiary *mat, + struct maple_enode *dead_enode) +{ + mte_set_node_dead(dead_enode); + mte_to_mat(dead_enode)->next = NULL; + if (!mat->tail) { + mat->tail = mat->head = dead_enode; + return; + } + + //* Set the next entry. + mte_to_mat(mat->tail)->next = dead_enode; + mat->tail = dead_enode; +} +/** Private + * mt_dead_list_free() - Free all nodes in a dead list. + * + * @mtree - the tree which contains the nodes that will be freed. + * @head - the start of the list of dead nodes to be freed. + */ +static inline void mat_free(struct ma_topiary *mat, bool recursive) +{ + struct maple_enode *next; + + while (mat->head) { + next = mte_to_mat(mat->head)->next; + if (recursive) + mte_destroy_walk(mat->head, mat->mtree); + else + mte_free(mat->head); + mat->head = next; + } +} + static inline void ma_set_rcu_slot(struct maple_node *mn, unsigned char slot, enum maple_type type, void *val) @@ -803,8 +867,6 @@ static inline void mas_update_limits(struct ma_state *mas, unsigned char slot, */ bool mas_retry(struct ma_state *mas, const void *entry) { - if (xa_is_skip(entry)) - return true; if (xa_is_deleted(entry)) return true; if (xa_is_zero(entry)) @@ -1357,33 +1419,6 @@ static inline unsigned long mas_first_entry(struct ma_state *mas, } -/* Private - * mte_destroy_walk: Free the sub-tree from @mn and below. - */ -void mte_destroy_walk(struct maple_enode *mn, struct maple_tree *mtree) -{ - struct maple_enode *node; - unsigned int type = mte_node_type(mn); - unsigned char slot_cnt = mt_slot_count(mn); - int i; - - switch (type) { - case maple_range_16: - case maple_range_32: - case maple_range_64: - case maple_arange_64: - for (i = 0; i < slot_cnt; i++) { - node = mte_get_rcu_slot(mn, i, mtree); - if (node) - mte_destroy_walk(node, mtree); - } - break; - default: - break; - } - mte_free(mn); - -} static inline void mas_adopt_children(struct ma_state *mas, struct maple_enode *parent) @@ -1476,6 +1511,9 @@ struct maple_big_node { struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS]; unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1]; unsigned long gap[MAPLE_BIG_NODE_SLOTS]; + unsigned long min; + unsigned char b_end; + enum maple_type type; }; struct maple_clean_list { @@ -1538,38 +1576,56 @@ static inline void mab_shift_right(struct maple_big_node *b_node, } static inline bool mab_middle_node(struct maple_big_node *b_node, int size, - unsigned char slot_cnt) + int split, unsigned char slot_cnt) { - int split = (size - 1) / 2; // Assume equal split. + if (size >= 2 * slot_cnt) + return true; - if (!b_node->slot[split] && (size >= (2 * slot_cnt - 1))) + if (!b_node->slot[split] && (size >= 2 * slot_cnt - 1)) return true; + return false; } -static inline int mab_calc_split(struct maple_big_node *b_node, int size, - unsigned char slot_cnt, unsigned long min, - enum maple_type type) + +static inline int mab_no_null_split(struct maple_big_node *b_node, + unsigned char split, unsigned char slot_cnt) { - int split = (size - 1) / 2; // Assume equal split. + if (!b_node->slot[split]) { + if (split < slot_cnt - 1) + split++; + else + split--; + } + return split; +} +static inline int mab_calc_split(struct maple_big_node *b_node, + unsigned char *mid_split) +{ + int split = b_node->b_end / 2; // Assume equal split. + unsigned char slot_cnt = mt_slots[b_node->type]; - if (mab_middle_node(b_node, size, slot_cnt)) { - split = (size + 1) / 3; + if (mab_middle_node(b_node, b_node->b_end, split, slot_cnt)) { + split = b_node->b_end / 3; + *mid_split = split * 2; } else { + *mid_split = 0; /* Avoid having a range less than the slot count unless it * causes one node to be deficient. */ - while (((b_node->pivot[split] - min) < slot_cnt - 1) && + /* FIXME: Linear allocations will cause either a wasted slot + * as is, or will cause a deficient node with not enough entries + */ + while (((b_node->pivot[split] - b_node->min) < slot_cnt - 1) && (split < slot_cnt) && - (size - split > mt_min_slots[type])) + (b_node->b_end - split > mt_min_slots[b_node->type])) split++; } /* Avoid ending a node on a NULL entry */ - if (!b_node->slot[split]) { - if (split < slot_cnt - 1) - split++; - else - split--; - } + split = mab_no_null_split(b_node, split, slot_cnt); + if (!(*mid_split)) + return split; + + *mid_split = mab_no_null_split(b_node, *mid_split, slot_cnt); return split; } @@ -1609,10 +1665,10 @@ static inline int mab_mas_cp(struct maple_big_node *b_node, struct ma_state *mas, unsigned char mas_start) { int i, j; + for (i = mab_start, j = mas_start; i <= mab_end; i++, j++) { - if(j && !b_node->pivot[i]) { + if(j && !b_node->pivot[i]) break; - } mas->max = b_node->pivot[i]; mte_set_rcu_slot(mas->node, j, b_node->slot[i]); @@ -1765,16 +1821,14 @@ static inline bool mas_next_sibling(struct ma_state *mas) /* mas_consume() - Add the portions of the tree which will be replaced by the * operation to the removal list; either to be @free or @discard (destroy walk). */ -static inline void -mas_consume(struct ma_state *l_mas, struct ma_state *r_mas, - struct maple_enode **free, unsigned char *f, - struct maple_enode **discard, unsigned char *d) +static inline void mas_consume(struct ma_state *l_mas, + struct ma_state *r_mas, + struct ma_topiary *destroy) { unsigned char l_slot, r_slot, slot, end; unsigned long l_min, range_min, range_max; // The left node is consumed, so add to the free list. - free[(*f)++] = l_mas->node; l_min = l_mas->index; l_mas->index = l_mas->last; mas_node_walk(l_mas, mte_node_type(l_mas->node), &range_min, &range_max); @@ -1789,23 +1843,123 @@ mas_consume(struct ma_state *l_mas, struct ma_state *r_mas, * use. */ for (slot = l_slot + 1; slot < r_slot; slot++) - discard[(*d)++] = mas_get_rcu_slot(l_mas, slot); + mat_add(destroy, mas_get_rcu_slot(l_mas, slot)); return; } /* r_mas is different and consumed. */ - free[(*f)++] = r_mas->node; if (mte_is_leaf(r_mas->node)) return; /* Now free l_slot + 1 -> end and 0 -> r_slot - 1 */ end = mas_data_end(l_mas); for (slot = l_slot + 1; slot <= end; slot++) - discard[(*d)++] = mas_get_rcu_slot(l_mas, slot); + mat_add(destroy, mas_get_rcu_slot(l_mas, slot)); for (slot = 0; slot < r_slot; slot++) - discard[(*d)++] = mas_get_rcu_slot(r_mas, slot); + mat_add(destroy, mas_get_rcu_slot(r_mas, slot)); } +static inline void +mas_ascend_free(struct ma_state *l_mas, struct ma_state *r_mas, + struct ma_topiary *free) +{ + struct maple_enode *left = l_mas->node; + struct maple_enode *right = r_mas->node; + mas_ascend(l_mas); + mas_ascend(r_mas); + mat_add(free, left); + if (left != right) + mat_add(free, right); +} +/* Private + * mas_mab_to_node() - Set up right and middle nodes + * + * @mas - the maple state that contains the allocations. + * @b_node - the node which contains the data. + * @left - The pointer which will have the left node + * @right - The pointer which may have the right node + * @middle - the pointer which may have the middle node (rare) + * @mid_split - the split location for the middle node + * + * returns the split of left. + */ +static inline unsigned char mas_mab_to_node(struct ma_state *mas, + struct maple_big_node *b_node, + struct maple_enode **left, + struct maple_enode **right, + struct maple_enode **middle, + unsigned char *mid_split) +{ + unsigned char split = 0; + unsigned char slot_cnt = mt_slots[b_node->type]; + + *left = mt_mk_node(ma_mnode_ptr(mas_next_alloc(mas)), b_node->type); + *right = NULL; + *middle = NULL; + *mid_split = 0; + + if (b_node->b_end < slot_cnt) { + split = b_node->b_end; + } else { + split = mab_calc_split(b_node, mid_split); + *right = mt_mk_node(ma_mnode_ptr(mas_next_alloc(mas)), + b_node->type); + } + + if (*mid_split) + *middle = mt_mk_node(ma_mnode_ptr(mas_next_alloc(mas)), + b_node->type); + + return split; + +} + +/* Private + * mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end + * pointer. + * @b_node - the big node to add the entry + * @mas - the maple state to get the pivot (mas->max) + * @entry - the entry to add, if NULL nothing happens. + */ +static inline void mab_set_b_end(struct maple_big_node *b_node, + struct ma_state *mas, + void *entry) +{ + if (!entry) + return; + + b_node->slot[b_node->b_end] = entry; + if (mt_is_alloc(mas->tree)) + b_node->gap[b_node->b_end] = mas_find_gap(mas); + b_node->pivot[b_node->b_end++] = mas->max; +} + +/* Private + * mas_combine_set_parent() - combine_separate helper function. Sets the parent + * of @mas->node to either @left or @right, depending on @slot and @split + * + * @mas - the maple state with the node that needs a parent + * @left - possible parent 1 + * @right - possible parent 2 + * @slot - the slot the mas->node was placed + * @split - the split location between @left and @right + */ +static inline void mas_separate_set_parent(struct ma_state *mas, + struct maple_enode *left, + struct maple_enode *right, + unsigned char *slot, + unsigned char split) +{ + if (!mas->node) + return; + + if ((*slot) <= split) + mte_set_parent(mas->node, left, *slot); + else + mte_set_parent(mas->node, right, (*slot) - split); + + (*slot)++; +} /* Private * * mas_combine_separate() - Follow the tree upwards from @l_mas and @r_mas for @@ -1818,77 +1972,44 @@ mas_consume(struct ma_state *l_mas, struct ma_state *r_mas, * orig_l_mas->index and orig_l_mas->last to account of what has been copied * into the new sub-tree. The update of orig_l_mas->last is used in mas_consume * to find the slots that will need to be either freed or destroyed. + * orig_l_mas->depth keeps track of the height of the new sub-tree in case the + * sub-tree becomes the full tree. * - * Returns the @count left. + * Returns the number of elements in b_node during the last loop. */ static inline int mas_combine_separate(struct ma_state *mas, struct ma_state *orig_l_mas, struct ma_state *orig_r_mas, struct maple_big_node *b_node, - unsigned char b_end, unsigned char count) { - unsigned char slot_cnt, split, r_end; - unsigned char child = 0, l_slot, r_slot = 0, i = 0, d = 0; - struct maple_enode *left, *middle, *right; + unsigned char split, r_end, mid_split; + unsigned char slot = 0, l_slot, r_slot = 0; + struct maple_enode *left = NULL, *middle = NULL, *right = NULL; unsigned long range_min, range_max; - struct maple_enode *free[100]; - struct maple_enode *destroy[100]; MA_STATE(l_mas, mas->tree, mas->index, mas->index); MA_STATE(r_mas, mas->tree, mas->index, mas->index); MA_STATE(m_mas, mas->tree, mas->index, mas->index); + MA_TOPIARY(free, mas->tree); + MA_TOPIARY(destroy, mas->tree); + + /* MA_START doesn't work here */ l_mas.node = r_mas.node = m_mas.node = NULL; - left = right = middle = NULL; - mas_consume(orig_l_mas, orig_r_mas, free, &i, destroy, &d); + mas_consume(orig_l_mas, orig_r_mas, &destroy); while(count--) { - b_end--; - slot_cnt = mt_slot_count(orig_l_mas->node); - right = NULL; - middle = NULL; - left = mt_mk_node(ma_mnode_ptr(mas_next_alloc(mas)), - mte_node_type(orig_l_mas->node)); - - /// - if (b_end < slot_cnt) { - split = b_end; - } else { - split = mab_calc_split(b_node, b_end, slot_cnt, - orig_l_mas->min, - mte_node_type(orig_l_mas->node)); - right = mt_mk_node(ma_mnode_ptr(mas_next_alloc(mas)), - mte_node_type(orig_l_mas->node)); - } - if (mab_middle_node(b_node, b_end, slot_cnt)) - middle = mt_mk_node(ma_mnode_ptr(mas_next_alloc(mas)), - mte_node_type(orig_l_mas->node)); + b_node->b_end--; + b_node->type = mte_node_type(orig_l_mas->node); + b_node->min = orig_l_mas->min; + split = mas_mab_to_node(mas, b_node, &left, &right, &middle, + &mid_split); /* Set parents from previous run */ - if (l_mas.node) { - if (child <= split) - mte_set_parent(l_mas.node, left, child); - else - mte_set_parent(l_mas.node, right, - child - split); - } - if (m_mas.node) { - child++; - if (child <= split) - mte_set_parent(m_mas.node, left, child); - else - mte_set_parent(m_mas.node, right, - child - split); - } - if (r_mas.node) { - child++; - if (child <= split) - mte_set_parent(r_mas.node, left, child); - else - mte_set_parent(r_mas.node, right, - child - split); - } + mas_separate_set_parent(&l_mas, left, right, &slot, split); + mas_separate_set_parent(&m_mas, left, right, &slot, split); + mas_separate_set_parent(&r_mas, left, right, &slot, split); /* Copy data from b_node to new nodes */ l_mas.node = left; @@ -1902,15 +2023,15 @@ static inline int mas_combine_separate(struct ma_state *mas, if (middle) { - mab_mas_cp(b_node, 1 + split, split * 2, &m_mas, 0); + mab_mas_cp(b_node, 1 + split, mid_split, &m_mas, 0); m_mas.min = b_node->pivot[split] + 1; - split *= 2; - m_mas.max = b_node->pivot[split]; + m_mas.max = b_node->pivot[mid_split]; + split = mid_split; } if (right) { - mab_mas_cp(b_node, 1 + split, b_end, &r_mas, 0); + mab_mas_cp(b_node, 1 + split, b_node->b_end, &r_mas, 0); r_mas.min = b_node->pivot[split] + 1; - r_mas.max = b_node->pivot[b_end]; + r_mas.max = b_node->pivot[b_node->b_end]; } /* Copy data from next level in the tree to b_node from next iteration */ @@ -1921,16 +2042,16 @@ static inline int mas_combine_separate(struct ma_state *mas, mas_mn(&l_mas)->parent = ma_parent_ptr( ((unsigned long)mas->tree | MA_ROOT_PARENT)); mas->depth = orig_l_mas->depth; - b_end = 0; + b_node->b_end = 0; if (mte_is_root(orig_l_mas->node)) { if ((orig_l_mas->node != mas->node) && (l_mas.depth > mas->tree->ma_height)) { - free[i++] = mas->node; + mat_add(&free, mas->node); } } else { do { - mas_ascend(orig_l_mas); - mas_ascend(orig_r_mas); + mas_ascend_free(orig_l_mas, orig_r_mas, + &free); mas_set_slot(orig_l_mas, 0); mas_set_slot(orig_r_mas, 0); mas_node_walk(orig_l_mas, @@ -1940,99 +2061,89 @@ static inline int mas_combine_separate(struct ma_state *mas, mte_node_type(orig_r_mas->node), &range_min, &range_max); mas_consume(orig_l_mas, orig_r_mas, - free, &i, destroy, &d); + &destroy); } while (!mte_is_root(orig_l_mas->node)); } + mat_add(&free, orig_l_mas->node); mas_dup_state(orig_l_mas, &l_mas); goto done; } - - mas_ascend(orig_l_mas); - mas_ascend(orig_r_mas); + mas_ascend_free(orig_l_mas, orig_r_mas, &free); + /* Set up the right side of things */ r_end = mas_data_end(orig_r_mas); - mas_set_slot(orig_r_mas, 0); orig_r_mas->index = r_mas.max; + /* last should be larger than or equal to index */ if (orig_r_mas->last < orig_r_mas->index) orig_r_mas->last = orig_r_mas->index; + /* The node may not contain the value so set slot to ensure all + * of the nodes contents are freed or destroyed. + */ if (!mas_node_walk(orig_r_mas, mte_node_type(orig_r_mas->node), &range_min, &range_max)) { - // The node doesn't contain the value so set - // slot to ensure all of the nodes contents are - // freed or destroyed. mas_set_slot(orig_r_mas, r_end + 1); } r_slot = mas_get_slot(orig_r_mas); - + /* Set up the left side of things */ mas_set_slot(orig_l_mas, 0); orig_l_mas->index = l_mas.min; mas_node_walk(orig_l_mas, mte_node_type(orig_l_mas->node), &range_min, &range_max); l_slot = mas_get_slot(orig_l_mas); - b_end = 0; + b_node->b_end = 0; if (l_slot) - b_end = mas_mab_cp(orig_l_mas, 0, l_slot - 1, b_node, 0); - - child = b_end; - // Track dead nodes here. - b_node->slot[b_end] = l_mas.node; - if(mt_is_alloc(mas->tree)) - b_node->gap[b_end] = mas_find_gap(&l_mas); - - b_node->pivot[b_end++] = l_mas.max; - - if (middle) { - b_node->slot[b_end] = middle; - if(mt_is_alloc(mas->tree)) - b_node->gap[b_end] = mas_find_gap(&m_mas); - - b_node->pivot[b_end++] = m_mas.max; - } + b_node->b_end = mas_mab_cp(orig_l_mas, 0, l_slot - 1, + b_node, 0); - if (right) { - b_node->slot[b_end] = right; - if(mt_is_alloc(mas->tree)) - b_node->gap[b_end] = mas_find_gap(&r_mas); - - b_node->pivot[b_end++] = r_mas.max; - } + slot = b_node->b_end; + mab_set_b_end(b_node, &l_mas, left); + mab_set_b_end(b_node, &m_mas, middle); + mab_set_b_end(b_node, &r_mas, right); // Copy anything necessary out of the right node. - if (b_node->pivot[b_end - 1] < orig_r_mas->max) { - b_end = mas_mab_cp(orig_r_mas, r_slot + 1, r_end, - b_node, b_end); + if (b_node->pivot[b_node->b_end - 1] < orig_r_mas->max) { + b_node->b_end = mas_mab_cp(orig_r_mas, r_slot + 1, + r_end, b_node, b_node->b_end); orig_r_mas->last = orig_r_mas->max; } - mas_consume(orig_l_mas, orig_r_mas, free, &i, destroy, &d); + mas_consume(orig_l_mas, orig_r_mas, &destroy); orig_l_mas->last = orig_l_mas->max; // Attempt to balance from this parent - if (b_end - 1 < mt_min_slot_cnt(orig_l_mas->node)) { + if (b_node->b_end - 1 < mt_min_slot_cnt(orig_l_mas->node)) { unsigned char end; + struct maple_enode *left = orig_l_mas->node; + struct maple_enode *right = orig_r_mas->node; if (mas_next_sibling(orig_r_mas)) { end = mas_data_end(orig_r_mas); - b_end = mas_mab_cp(orig_r_mas, 0, end, b_node, - b_end); - free[i++] = orig_r_mas->node; + b_node->b_end = mas_mab_cp(orig_r_mas, 0, end, + b_node, b_node->b_end); + mat_add(&free, right); + if (right == left) + orig_l_mas->node = orig_r_mas->node; + //free[i++] = orig_r_mas->node; orig_r_mas->last = orig_r_mas->max; if (!count) count++; } else if (mas_prev_sibling(orig_l_mas)) { end = mas_data_end(orig_l_mas); // shift b_node by prev size - mab_shift_right(b_node, b_end - 1, end + 1, - (mt_is_alloc(mas->tree) ? true : false)); + mab_shift_right(b_node, b_node->b_end - 1, + end + 1, + (mt_is_alloc(mas->tree) ? true : false)); // copy in prev. mas_mab_cp(orig_l_mas, 0, end, b_node, 0); - free[i++] = orig_l_mas->node; + mat_add(&free, left); + if (right == left) + orig_r_mas->node = orig_l_mas->node; l_mas.min = orig_l_mas->min; orig_l_mas->index = orig_l_mas->min; - b_end += end + 1; - child += end + 1; + b_node->b_end += end + 1; + slot += end + 1; if (!count) count++; } @@ -2040,8 +2151,11 @@ static inline int mas_combine_separate(struct ma_state *mas, // Ensure there is enough data for the next iteration. - if (b_end - 1 < mt_min_slot_cnt(orig_l_mas->node)) { + if (b_node->b_end - 1 < mt_min_slot_cnt(orig_l_mas->node)) { unsigned char end; + struct maple_enode *left = orig_l_mas->node; + struct maple_enode *right = orig_r_mas->node; + mas_set_slot(orig_r_mas, mte_parent_slot(orig_r_mas->node)); mas_next_node(orig_r_mas, ULONG_MAX); @@ -2049,10 +2163,13 @@ static inline int mas_combine_separate(struct ma_state *mas, // Add r_mas.node to clean_list. // r_mas is now at the next node.. end = mas_data_end(orig_r_mas); - b_end = mas_mab_cp(orig_r_mas, 0, end, - b_node, b_end); + b_node->b_end = mas_mab_cp(orig_r_mas, 0, end, + b_node, b_node->b_end); orig_r_mas->last = orig_r_mas->max; - free[i++] = orig_r_mas->node; + mat_add(&free, right); + if (right == left) + orig_l_mas->node = orig_r_mas->node; + //free[i++] = orig_r_mas->node; } else { // Put left into right. mas_dup_state(orig_r_mas, orig_l_mas); @@ -2064,24 +2181,26 @@ static inline int mas_combine_separate(struct ma_state *mas, // This is going to be a new root of // only what is in b_node mas_dup_state(orig_l_mas, orig_r_mas); - b_end--; + b_node->b_end--; break; } - free[i++] = orig_l_mas->node; + mat_add(&free, left); + if (right == left) + orig_r_mas->node = orig_l_mas->node; + //free[i++] = orig_l_mas->node; mas_set_slot(orig_l_mas, 0); orig_l_mas->index = orig_l_mas->min; l_slot = 0; end = mas_data_end(orig_l_mas); // shift b_node by prev size - mab_shift_right(b_node, b_end - 1, end + 1, + mab_shift_right(b_node, b_node->b_end - 1, end + 1, (mt_is_alloc(mas->tree) ? true : false)); // copy in prev. mas_mab_cp(orig_l_mas, 0, end, b_node, 0); l_mas.min = orig_l_mas->min; - free[i++] = orig_l_mas->node; - child += end + 1; - b_end += end + 1; + slot += end + 1; + b_node->b_end += end + 1; } if (!count) count++; @@ -2092,71 +2211,60 @@ static inline int mas_combine_separate(struct ma_state *mas, mte_node_type(orig_l_mas->node)); orig_l_mas->depth++; - //mas_consume(orig_l_mas, orig_r_mas, free, &i, destroy, &d); - mab_mas_cp(b_node, 0, b_end, &l_mas, 0); - mte_set_parent(left, l_mas.node, child); + mab_mas_cp(b_node, 0, b_node->b_end, &l_mas, 0); + mte_set_parent(left, l_mas.node, slot); if (middle) - mte_set_parent(middle, l_mas.node, ++child); + mte_set_parent(middle, l_mas.node, ++slot); if (right) - mte_set_parent(right, l_mas.node, ++child); + mte_set_parent(right, l_mas.node, ++slot); - if (!b_end) { - mas_mn(&l_mas)->parent = ma_parent_ptr( - ((unsigned long)mas->tree | MA_ROOT_PARENT)); + if (!b_node->b_end) { + mas_mn(&l_mas)->parent = + ma_parent_ptr(((unsigned long)mas->tree | MA_ROOT_PARENT)); } else { mas_mn(&l_mas)->parent = mas_mn(orig_l_mas)->parent; } + mat_add(&free, orig_l_mas->node); mas_dup_state(orig_l_mas, &l_mas); mas->depth = orig_l_mas->depth; -done: +done: + mte_set_node_dead(mas->node); // Set up mas for insertion. - mas_set_node_dead(mas); mas_dup_state(mas, orig_l_mas); smp_wmb(); // Insert new sub-tree _mas_replace(mas, false, false, false); - - - if (!mte_is_leaf(mas->node)) mas_descend_adopt(mas); - do { - mte_free(free[--i]); - } while (i); - - if (d) { - do { - mte_destroy_walk(destroy[--d], mas->tree); - } while (d); - } + mat_free(&free, false); + mat_free(&destroy, true); if (mte_is_leaf(mas->node)) - return b_end; + return b_node->b_end; if (mt_is_alloc(mas->tree)) mas_update_gap(mas, false); - return b_end; + return b_node->b_end; } static inline int mas_rebalance(struct ma_state *mas, - struct maple_big_node *b_node, - unsigned char new_end) + struct maple_big_node *b_node) { - char empty = mas->full_cnt * -1; - unsigned char b_end = 0; + char empty_cnt = mas->full_cnt * -1; - mas_node_cnt(mas, 1 + empty * 2); + MA_STATE(l_mas, mas->tree, mas->index, mas->last); + MA_STATE(r_mas, mas->tree, mas->index, mas->last); + + mas_node_cnt(mas, 1 + empty_cnt * 2); if (mas_is_err(mas)) return 0; - MA_STATE(l_mas, mas->tree, mas->index, mas->last); - MA_STATE(r_mas, mas->tree, mas->index, mas->last); mas_dup_state(&l_mas, mas); mas_dup_state(&r_mas, mas); @@ -2164,22 +2272,22 @@ static inline int mas_rebalance(struct ma_state *mas, if (mas_next_sibling(&r_mas)) { - b_end = mas_mab_cp(&r_mas, 0, mas_data_end(&r_mas), b_node, - new_end + 1); + b_node->b_end = mas_mab_cp(&r_mas, 0, mas_data_end(&r_mas), + b_node, b_node->b_end + 1); r_mas.last = r_mas.index = r_mas.max; } else { unsigned char shift; mas_prev_sibling(&l_mas); shift = mas_data_end(&l_mas) + 1; - mab_shift_right(b_node, new_end, shift, + mab_shift_right(b_node, b_node->b_end, shift, (mt_is_alloc(mas->tree) ? true : false)); - b_end = mas_mab_cp(&l_mas, 0, mas_data_end(&l_mas), b_node, 0); - b_end += new_end + 1; + mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0); + b_node->b_end += shift + 1; l_mas.index = l_mas.last = l_mas.min; } - return mas_combine_separate(mas, &l_mas, &r_mas, b_node, b_end, empty); + return mas_combine_separate(mas, &l_mas, &r_mas, b_node, empty_cnt); } static inline int mas_split(struct ma_state *mas, @@ -2190,7 +2298,7 @@ static inline int mas_split(struct ma_state *mas, struct maple_enode *ancestor = MAS_NONE; unsigned char split = 0; int j, i = 0, height = 0; - struct maple_enode *list[100]; + struct maple_enode *list[15]; // Enough for an 8 level tree. MA_STATE(l_mas, mas->tree, mas->index, mas->last); MA_STATE(r_mas, mas->tree, mas->index, mas->last); @@ -2218,6 +2326,7 @@ static inline int mas_split(struct ma_state *mas, while (height++ <= mas->full_cnt) { struct maple_node *l, *r; enum maple_type type = mte_node_type(mas->node); + unsigned char mid_split; unsigned char slot_cnt = mt_slot_count(mas->node); /* should be full. */ bool cp = true; @@ -2258,6 +2367,9 @@ static inline int mas_split(struct ma_state *mas, continue; } + b_node->b_end = new_end; + b_node->min = mas->min; + b_node->type = type; l = ma_mnode_ptr(mas_next_alloc(mas)); r = ma_mnode_ptr(mas_next_alloc(mas)); @@ -2266,8 +2378,7 @@ static inline int mas_split(struct ma_state *mas, l_mas.node = mt_mk_node(l, type); r_mas.node = mt_mk_node(r, type); if (mte_is_leaf(mas->node)) { - split = mab_calc_split(b_node, new_end, slot_cnt, - mas->min, type); + split = mab_calc_split(b_node, &mid_split); if (split < slot_cnt) j = mab_mas_cp(b_node, 0, split, &l_mas, 0); else @@ -2354,7 +2465,7 @@ static inline int mas_split(struct ma_state *mas, mas->node = ancestor; BUG_ON(mas_is_none(mas)); // Set the original node as dead - mte_to_node(list[0])->parent = ma_parent_ptr(mte_to_node(list[0])); + mte_set_node_dead(list[0]); smp_wmb(); // Insert the new data in the tree @@ -2372,17 +2483,17 @@ static inline int mas_split(struct ma_state *mas, } static inline int mas_commit_b_node(struct ma_state *mas, - struct maple_big_node *b_node, - unsigned char end) + struct maple_big_node *b_node) { struct maple_enode *new_node; - if ((end < mt_min_slot_cnt(mas->node)) && !mte_is_root(mas->node) && - (mas->tree->ma_height > 1) ) { - return mas_rebalance(mas, b_node, end); - } - else if (end >= mt_slot_count(mas->node)) - return mas_split(mas, b_node, end); + if ((b_node->b_end < mt_min_slot_cnt(mas->node)) && + (!mte_is_root(mas->node)) && + (mas->tree->ma_height > 1) ) + return mas_rebalance(mas, b_node); + + if (b_node->b_end >= mt_slot_count(mas->node)) + return mas_split(mas, b_node, b_node->b_end); mas_node_cnt(mas, 1); if (mas_is_err(mas)) @@ -2392,7 +2503,7 @@ static inline int mas_commit_b_node(struct ma_state *mas, mte_to_node(new_node)->parent = mas_mn(mas)->parent; mas->node = new_node; - mab_mas_cp(b_node, 0, end, mas, 0); + mab_mas_cp(b_node, 0, b_node->b_end, mas, 0); _mas_replace(mas, true, false, true); if (mt_is_alloc(mas->tree)) mas_update_gap(mas, false); @@ -2738,7 +2849,6 @@ static inline bool __mas_walk(struct ma_state *mas, unsigned long *range_min, static inline int mas_spanning_store(struct ma_state *mas, void *entry) { unsigned long range_min, range_max; - unsigned char b_end = 0; // big_node end. unsigned char count = 0; struct maple_big_node b_node; int node_cnt = 0; @@ -2753,10 +2863,9 @@ static inline int mas_spanning_store(struct ma_state *mas, void *entry) else node_cnt = 1 + -1 * mas->full_cnt * 2; // For rebalance upwards. - /* one for parent, one for potential 3rd node at bottom, - * two for every level below the split location. - */ - node_cnt += 2 + (mas->tree->ma_height - mas->depth) * 2; + /* Node rebalancing may occur due to a store, so there may be two new + * entries per level plus a new root. */ + node_cnt += 1 + mas->tree->ma_height * 2; mas_node_cnt(mas, node_cnt); if (mas_is_err(mas)) return 0; @@ -2791,10 +2900,11 @@ static inline int mas_spanning_store(struct ma_state *mas, void *entry) // Copy l_mas and store the value in b_node. - b_end = mas_store_b_node(&l_mas, &b_node, entry); + b_node.b_end = mas_store_b_node(&l_mas, &b_node, entry); // Copy r_mas into b_node. - b_end = mas_mab_cp(&r_mas, mas_get_slot(&r_mas), mas_data_end(&r_mas), - &b_node, b_end + 1); + b_node.b_end = mas_mab_cp(&r_mas, mas_get_slot(&r_mas), + mas_data_end(&r_mas), &b_node, + b_node.b_end + 1); // Stop spanning searches by searching for just index. l_mas.index = l_mas.last = mas->index; // Calc the number of iterations of combining and splitting that will @@ -2803,16 +2913,13 @@ static inline int mas_spanning_store(struct ma_state *mas, void *entry) // Combine l_mas and r_mas and split them up evenly again. l_mas.depth = 0; - b_end = mas_combine_separate(mas, &l_mas, &r_mas, &b_node, b_end, - count); - - return b_end; + return mas_combine_separate(mas, &l_mas, &r_mas, &b_node, count); } static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite) { unsigned long r_max, r_min; - unsigned char end, new_end, slot; + unsigned char end, slot; unsigned char slot_cnt; void *content = NULL; struct maple_big_node b_node; @@ -2865,11 +2972,11 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite memset(&b_node, 0, sizeof(struct maple_big_node)); mas_set_slot(mas, slot); - new_end = mas_store_b_node(mas, &b_node, entry); + b_node.b_end = mas_store_b_node(mas, &b_node, entry); // Check if this is an append operation. end = mas_data_end(mas); - if ((new_end < slot_cnt) && ((slot > end) || !end)) { + if ((b_node.b_end < slot_cnt) && ((slot > end) || !end)) { // Appending if (r_min < mas->index) mte_set_pivot(mas->node, slot++, mas->index - 1); @@ -2880,12 +2987,12 @@ static inline void *_mas_store(struct ma_state *mas, void *entry, bool overwrite } // count the node as full if it has not already been counted. - if (new_end >= slot_cnt && end < slot_cnt) + if (b_node.b_end >= slot_cnt && end < slot_cnt) mas_cnt_full(mas); - else if (new_end < mt_min_slot_cnt(mas->node)) + else if (b_node.b_end < mt_min_slot_cnt(mas->node)) mas_cnt_empty(mas); - mas_commit_b_node(mas, &b_node, new_end); + mas_commit_b_node(mas, &b_node); if (mas_is_err(mas)) ret = 3; @@ -4816,6 +4923,7 @@ counted: void mas_validate_parent_slot(struct ma_state *mas) { struct maple_node *parent; + struct maple_enode *node; enum maple_type p_type = mas_parent_enum(mas, mas->node); unsigned char p_slot = mte_parent_slot(mas->node); int i; @@ -4829,17 +4937,16 @@ void mas_validate_parent_slot(struct ma_state *mas) // Check prev/next parent slot for duplicate node entry for (i = 0; i < mt_slots[p_type]; i++) { + node = ma_get_rcu_slot(parent, i, p_type, mas->tree); if (i == p_slot) { - MT_BUG_ON(mas->tree, - ma_get_rcu_slot(parent, i, p_type, mas->tree) != - mas->node); - } else if (ma_get_rcu_slot(parent, i, p_type, mas->tree) == - mas->node) { + if (node != mas->node) + pr_err("parent %p[%u] does not have %p\n", + parent, i, mas_mn(mas)); + MT_BUG_ON(mas->tree, node != mas->node); + } else if (node == mas->node) { pr_err("parent contains invalid child at "MA_PTR"[%u] " MA_PTR" p_slot %u\n", parent, i, mas_mn(mas), p_slot); - MT_BUG_ON(mas->tree, - ma_get_rcu_slot(parent, i, p_type, mas->tree) == - mas->node); + MT_BUG_ON(mas->tree, node == mas->node); } } } diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c index 6632772370c7..6cd3421c291e 100644 --- a/lib/test_maple_tree.c +++ b/lib/test_maple_tree.c @@ -29681,6 +29681,1130 @@ ERASE, 140612699410432, 140612707799039, #endif }; + unsigned long set39[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140736271417344, 140737488351231, +SNULL, 140736271421439, 140737488351231, +STORE, 140736271417344, 140736271421439, +STORE, 140736271286272, 140736271421439, +STORE, 94412930822144, 94412933074943, +SNULL, 94412930953215, 94412933074943, +STORE, 94412930822144, 94412930953215, +STORE, 94412930953216, 94412933074943, +ERASE, 94412930953216, 94412933074943, +STORE, 94412933046272, 94412933054463, +STORE, 94412933054464, 94412933074943, +STORE, 140326136901632, 140326139154431, +SNULL, 140326137044991, 140326139154431, +STORE, 140326136901632, 140326137044991, +STORE, 140326137044992, 140326139154431, +ERASE, 140326137044992, 140326139154431, +STORE, 140326139142144, 140326139150335, +STORE, 140326139150336, 140326139154431, +STORE, 140736271585280, 140736271589375, +STORE, 140736271572992, 140736271585279, +STORE, 140326139113472, 140326139142143, +STORE, 140326139105280, 140326139113471, +STORE, 140326134685696, 140326136901631, +SNULL, 140326134685696, 140326134783999, +STORE, 140326134784000, 140326136901631, +STORE, 140326134685696, 140326134783999, +SNULL, 140326136877055, 140326136901631, +STORE, 140326134784000, 140326136877055, +STORE, 140326136877056, 140326136901631, +SNULL, 140326136877056, 140326136885247, +STORE, 140326136885248, 140326136901631, +STORE, 140326136877056, 140326136885247, +ERASE, 140326136877056, 140326136885247, +STORE, 140326136877056, 140326136885247, +ERASE, 140326136885248, 140326136901631, +STORE, 140326136885248, 140326136901631, +STORE, 140326130888704, 140326134685695, +SNULL, 140326130888704, 140326132547583, +STORE, 140326132547584, 140326134685695, +STORE, 140326130888704, 140326132547583, +SNULL, 140326134644735, 140326134685695, +STORE, 140326132547584, 140326134644735, +STORE, 140326134644736, 140326134685695, +SNULL, 140326134644736, 140326134669311, +STORE, 140326134669312, 140326134685695, +STORE, 140326134644736, 140326134669311, +ERASE, 140326134644736, 140326134669311, +STORE, 140326134644736, 140326134669311, +ERASE, 140326134669312, 140326134685695, +STORE, 140326134669312, 140326134685695, +STORE, 140326139097088, 140326139113471, +SNULL, 140326134661119, 140326134669311, +STORE, 140326134644736, 140326134661119, +STORE, 140326134661120, 140326134669311, +SNULL, 140326136881151, 140326136885247, +STORE, 140326136877056, 140326136881151, +STORE, 140326136881152, 140326136885247, +SNULL, 94412933050367, 94412933054463, +STORE, 94412933046272, 94412933050367, +STORE, 94412933050368, 94412933054463, +SNULL, 140326139146239, 140326139150335, +STORE, 140326139142144, 140326139146239, +STORE, 140326139146240, 140326139150335, +ERASE, 140326139113472, 140326139142143, +STORE, 94412939493376, 94412939628543, +STORE, 140326122496000, 140326130888703, +SNULL, 140326122500095, 140326130888703, +STORE, 140326122496000, 140326122500095, +STORE, 140326122500096, 140326130888703, +STORE, 140326114103296, 140326122495999, +STORE, 140325979885568, 140326114103295, +SNULL, 140325979885568, 140326043910143, +STORE, 140326043910144, 140326114103295, +STORE, 140325979885568, 140326043910143, +ERASE, 140325979885568, 140326043910143, +SNULL, 140326111019007, 140326114103295, +STORE, 140326043910144, 140326111019007, +STORE, 140326111019008, 140326114103295, +ERASE, 140326111019008, 140326114103295, +SNULL, 140326044045311, 140326111019007, +STORE, 140326043910144, 140326044045311, +STORE, 140326044045312, 140326111019007, +SNULL, 140326114107391, 140326122495999, +STORE, 140326114103296, 140326114107391, +STORE, 140326114107392, 140326122495999, +STORE, 140326035517440, 140326043910143, +SNULL, 140326035521535, 140326043910143, +STORE, 140326035517440, 140326035521535, +STORE, 140326035521536, 140326043910143, +STORE, 140326027124736, 140326035517439, +SNULL, 140326027128831, 140326035517439, +STORE, 140326027124736, 140326027128831, +STORE, 140326027128832, 140326035517439, +STORE, 140326018732032, 140326027124735, +SNULL, 140326018736127, 140326027124735, +STORE, 140326018732032, 140326018736127, +STORE, 140326018736128, 140326027124735, +STORE, 140326010339328, 140326018732031, +STORE, 140326001946624, 140326018732031, +STORE, 140325993553920, 140326018732031, +STORE, 140325859336192, 140325993553919, +SNULL, 140325859336192, 140325909692415, +STORE, 140325909692416, 140325993553919, +STORE, 140325859336192, 140325909692415, +ERASE, 140325859336192, 140325909692415, +SNULL, 140325976801279, 140325993553919, +STORE, 140325909692416, 140325976801279, +STORE, 140325976801280, 140325993553919, +ERASE, 140325976801280, 140325993553919, +STORE, 140325985161216, 140326018732031, +STORE, 140325775474688, 140325976801279, +STORE, 140325708365824, 140325976801279, +SNULL, 140325708500991, 140325976801279, +STORE, 140325708365824, 140325708500991, +STORE, 140325708500992, 140325976801279, +SNULL, 140325708500992, 140325909692415, +STORE, 140325909692416, 140325976801279, +STORE, 140325708500992, 140325909692415, +SNULL, 140325909827583, 140325976801279, +STORE, 140325909692416, 140325909827583, +STORE, 140325909827584, 140325976801279, +SNULL, 140325842583551, 140325909692415, +STORE, 140325708500992, 140325842583551, +STORE, 140325842583552, 140325909692415, +ERASE, 140325842583552, 140325909692415, +SNULL, 140325708500992, 140325775474687, +STORE, 140325775474688, 140325842583551, +STORE, 140325708500992, 140325775474687, +SNULL, 140325775609855, 140325842583551, +STORE, 140325775474688, 140325775609855, +STORE, 140325775609856, 140325842583551, +STORE, 140325775609856, 140325909692415, +SNULL, 140325775609856, 140325842583551, +STORE, 140325842583552, 140325909692415, +STORE, 140325775609856, 140325842583551, +SNULL, 140325842718719, 140325909692415, +STORE, 140325842583552, 140325842718719, +STORE, 140325842718720, 140325909692415, +SNULL, 140325985161216, 140325993553919, +STORE, 140325993553920, 140326018732031, +STORE, 140325985161216, 140325993553919, +SNULL, 140325993558015, 140326018732031, +STORE, 140325993553920, 140325993558015, +STORE, 140325993558016, 140326018732031, +SNULL, 140325985165311, 140325993553919, +STORE, 140325985161216, 140325985165311, +STORE, 140325985165312, 140325993553919, +SNULL, 140325993558016, 140326001946623, +STORE, 140326001946624, 140326018732031, +STORE, 140325993558016, 140326001946623, +SNULL, 140326001950719, 140326018732031, +STORE, 140326001946624, 140326001950719, +STORE, 140326001950720, 140326018732031, +SNULL, 140326001950720, 140326010339327, +STORE, 140326010339328, 140326018732031, +STORE, 140326001950720, 140326010339327, +SNULL, 140326010343423, 140326018732031, +STORE, 140326010339328, 140326010343423, +STORE, 140326010343424, 140326018732031, +STORE, 140325699973120, 140325708365823, +STORE, 140325691580416, 140325708365823, +STORE, 140325683187712, 140325708365823, +SNULL, 140325683191807, 140325708365823, +STORE, 140325683187712, 140325683191807, +STORE, 140325683191808, 140325708365823, +SNULL, 140325683191808, 140325699973119, +STORE, 140325699973120, 140325708365823, +STORE, 140325683191808, 140325699973119, +SNULL, 140325699977215, 140325708365823, +STORE, 140325699973120, 140325699977215, +STORE, 140325699977216, 140325708365823, +STORE, 140325674795008, 140325683187711, +STORE, 140325666402304, 140325683187711, +STORE, 140325658009600, 140325683187711, +SNULL, 140325658009600, 140325666402303, +STORE, 140325666402304, 140325683187711, +STORE, 140325658009600, 140325666402303, +SNULL, 140325666406399, 140325683187711, +STORE, 140325666402304, 140325666406399, +STORE, 140325666406400, 140325683187711, +SNULL, 140325683191808, 140325691580415, +STORE, 140325691580416, 140325699973119, +STORE, 140325683191808, 140325691580415, +SNULL, 140325691584511, 140325699973119, +STORE, 140325691580416, 140325691584511, +STORE, 140325691584512, 140325699973119, +SNULL, 140325666406400, 140325674795007, +STORE, 140325674795008, 140325683187711, +STORE, 140325666406400, 140325674795007, +SNULL, 140325674799103, 140325683187711, +STORE, 140325674795008, 140325674799103, +STORE, 140325674799104, 140325683187711, +STORE, 140325649616896, 140325666402303, +SNULL, 140325649616896, 140325658009599, +STORE, 140325658009600, 140325666402303, +STORE, 140325649616896, 140325658009599, +SNULL, 140325658013695, 140325666402303, +STORE, 140325658009600, 140325658013695, +STORE, 140325658013696, 140325666402303, +SNULL, 140325649620991, 140325658009599, +STORE, 140325649616896, 140325649620991, +STORE, 140325649620992, 140325658009599, +STORE, 140325641224192, 140325649616895, +STORE, 140325632831488, 140325649616895, +SNULL, 140325632835583, 140325649616895, +STORE, 140325632831488, 140325632835583, +STORE, 140325632835584, 140325649616895, +STORE, 140325624438784, 140325632831487, +SNULL, 140325624442879, 140325632831487, +STORE, 140325624438784, 140325624442879, +STORE, 140325624442880, 140325632831487, +SNULL, 140325632835584, 140325641224191, +STORE, 140325641224192, 140325649616895, +STORE, 140325632835584, 140325641224191, +SNULL, 140325641228287, 140325649616895, +STORE, 140325641224192, 140325641228287, +STORE, 140325641228288, 140325649616895, +STORE, 140325616046080, 140325624438783, +SNULL, 140325616050175, 140325624438783, +STORE, 140325616046080, 140325616050175, +STORE, 140325616050176, 140325624438783, +STORE, 140325607653376, 140325616046079, +SNULL, 140325607657471, 140325616046079, +STORE, 140325607653376, 140325607657471, +STORE, 140325607657472, 140325616046079, +STORE, 140325599260672, 140325607653375, +STORE, 140325590867968, 140325607653375, +STORE, 140325456650240, 140325590867967, +SNULL, 140325456650240, 140325507039231, +STORE, 140325507039232, 140325590867967, +STORE, 140325456650240, 140325507039231, +ERASE, 140325456650240, 140325507039231, +STORE, 140325498646528, 140325507039231, +STORE, 140325364428800, 140325498646527, +SNULL, 140325364428800, 140325372821503, +STORE, 140325372821504, 140325498646527, +STORE, 140325364428800, 140325372821503, +ERASE, 140325364428800, 140325372821503, +STORE, 140325364428800, 140325372821503, +STORE, 140325356036096, 140325372821503, +STORE, 140325221818368, 140325356036095, +SNULL, 140325221818368, 140325238603775, +STORE, 140325238603776, 140325356036095, +STORE, 140325221818368, 140325238603775, +ERASE, 140325221818368, 140325238603775, +STORE, 140325230211072, 140325238603775, +STORE, 140325221818368, 140325238603775, +STORE, 140325087600640, 140325221818367, +STORE, 140325079207936, 140325087600639, +SNULL, 140325087600640, 140325104386047, +STORE, 140325104386048, 140325221818367, +STORE, 140325087600640, 140325104386047, +ERASE, 140325087600640, 140325104386047, +STORE, 140325095993344, 140325104386047, +STORE, 140325079207936, 140325104386047, +STORE, 140324944990208, 140325079207935, +SNULL, 140324944990208, 140324970168319, +STORE, 140324970168320, 140325079207935, +STORE, 140324944990208, 140324970168319, +ERASE, 140324944990208, 140324970168319, +STORE, 140324961775616, 140324970168319, +STORE, 140324953382912, 140324970168319, +STORE, 140324819165184, 140324953382911, +STORE, 140324684947456, 140324953382911, +STORE, 140324676554752, 140324684947455, +STORE, 140324668162048, 140324684947455, +STORE, 140324533944320, 140324668162047, +STORE, 140324525551616, 140324533944319, +SNULL, 140324533944320, 140324567515135, +STORE, 140324567515136, 140324668162047, +STORE, 140324533944320, 140324567515135, +ERASE, 140324533944320, 140324567515135, +STORE, 140324559122432, 140324567515135, +STORE, 140324391333888, 140324525551615, +SNULL, 140325574148095, 140325590867967, +STORE, 140325507039232, 140325574148095, +STORE, 140325574148096, 140325590867967, +ERASE, 140325574148096, 140325590867967, +SNULL, 140325439930367, 140325498646527, +STORE, 140325372821504, 140325439930367, +STORE, 140325439930368, 140325498646527, +ERASE, 140325439930368, 140325498646527, +SNULL, 140325305712639, 140325356036095, +STORE, 140325238603776, 140325305712639, +STORE, 140325305712640, 140325356036095, +ERASE, 140325305712640, 140325356036095, +SNULL, 140325171494911, 140325221818367, +STORE, 140325104386048, 140325171494911, +STORE, 140325171494912, 140325221818367, +ERASE, 140325171494912, 140325221818367, +SNULL, 140325104521215, 140325171494911, +STORE, 140325104386048, 140325104521215, +STORE, 140325104521216, 140325171494911, +STORE, 140324257116160, 140324525551615, +SNULL, 140324257116160, 140324299079679, +STORE, 140324299079680, 140324525551615, +STORE, 140324257116160, 140324299079679, +ERASE, 140324257116160, 140324299079679, +SNULL, 140325037277183, 140325079207935, +STORE, 140324970168320, 140325037277183, +STORE, 140325037277184, 140325079207935, +ERASE, 140325037277184, 140325079207935, +SNULL, 140324819165183, 140324953382911, +STORE, 140324684947456, 140324819165183, +STORE, 140324819165184, 140324953382911, +SNULL, 140324819165184, 140324835950591, +STORE, 140324835950592, 140324953382911, +STORE, 140324819165184, 140324835950591, +ERASE, 140324819165184, 140324835950591, +SNULL, 140324903059455, 140324953382911, +STORE, 140324835950592, 140324903059455, +STORE, 140324903059456, 140324953382911, +ERASE, 140324903059456, 140324953382911, +SNULL, 140324684947456, 140324701732863, +STORE, 140324701732864, 140324819165183, +STORE, 140324684947456, 140324701732863, +ERASE, 140324684947456, 140324701732863, +SNULL, 140324768841727, 140324819165183, +STORE, 140324701732864, 140324768841727, +STORE, 140324768841728, 140324819165183, +ERASE, 140324768841728, 140324819165183, +SNULL, 140324634623999, 140324668162047, +STORE, 140324567515136, 140324634623999, +STORE, 140324634624000, 140324668162047, +ERASE, 140324634624000, 140324668162047, +SNULL, 140324391333887, 140324525551615, +STORE, 140324299079680, 140324391333887, +STORE, 140324391333888, 140324525551615, +SNULL, 140324391333888, 140324433297407, +STORE, 140324433297408, 140324525551615, +STORE, 140324391333888, 140324433297407, +ERASE, 140324391333888, 140324433297407, +SNULL, 140325507174399, 140325574148095, +STORE, 140325507039232, 140325507174399, +STORE, 140325507174400, 140325574148095, +SNULL, 140325590867968, 140325599260671, +STORE, 140325599260672, 140325607653375, +STORE, 140325590867968, 140325599260671, +SNULL, 140325599264767, 140325607653375, +STORE, 140325599260672, 140325599264767, +STORE, 140325599264768, 140325607653375, +SNULL, 140325372956671, 140325439930367, +STORE, 140325372821504, 140325372956671, +STORE, 140325372956672, 140325439930367, +SNULL, 140324668166143, 140324684947455, +STORE, 140324668162048, 140324668166143, +STORE, 140324668166144, 140324684947455, +SNULL, 140324525555711, 140324533944319, +STORE, 140324525551616, 140324525555711, +STORE, 140324525555712, 140324533944319, +SNULL, 140324953382912, 140324961775615, +STORE, 140324961775616, 140324970168319, +STORE, 140324953382912, 140324961775615, +SNULL, 140324961779711, 140324970168319, +STORE, 140324961775616, 140324961779711, +STORE, 140324961779712, 140324970168319, +SNULL, 140325079212031, 140325104386047, +STORE, 140325079207936, 140325079212031, +STORE, 140325079212032, 140325104386047, +SNULL, 140325221818368, 140325230211071, +STORE, 140325230211072, 140325238603775, +STORE, 140325221818368, 140325230211071, +SNULL, 140325230215167, 140325238603775, +STORE, 140325230211072, 140325230215167, +STORE, 140325230215168, 140325238603775, +SNULL, 140325356036096, 140325364428799, +STORE, 140325364428800, 140325372821503, +STORE, 140325356036096, 140325364428799, +SNULL, 140325364432895, 140325372821503, + }; + unsigned long set40[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140734309167104, 140737488351231, +SNULL, 140734309171199, 140737488351231, +STORE, 140734309167104, 140734309171199, +STORE, 140734309036032, 140734309171199, +STORE, 94270500081664, 94270502334463, +SNULL, 94270500212735, 94270502334463, +STORE, 94270500081664, 94270500212735, +STORE, 94270500212736, 94270502334463, +ERASE, 94270500212736, 94270502334463, +STORE, 94270502305792, 94270502313983, +STORE, 94270502313984, 94270502334463, +STORE, 140321935110144, 140321937362943, +SNULL, 140321935253503, 140321937362943, +STORE, 140321935110144, 140321935253503, +STORE, 140321935253504, 140321937362943, +ERASE, 140321935253504, 140321937362943, +STORE, 140321937350656, 140321937358847, +STORE, 140321937358848, 140321937362943, +STORE, 140734309625856, 140734309629951, +STORE, 140734309613568, 140734309625855, +STORE, 140321937321984, 140321937350655, +STORE, 140321937313792, 140321937321983, +STORE, 140321932894208, 140321935110143, +SNULL, 140321932894208, 140321932992511, +STORE, 140321932992512, 140321935110143, +STORE, 140321932894208, 140321932992511, +SNULL, 140321935085567, 140321935110143, +STORE, 140321932992512, 140321935085567, +STORE, 140321935085568, 140321935110143, +SNULL, 140321935085568, 140321935093759, +STORE, 140321935093760, 140321935110143, +STORE, 140321935085568, 140321935093759, +ERASE, 140321935085568, 140321935093759, +STORE, 140321935085568, 140321935093759, +ERASE, 140321935093760, 140321935110143, +STORE, 140321935093760, 140321935110143, +STORE, 140321929097216, 140321932894207, +SNULL, 140321929097216, 140321930756095, +STORE, 140321930756096, 140321932894207, +STORE, 140321929097216, 140321930756095, +SNULL, 140321932853247, 140321932894207, +STORE, 140321930756096, 140321932853247, +STORE, 140321932853248, 140321932894207, +SNULL, 140321932853248, 140321932877823, +STORE, 140321932877824, 140321932894207, +STORE, 140321932853248, 140321932877823, +ERASE, 140321932853248, 140321932877823, +STORE, 140321932853248, 140321932877823, +ERASE, 140321932877824, 140321932894207, +STORE, 140321932877824, 140321932894207, +STORE, 140321937305600, 140321937321983, +SNULL, 140321932869631, 140321932877823, +STORE, 140321932853248, 140321932869631, +STORE, 140321932869632, 140321932877823, +SNULL, 140321935089663, 140321935093759, +STORE, 140321935085568, 140321935089663, +STORE, 140321935089664, 140321935093759, +SNULL, 94270502309887, 94270502313983, +STORE, 94270502305792, 94270502309887, +STORE, 94270502309888, 94270502313983, +SNULL, 140321937354751, 140321937358847, +STORE, 140321937350656, 140321937354751, +STORE, 140321937354752, 140321937358847, +ERASE, 140321937321984, 140321937350655, +STORE, 94270507364352, 94270507499519, +STORE, 140321920704512, 140321929097215, +SNULL, 140321920708607, 140321929097215, +STORE, 140321920704512, 140321920708607, +STORE, 140321920708608, 140321929097215, +STORE, 140321912311808, 140321920704511, +STORE, 140321778094080, 140321912311807, +SNULL, 140321778094080, 140321816051711, +STORE, 140321816051712, 140321912311807, +STORE, 140321778094080, 140321816051711, +ERASE, 140321778094080, 140321816051711, +SNULL, 140321883160575, 140321912311807, +STORE, 140321816051712, 140321883160575, +STORE, 140321883160576, 140321912311807, +ERASE, 140321883160576, 140321912311807, +SNULL, 140321816186879, 140321883160575, +STORE, 140321816051712, 140321816186879, +STORE, 140321816186880, 140321883160575, +SNULL, 140321912315903, 140321920704511, +STORE, 140321912311808, 140321912315903, +STORE, 140321912315904, 140321920704511, +STORE, 140321903919104, 140321912311807, +SNULL, 140321903923199, 140321912311807, +STORE, 140321903919104, 140321903923199, +STORE, 140321903923200, 140321912311807, +STORE, 140321895526400, 140321903919103, +SNULL, 140321895530495, 140321903919103, +STORE, 140321895526400, 140321895530495, +STORE, 140321895530496, 140321903919103, +STORE, 140321887133696, 140321895526399, +SNULL, 140321887137791, 140321895526399, +STORE, 140321887133696, 140321887137791, +STORE, 140321887137792, 140321895526399, +STORE, 140321807659008, 140321816051711, +STORE, 140321673441280, 140321807659007, +SNULL, 140321673441280, 140321681833983, +STORE, 140321681833984, 140321807659007, +STORE, 140321673441280, 140321681833983, +ERASE, 140321673441280, 140321681833983, +SNULL, 140321748942847, 140321807659007, +STORE, 140321681833984, 140321748942847, +STORE, 140321748942848, 140321807659007, +ERASE, 140321748942848, 140321807659007, +STORE, 140321799266304, 140321816051711, +STORE, 140321790873600, 140321816051711, +STORE, 140321782480896, 140321816051711, +STORE, 140321547616256, 140321748942847, +SNULL, 140321614725119, 140321748942847, +STORE, 140321547616256, 140321614725119, +STORE, 140321614725120, 140321748942847, +SNULL, 140321614725120, 140321681833983, +STORE, 140321681833984, 140321748942847, +STORE, 140321614725120, 140321681833983, +ERASE, 140321614725120, 140321681833983, +SNULL, 140321681969151, 140321748942847, +STORE, 140321681833984, 140321681969151, +STORE, 140321681969152, 140321748942847, +STORE, 140321547616256, 140321681833983, +SNULL, 140321547616256, 140321614725119, +STORE, 140321614725120, 140321681833983, +STORE, 140321547616256, 140321614725119, +SNULL, 140321614860287, 140321681833983, +STORE, 140321614725120, 140321614860287, +STORE, 140321614860288, 140321681833983, +SNULL, 140321547751423, 140321614725119, +STORE, 140321547616256, 140321547751423, +STORE, 140321547751424, 140321614725119, +STORE, 140321480507392, 140321547616255, +SNULL, 140321782480896, 140321799266303, +STORE, 140321799266304, 140321816051711, +STORE, 140321782480896, 140321799266303, +SNULL, 140321799270399, 140321816051711, +STORE, 140321799266304, 140321799270399, +STORE, 140321799270400, 140321816051711, +STORE, 140321774088192, 140321799266303, +SNULL, 140321774088192, 140321790873599, +STORE, 140321790873600, 140321799266303, +STORE, 140321774088192, 140321790873599, +SNULL, 140321790877695, 140321799266303, +STORE, 140321790873600, 140321790877695, +STORE, 140321790877696, 140321799266303, +SNULL, 140321480642559, 140321547616255, +STORE, 140321480507392, 140321480642559, +STORE, 140321480642560, 140321547616255, +SNULL, 140321774088192, 140321782480895, +STORE, 140321782480896, 140321790873599, +STORE, 140321774088192, 140321782480895, +SNULL, 140321782484991, 140321790873599, +STORE, 140321782480896, 140321782484991, +STORE, 140321782484992, 140321790873599, +SNULL, 140321799270400, 140321807659007, +STORE, 140321807659008, 140321816051711, +STORE, 140321799270400, 140321807659007, +SNULL, 140321807663103, 140321816051711, +STORE, 140321807659008, 140321807663103, +STORE, 140321807663104, 140321816051711, +STORE, 140321765695488, 140321782480895, +STORE, 140321757302784, 140321782480895, +SNULL, 140321757306879, 140321782480895, +STORE, 140321757302784, 140321757306879, +STORE, 140321757306880, 140321782480895, +STORE, 140321472114688, 140321480507391, +STORE, 140321463721984, 140321480507391, +SNULL, 140321463726079, 140321480507391, +STORE, 140321463721984, 140321463726079, +STORE, 140321463726080, 140321480507391, +SNULL, 140321757306880, 140321774088191, +STORE, 140321774088192, 140321782480895, +STORE, 140321757306880, 140321774088191, +SNULL, 140321774092287, 140321782480895, +STORE, 140321774088192, 140321774092287, +STORE, 140321774092288, 140321782480895, +SNULL, 140321463726080, 140321472114687, +STORE, 140321472114688, 140321480507391, +STORE, 140321463726080, 140321472114687, +SNULL, 140321472118783, 140321480507391, +STORE, 140321472114688, 140321472118783, +STORE, 140321472118784, 140321480507391, +SNULL, 140321757306880, 140321765695487, +STORE, 140321765695488, 140321774088191, +STORE, 140321757306880, 140321765695487, +SNULL, 140321765699583, 140321774088191, +STORE, 140321765695488, 140321765699583, +STORE, 140321765699584, 140321774088191, +STORE, 140321455329280, 140321463721983, +SNULL, 140321455333375, 140321463721983, +STORE, 140321455329280, 140321455333375, +STORE, 140321455333376, 140321463721983, +STORE, 140321446936576, 140321455329279, +STORE, 140321438543872, 140321455329279, +STORE, 140321430151168, 140321455329279, +SNULL, 140321430155263, 140321455329279, +STORE, 140321430151168, 140321430155263, +STORE, 140321430155264, 140321455329279, +SNULL, 140321430155264, 140321446936575, +STORE, 140321446936576, 140321455329279, +STORE, 140321430155264, 140321446936575, +SNULL, 140321446940671, 140321455329279, +STORE, 140321446936576, 140321446940671, +STORE, 140321446940672, 140321455329279, +SNULL, 140321430155264, 140321438543871, +STORE, 140321438543872, 140321446936575, +STORE, 140321430155264, 140321438543871, +SNULL, 140321438547967, 140321446936575, +STORE, 140321438543872, 140321438547967, +STORE, 140321438547968, 140321446936575, +STORE, 140321421758464, 140321430151167, +SNULL, 140321421762559, 140321430151167, +STORE, 140321421758464, 140321421762559, +STORE, 140321421762560, 140321430151167, +STORE, 140321413365760, 140321421758463, +SNULL, 140321413369855, 140321421758463, +STORE, 140321413365760, 140321413369855, +STORE, 140321413369856, 140321421758463, +STORE, 140321404973056, 140321413365759, +SNULL, 140321404977151, 140321413365759, +STORE, 140321404973056, 140321404977151, +STORE, 140321404977152, 140321413365759, +STORE, 140321396580352, 140321404973055, +STORE, 140321388187648, 140321404973055, +STORE, 140321253969920, 140321388187647, +SNULL, 140321253969920, 140321279180799, +STORE, 140321279180800, 140321388187647, +STORE, 140321253969920, 140321279180799, +ERASE, 140321253969920, 140321279180799, +SNULL, 140321346289663, 140321388187647, +STORE, 140321279180800, 140321346289663, +STORE, 140321346289664, 140321388187647, +ERASE, 140321346289664, 140321388187647, +STORE, 140321144963072, 140321346289663, +STORE, 140321379794944, 140321404973055, +STORE, 140321371402240, 140321404973055, +STORE, 140321010745344, 140321346289663, +STORE, 140321363009536, 140321404973055, +SNULL, 140321077854207, 140321346289663, +STORE, 140321010745344, 140321077854207, +STORE, 140321077854208, 140321346289663, +SNULL, 140321077854208, 140321144963071, +STORE, 140321144963072, 140321346289663, +STORE, 140321077854208, 140321144963071, +ERASE, 140321077854208, 140321144963071, +STORE, 140321354616832, 140321404973055, +STORE, 140321136570368, 140321144963071, +STORE, 140320943636480, 140321077854207, +STORE, 140320876527616, 140321077854207, +STORE, 140321128177664, 140321144963071, +SNULL, 140320876662783, 140321077854207, +STORE, 140320876527616, 140320876662783, +STORE, 140320876662784, 140321077854207, +STORE, 140321119784960, 140321144963071, +STORE, 140321111392256, 140321144963071, +STORE, 140320742309888, 140320876527615, +STORE, 140321102999552, 140321144963071, +STORE, 140320608092160, 140320876527615, +SNULL, 140320675201023, 140320876527615, +STORE, 140320608092160, 140320675201023, +STORE, 140320675201024, 140320876527615, +SNULL, 140320675201024, 140320742309887, +STORE, 140320742309888, 140320876527615, +STORE, 140320675201024, 140320742309887, +ERASE, 140320675201024, 140320742309887, +STORE, 140321094606848, 140321144963071, +STORE, 140321086214144, 140321144963071, +STORE, 140320608092160, 140320876527615, +SNULL, 140320608092160, 140320675201023, +STORE, 140320675201024, 140320876527615, +STORE, 140320608092160, 140320675201023, +SNULL, 140320675336191, 140320876527615, +STORE, 140320675201024, 140320675336191, +STORE, 140320675336192, 140320876527615, +STORE, 140320599699456, 140320608092159, +STORE, 140320591306752, 140320608092159, +STORE, 140320457089024, 140320591306751, +STORE, 140320448696320, 140320457089023, +STORE, 140320314478592, 140320448696319, +SNULL, 140321144963072, 140321279180799, +STORE, 140321279180800, 140321346289663, +STORE, 140321144963072, 140321279180799, +SNULL, 140321279315967, 140321346289663, +STORE, 140321279180800, 140321279315967, +STORE, 140321279315968, 140321346289663, +SNULL, 140321086214144, 140321136570367, +STORE, 140321136570368, 140321144963071, +STORE, 140321086214144, 140321136570367, +SNULL, 140321136574463, 140321144963071, +STORE, 140321136570368, 140321136574463, +STORE, 140321136574464, 140321144963071, +SNULL, 140321212071935, 140321279180799, +STORE, 140321144963072, 140321212071935, +STORE, 140321212071936, 140321279180799, +ERASE, 140321212071936, 140321279180799, +SNULL, 140321145098239, 140321212071935, +STORE, 140321144963072, 140321145098239, +STORE, 140321145098240, 140321212071935, +SNULL, 140320876662784, 140321010745343, +STORE, 140321010745344, 140321077854207, +STORE, 140320876662784, 140321010745343, +SNULL, 140321010880511, 140321077854207, +STORE, 140321010745344, 140321010880511, +STORE, 140321010880512, 140321077854207, +SNULL, 140321354616832, 140321379794943, +STORE, 140321379794944, 140321404973055, +STORE, 140321354616832, 140321379794943, +SNULL, 140321379799039, 140321404973055, +STORE, 140321379794944, 140321379799039, +STORE, 140321379799040, 140321404973055, +SNULL, 140320876662784, 140320943636479, +STORE, 140320943636480, 140321010745343, +STORE, 140320876662784, 140320943636479, +SNULL, 140320943771647, 140321010745343, +STORE, 140320943636480, 140320943771647, +STORE, 140320943771648, 140321010745343, +SNULL, 140320809418751, 140320876527615, +STORE, 140320675336192, 140320809418751, +STORE, 140320809418752, 140320876527615, +ERASE, 140320809418752, 140320876527615, +SNULL, 140320675336192, 140320742309887, +STORE, 140320742309888, 140320809418751, +STORE, 140320675336192, 140320742309887, +SNULL, 140320742445055, 140320809418751, +STORE, 140320742309888, 140320742445055, +STORE, 140320742445056, 140320809418751, +SNULL, 140320608227327, 140320675201023, +STORE, 140320608092160, 140320608227327, +STORE, 140320608227328, 140320675201023, +SNULL, 140320457089024, 140320473874431, +STORE, 140320473874432, 140320591306751, +STORE, 140320457089024, 140320473874431, +ERASE, 140320457089024, 140320473874431, +SNULL, 140320540983295, 140320591306751, +STORE, 140320473874432, 140320540983295, +STORE, 140320540983296, 140320591306751, +ERASE, 140320540983296, 140320591306751, +SNULL, 140320314478592, 140320339656703, +STORE, 140320339656704, 140320448696319, +STORE, 140320314478592, 140320339656703, +ERASE, 140320314478592, 140320339656703, +SNULL, 140321086214144, 140321128177663, +STORE, 140321128177664, 140321136570367, +STORE, 140321086214144, 140321128177663, +SNULL, 140321128181759, 140321136570367, +STORE, 140321128177664, 140321128181759, +STORE, 140321128181760, 140321136570367, +SNULL, 140321354616832, 140321371402239, +STORE, 140321371402240, 140321379794943, +STORE, 140321354616832, 140321371402239, +SNULL, 140321371406335, 140321379794943, +STORE, 140321371402240, 140321371406335, +STORE, 140321371406336, 140321379794943, +SNULL, 140320591310847, 140320608092159, +STORE, 140320591306752, 140320591310847, +STORE, 140320591310848, 140320608092159, +SNULL, 140321354616832, 140321363009535, +STORE, 140321363009536, 140321371402239, +STORE, 140321354616832, 140321363009535, +SNULL, 140321363013631, 140321371402239, +STORE, 140321363009536, 140321363013631, +STORE, 140321363013632, 140321371402239, +SNULL, 140321086214144, 140321119784959, +STORE, 140321119784960, 140321128177663, +STORE, 140321086214144, 140321119784959, +SNULL, 140321119789055, 140321128177663, +STORE, 140321119784960, 140321119789055, +STORE, 140321119789056, 140321128177663, +SNULL, 140321086218239, 140321119784959, +STORE, 140321086214144, 140321086218239, +STORE, 140321086218240, 140321119784959, +SNULL, 140321086218240, 140321094606847, +STORE, 140321094606848, 140321119784959, +STORE, 140321086218240, 140321094606847, +SNULL, 140321094610943, 140321119784959, +STORE, 140321094606848, 140321094610943, +STORE, 140321094610944, 140321119784959, +SNULL, 140320474009599, 140320540983295, +STORE, 140320473874432, 140320474009599, +STORE, 140320474009600, 140320540983295, +SNULL, 140320406765567, 140320448696319, +STORE, 140320339656704, 140320406765567, +STORE, 140320406765568, 140320448696319, +ERASE, 140320406765568, 140320448696319, +SNULL, 140320339791871, 140320406765567, +STORE, 140320339656704, 140320339791871, +STORE, 140320339791872, 140320406765567, +STORE, 140321270788096, 140321279180799, +STORE, 140321262395392, 140321279180799, +STORE, 140321254002688, 140321279180799, +SNULL, 140321254002688, 140321262395391, +STORE, 140321262395392, 140321279180799, +STORE, 140321254002688, 140321262395391, +SNULL, 140321262399487, 140321279180799, +STORE, 140321262395392, 140321262399487, +STORE, 140321262399488, 140321279180799, +STORE, 140321245609984, 140321262395391, +STORE, 140321237217280, 140321262395391, +SNULL, 140321237217280, 140321245609983, +STORE, 140321245609984, 140321262395391, +STORE, 140321237217280, 140321245609983, +SNULL, 140321245614079, 140321262395391, +STORE, 140321245609984, 140321245614079, +STORE, 140321245614080, 140321262395391, +SNULL, 140321379799040, 140321388187647, +STORE, 140321388187648, 140321404973055, +STORE, 140321379799040, 140321388187647, +SNULL, 140321388191743, 140321404973055, +STORE, 140321388187648, 140321388191743, +STORE, 140321388191744, 140321404973055, +SNULL, 140321354620927, 140321363009535, +STORE, 140321354616832, 140321354620927, +STORE, 140321354620928, 140321363009535, +SNULL, 140321388191744, 140321396580351, +STORE, 140321396580352, 140321404973055, +STORE, 140321388191744, 140321396580351, +SNULL, 140321396584447, 140321404973055, +STORE, 140321396580352, 140321396584447, +STORE, 140321396584448, 140321404973055, +SNULL, 140321094610944, 140321111392255, +STORE, 140321111392256, 140321119784959, +STORE, 140321094610944, 140321111392255, +SNULL, 140321111396351, 140321119784959, +STORE, 140321111392256, 140321111396351, +STORE, 140321111396352, 140321119784959, +STORE, 140321228824576, 140321245609983, +SNULL, 140321094610944, 140321102999551, +STORE, 140321102999552, 140321111392255, +STORE, 140321094610944, 140321102999551, +SNULL, 140321103003647, 140321111392255, +STORE, 140321102999552, 140321103003647, +STORE, 140321103003648, 140321111392255, +STORE, 140321220431872, 140321245609983, +SNULL, 140321220435967, 140321245609983, +STORE, 140321220431872, 140321220435967, +STORE, 140321220435968, 140321245609983, +STORE, 140320868134912, 140320876527615, +SNULL, 140320868139007, 140320876527615, +STORE, 140320868134912, 140320868139007, +STORE, 140320868139008, 140320876527615, +SNULL, 140320591310848, 140320599699455, +STORE, 140320599699456, 140320608092159, +STORE, 140320591310848, 140320599699455, +SNULL, 140320599703551, 140320608092159, +STORE, 140320599699456, 140320599703551, +STORE, 140320599703552, 140320608092159, +STORE, 140320859742208, 140320868134911, +SNULL, 140321262399488, 140321270788095, +STORE, 140321270788096, 140321279180799, +STORE, 140321262399488, 140321270788095, +SNULL, 140321270792191, 140321279180799, +STORE, 140321270788096, 140321270792191, +STORE, 140321270792192, 140321279180799, +STORE, 140320851349504, 140320868134911, +STORE, 140320842956800, 140320868134911, +STORE, 140320834564096, 140320868134911, +STORE, 140320826171392, 140320868134911, +SNULL, 140320826171392, 140320834564095, +STORE, 140320834564096, 140320868134911, +STORE, 140320826171392, 140320834564095, +SNULL, 140320834568191, 140320868134911, +STORE, 140320834564096, 140320834568191, +STORE, 140320834568192, 140320868134911, +SNULL, 140321220435968, 140321228824575, +STORE, 140321228824576, 140321245609983, +STORE, 140321220435968, 140321228824575, +SNULL, 140321228828671, 140321245609983, +STORE, 140321228824576, 140321228828671, +STORE, 140321228828672, 140321245609983, +STORE, 140320817778688, 140320834564095, +SNULL, 140320817782783, 140320834564095, +STORE, 140320817778688, 140320817782783, +STORE, 140320817782784, 140320834564095, +STORE, 140320582914048, 140320591306751, +SNULL, 140321228828672, 140321237217279, +STORE, 140321237217280, 140321245609983, +STORE, 140321228828672, 140321237217279, +SNULL, 140321237221375, 140321245609983, +STORE, 140321237217280, 140321237221375, +STORE, 140321237221376, 140321245609983, +SNULL, 140320448700415, 140320457089023, +STORE, 140320448696320, 140320448700415, +STORE, 140320448700416, 140320457089023, +SNULL, 140321245614080, 140321254002687, +STORE, 140321254002688, 140321262395391, +STORE, 140321245614080, 140321254002687, +SNULL, 140321254006783, 140321262395391, +STORE, 140321254002688, 140321254006783, +STORE, 140321254006784, 140321262395391, +STORE, 140320574521344, 140320591306751, +SNULL, 140320574525439, 140320591306751, +STORE, 140320574521344, 140320574525439, +STORE, 140320574525440, 140320591306751, +STORE, 140320566128640, 140320574521343, +SNULL, 140320566132735, 140320574521343, +STORE, 140320566128640, 140320566132735, +STORE, 140320566132736, 140320574521343, +SNULL, 140320574525440, 140320582914047, +STORE, 140320582914048, 140320591306751, +STORE, 140320574525440, 140320582914047, +SNULL, 140320582918143, 140320591306751, +STORE, 140320582914048, 140320582918143, +STORE, 140320582918144, 140320591306751, +STORE, 140320557735936, 140320566128639, +SNULL, 140320557740031, 140320566128639, +STORE, 140320557735936, 140320557740031, +STORE, 140320557740032, 140320566128639, +STORE, 140320549343232, 140320557735935, +STORE, 140320465481728, 140320473874431, +STORE, 140320448700416, 140320473874431, +SNULL, 140320834568192, 140320859742207, +STORE, 140320859742208, 140320868134911, +STORE, 140320834568192, 140320859742207, +SNULL, 140320859746303, 140320868134911, +STORE, 140320859742208, 140320859746303, +STORE, 140320859746304, 140320868134911, +STORE, 140320440303616, 140320448696319, +STORE, 140320431910912, 140320448696319, +SNULL, 140320834568192, 140320851349503, +STORE, 140320851349504, 140320859742207, +STORE, 140320834568192, 140320851349503, +SNULL, 140320851353599, 140320859742207, +STORE, 140320851349504, 140320851353599, +STORE, 140320851353600, 140320859742207, +SNULL, 140320817782784, 140320826171391, +STORE, 140320826171392, 140320834564095, +STORE, 140320817782784, 140320826171391, +SNULL, 140320826175487, 140320834564095, +STORE, 140320826171392, 140320826175487, +STORE, 140320826175488, 140320834564095, +SNULL, 140320834568192, 140320842956799, +STORE, 140320842956800, 140320851349503, +STORE, 140320834568192, 140320842956799, +SNULL, 140320842960895, 140320851349503, +STORE, 140320842956800, 140320842960895, +STORE, 140320842960896, 140320851349503, +STORE, 140320423518208, 140320448696319, +SNULL, 140320423522303, 140320448696319, +STORE, 140320423518208, 140320423522303, +STORE, 140320423522304, 140320448696319, +STORE, 140320415125504, 140320423518207, +STORE, 140320331264000, 140320339656703, +STORE, 140320322871296, 140320339656703, +STORE, 140320314478592, 140320339656703, +SNULL, 140320314482687, 140320339656703, +STORE, 140320314478592, 140320314482687, +STORE, 140320314482688, 140320339656703, +STORE, 140320306085888, 140320314478591, +SNULL, 140320306089983, 140320314478591, +STORE, 140320306085888, 140320306089983, +STORE, 140320306089984, 140320314478591, +STORE, 140320297693184, 140320306085887, +SNULL, 140320297697279, 140320306085887, +STORE, 140320297693184, 140320297697279, +STORE, 140320297697280, 140320306085887, +STORE, 140320289300480, 140320297693183, +STORE, 140320280907776, 140320297693183, +SNULL, 140320280911871, 140320297693183, +STORE, 140320280907776, 140320280911871, +STORE, 140320280911872, 140320297693183, +SNULL, 140320423522304, 140320431910911, +STORE, 140320431910912, 140320448696319, +STORE, 140320423522304, 140320431910911, +SNULL, 140320431915007, 140320448696319, +STORE, 140320431910912, 140320431915007, +STORE, 140320431915008, 140320448696319, +SNULL, 140320549347327, 140320557735935, +STORE, 140320549343232, 140320549347327, +STORE, 140320549347328, 140320557735935, +STORE, 140320272515072, 140320280907775, +SNULL, 140320448700416, 140320457089023, +STORE, 140320457089024, 140320473874431, +STORE, 140320448700416, 140320457089023, +SNULL, 140320457093119, 140320473874431, +STORE, 140320457089024, 140320457093119, +STORE, 140320457093120, 140320473874431, +STORE, 140320264122368, 140320280907775, +SNULL, 140320457093120, 140320465481727, +STORE, 140320465481728, 140320473874431, +STORE, 140320457093120, 140320465481727, +SNULL, 140320465485823, 140320473874431, +STORE, 140320465481728, 140320465485823, +STORE, 140320465485824, 140320473874431, +SNULL, 140320431915008, 140320440303615, +STORE, 140320440303616, 140320448696319, +STORE, 140320431915008, 140320440303615, +SNULL, 140320440307711, 140320448696319, +STORE, 140320440303616, 140320440307711, +STORE, 140320440307712, 140320448696319, +STORE, 140320255729664, 140320280907775, +STORE, 140320247336960, 140320280907775, +SNULL, 140320247341055, 140320280907775, +STORE, 140320247336960, 140320247341055, +STORE, 140320247341056, 140320280907775, +STORE, 140320238944256, 140320247336959, +STORE, 140320230551552, 140320247336959, +SNULL, 140320230551552, 140320238944255, +STORE, 140320238944256, 140320247336959, +STORE, 140320230551552, 140320238944255, +SNULL, 140320238948351, 140320247336959, +STORE, 140320238944256, 140320238948351, +STORE, 140320238948352, 140320247336959, +SNULL, 140320314482688, 140320331263999, +STORE, 140320331264000, 140320339656703, +STORE, 140320314482688, 140320331263999, +SNULL, 140320331268095, 140320339656703, +STORE, 140320331264000, 140320331268095, +STORE, 140320331268096, 140320339656703, +SNULL, 140320280911872, 140320289300479, +STORE, 140320289300480, 140320297693183, +STORE, 140320280911872, 140320289300479, +SNULL, 140320289304575, 140320297693183, +STORE, 140320289300480, 140320289304575, +STORE, 140320289304576, 140320297693183, +SNULL, 140320415129599, 140320423518207, +STORE, 140320415125504, 140320415129599, +STORE, 140320415129600, 140320423518207, +STORE, 140320222158848, 140320238944255, +STORE, 140320213766144, 140320238944255, +STORE, 140320205373440, 140320238944255, +SNULL, 140320205377535, 140320238944255, +STORE, 140320205373440, 140320205377535, +STORE, 140320205377536, 140320238944255, +SNULL, 140320314482688, 140320322871295, +STORE, 140320322871296, 140320331263999, +STORE, 140320314482688, 140320322871295, +SNULL, 140320322875391, 140320331263999, +STORE, 140320322871296, 140320322875391, +STORE, 140320322875392, 140320331263999, +SNULL, 140320247341056, 140320272515071, +STORE, 140320272515072, 140320280907775, +STORE, 140320247341056, 140320272515071, +SNULL, 140320272519167, 140320280907775, +STORE, 140320272515072, 140320272519167, +STORE, 140320272519168, 140320280907775, +SNULL, 140320247341056, 140320264122367, +STORE, 140320264122368, 140320272515071, +STORE, 140320247341056, 140320264122367, +SNULL, 140320264126463, 140320272515071, +STORE, 140320264122368, 140320264126463, +STORE, 140320264126464, 140320272515071, +SNULL, 140320205377536, 140320230551551, +STORE, 140320230551552, 140320238944255, +STORE, 140320205377536, 140320230551551, +SNULL, 140320230555647, 140320238944255, +STORE, 140320230551552, 140320230555647, +STORE, 140320230555648, 140320238944255, +STORE, 140320196980736, 140320205373439, +SNULL, 140320196984831, 140320205373439, +STORE, 140320196980736, 140320196984831, +STORE, 140320196984832, 140320205373439, +STORE, 140320188588032, 140320196980735, +SNULL, 140320247341056, 140320255729663, +STORE, 140320255729664, 140320264122367, +STORE, 140320247341056, 140320255729663, +SNULL, 140320255733759, 140320264122367, +STORE, 140320255729664, 140320255733759, +STORE, 140320255733760, 140320264122367, +STORE, 140320180195328, 140320196980735, +SNULL, 140320180199423, 140320196980735, +STORE, 140320180195328, 140320180199423, +STORE, 140320180199424, 140320196980735, +STORE, 140320171802624, 140320180195327, +STORE, 140320163409920, 140320180195327, +SNULL, 140320163414015, 140320180195327, +STORE, 140320163409920, 140320163414015, +STORE, 140320163414016, 140320180195327, +SNULL, 140320205377536, 140320222158847, +STORE, 140320222158848, 140320230551551, +STORE, 140320205377536, 140320222158847, +SNULL, 140320222162943, 140320230551551, +STORE, 140320222158848, 140320222162943, +STORE, 140320222162944, 140320230551551, +SNULL, 140320205377536, 140320213766143, +STORE, 140320213766144, 140320222158847, +STORE, 140320205377536, 140320213766143, +SNULL, 140320213770239, 140320222158847, +STORE, 140320213766144, 140320213770239, +STORE, 140320213770240, 140320222158847, +STORE, 140320155017216, 140320163409919, +SNULL, 140320180199424, 140320188588031, +STORE, 140320188588032, 140320196980735, +STORE, 140320180199424, 140320188588031, +SNULL, 140320188592127, 140320196980735, +STORE, 140320188588032, 140320188592127, +STORE, 140320188592128, 140320196980735, +SNULL, 140320155021311, 140320163409919, +STORE, 140320155017216, 140320155021311, +STORE, 140320155021312, 140320163409919, +SNULL, 140320163414016, 140320171802623, +STORE, 140320171802624, 140320180195327, +STORE, 140320163414016, 140320171802623, +SNULL, 140320171806719, 140320180195327, +STORE, 140320171802624, 140320171806719, +STORE, 140320171806720, 140320180195327, +STORE, 140320146624512, 140320155017215, +SNULL, 140320146628607, 140320155017215, +STORE, 140320146624512, 140320146628607, +STORE, 140320146628608, 140320155017215, +STORE, 140321937321984, 140321937350655, +STORE, 140321884942336, 140321887133695, +SNULL, 140321884942336, 140321885032447, +STORE, 140321885032448, 140321887133695, +STORE, 140321884942336, 140321885032447, +SNULL, 140321887125503, 140321887133695, +STORE, 140321885032448, 140321887125503, +STORE, 140321887125504, 140321887133695, +ERASE, 140321887125504, 140321887133695, +STORE, 140321887125504, 140321887133695, +SNULL, 140321887129599, 140321887133695, +STORE, 140321887125504, 140321887129599, +STORE, 140321887129600, 140321887133695, +ERASE, 140321937321984, 140321937350655, +ERASE, 140321086214144, 140321086218239, +ERASE, 140321086218240, 140321094606847, +ERASE, 140321119784960, 140321119789055, +ERASE, 140321119789056, 140321128177663, +ERASE, 140321245609984, 140321245614079, +ERASE, 140321245614080, 140321254002687, +ERASE, 140320574521344, 140320574525439, +ERASE, 140320574525440, 140320582914047, +ERASE, 140320297693184, 140320297697279, +ERASE, 140320297697280, 140320306085887, +ERASE, 140321354616832, 140321354620927, +ERASE, 140321354620928, 140321363009535, +ERASE, 140320834564096, 140320834568191, +ERASE, 140320834568192, 140320842956799, +ERASE, 140320591306752, 140320591310847, +ERASE, 140320591310848, 140320599699455, +ERASE, 140321136570368, 140321136574463, +ERASE, 140321136574464, 140321144963071, +ERASE, 140321237217280, 140321237221375, +ERASE, 140321237221376, 140321245609983, +ERASE, 140321363009536, 140321363013631, +ERASE, 140321363013632, 140321371402239, +ERASE, 140320599699456, 140320599703551, +ERASE, 140320599703552, 140320608092159, +ERASE, 140321396580352, 140321396584447, +ERASE, 140321396584448, 140321404973055, +ERASE, 140320566128640, 140320566132735, +ERASE, 140320566132736, 140320574521343, +ERASE, 140321094606848, 140321094610943, +ERASE, 140321094610944, 140321102999551, +ERASE, 140320582914048, 140320582918143, +ERASE, 140320582918144, 140320591306751, +ERASE, 140320289300480, 140320289304575, +ERASE, 140320289304576, 140320297693183, +ERASE, 140320163409920, 140320163414015, + }; + int cnt = 0; void *ptr = NULL; MA_STATE(mas, mt, 0, 0); @@ -30034,6 +31158,20 @@ ERASE, 140612699410432, 140612707799039, mt_validate(mt); mtree_destroy(mt); + mas_reset(&mas); + mtree_init(mt, MAPLE_ALLOC_RANGE); + check_erase2_testset(mt, set39, ARRAY_SIZE(set39)); + rcu_barrier(); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mtree_init(mt, MAPLE_ALLOC_RANGE); + check_erase2_testset(mt, set40, ARRAY_SIZE(set40)); + rcu_barrier(); + mt_validate(mt); + mtree_destroy(mt); + } @@ -30727,7 +31865,6 @@ static int maple_tree_seed(void) void *ptr = &set; pr_info("\nTEST STARTING\n\n"); - mtree_init(&tree, 0); check_new_node(&tree); mtree_destroy(&tree); diff --git a/mm/mmap.c b/mm/mmap.c index 10be7be153ae..0bac499f0d58 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -60,6 +60,7 @@ #define CREATE_TRACE_POINTS #include #define CONFIG_DEBUG_MAPLE_TREE +#define CONFIG_DEBUG_MAPLE_TREE_VERBOSE #define CONFIG_DEBUG_VM_RB 1 extern void mt_validate(struct maple_tree *mt); @@ -822,8 +823,8 @@ static void __vma_link_file(struct vm_area_struct *vma) } static void __vma_mt_erase(struct mm_struct *mm, struct vm_area_struct *vma) { -#if defined(CONFIG_DEBUG_MAPLE_TREE_VERBOSE) trace___vma_mt_erase(mm, vma); +#if defined(CONFIG_DEBUG_MAPLE_TREE_VERBOSE) printk("mt_mod %px, (%px), ERASE, %lu, %lu,", mm, vma, vma->vm_start, vma->vm_end - 1); #endif @@ -833,8 +834,8 @@ static void __vma_mt_erase(struct mm_struct *mm, struct vm_area_struct *vma) static void __vma_mt_szero(struct mm_struct *mm, unsigned long start, unsigned long end) { -#if defined(CONFIG_DEBUG_MAPLE_TREE_VERBOSE) trace___vma_mt_szero(mm, start, end); +#if defined(CONFIG_DEBUG_MAPLE_TREE_VERBOSE) printk("mt_mod %px, (%px), SNULL, %lu, %lu,", mm, NULL, start, end - 1); #endif @@ -842,8 +843,8 @@ static void __vma_mt_szero(struct mm_struct *mm, unsigned long start, } static void __vma_mt_store(struct mm_struct *mm, struct vm_area_struct *vma) { -#if defined(CONFIG_DEBUG_MAPLE_TREE_VERBOSE) trace___vma_mt_store(mm, vma); +#if defined(CONFIG_DEBUG_MAPLE_TREE_VERBOSE) printk("mt_mod %px, (%px), STORE, %lu, %lu,", mm, vma, vma->vm_start, vma->vm_end - 1); #endif @@ -3597,6 +3598,7 @@ void exit_mmap(struct mm_struct *mm) } mtree_destroy(&mm->mm_mt); + trace_exit_mmap(mm); #if defined(CONFIG_DEBUG_MAPLE_TREE_VERBOSE) printk("mt_mod %px, (%px), DESTROY", mm, &mm->mm_mt); #endif