From: Liam R. Howlett Date: Wed, 2 Jan 2019 18:31:54 +0000 (-0500) Subject: maple_tree: Implement expanding root. X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=abebca429edc00e65c6d9014eda1a60cd40314d0;p=users%2Fjedix%2Flinux-maple.git maple_tree: Implement expanding root. Signed-off-by: Liam R. Howlett --- diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 3c04426d9f59..8fd4c0ebbb7b 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -189,9 +189,9 @@ struct ma_state { /* * Special values for ma_state.node. * MAS_START means we have not searched the tree. - * MAS_ROOT means we have searched the tree and the entry we found lives - * in the root of the tree (ie it has index 0, length 1 and is the only - * entry in the tree). + * MAS_ROOT means we have searched the tree and the entry we found leaves in + * the root of the tree (ie it has index 0, length 1 and is the only entry in + * the tree). * MAS_NONE means we have searched the tree and there is no node in the * tree for this entry. For example, we searched for index 1 in an empty * tree. Or we have a tree which points to a full leaf node and we diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 304a6c067bf8..a900ffea13f8 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -7,8 +7,11 @@ */ #include +#include #include +#define MA_ROOT_PARENT 1 + static struct kmem_cache *maple_node_cache; static struct maple_node *mt_alloc_one(gfp_t gfp) @@ -43,6 +46,10 @@ static inline bool mt_is_reserved(const void *entry) return ((unsigned long)entry < 4096) && xa_is_internal(entry); } +static inline void mas_set_err(struct ma_state *mas, long err) +{ + mas->node = MA_ERROR(err); +} static inline bool mas_is_start(struct ma_state *ms) { return ms->node == MAS_START; @@ -61,44 +68,413 @@ void *mt_mk_node(const struct maple_node *node, enum maple_type type) return (void *)((unsigned long)node | (type << 3) | 2); } -void mtree_init(struct maple_tree *mt) +static inline bool mas_is_err(struct ma_state *mas) { - spin_lock_init(&mt->ma_lock); - mt->ma_flags = 0; - mt->ma_root = NULL; + return xa_is_err(mas->node); } -void *mtree_load(struct maple_tree *mt, unsigned long index) +static inline struct maple_node *ma_get_alloc(const struct ma_state *ms) { - return NULL; + return (struct maple_node *)((unsigned long)ms->alloc & ~0x7F); } -int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry, gfp_t gfp) +static inline int ma_get_alloc_cnt(const struct ma_state *ms) { - return -EINVAL; + struct maple_node *node = ma_get_alloc(ms); + + if (!node) + return 0; + if (!node->slot[0]) + return 1; + if (!node->slot[1]) + return 2; + return 3; } -int mtree_insert_range(struct maple_tree *mt, unsigned long first, - unsigned long last, void *entry, gfp_t gfp) +static inline void ma_set_alloc_req(struct ma_state *ms, int count) { - return -EINVAL; + ms->alloc = (struct maple_node *)((unsigned long)ms->alloc & ~0x03); + ms->alloc = (struct maple_node *)((unsigned long)ms->alloc | count); } -int mtree_erase(struct maple_tree *mt, unsigned long index) +static inline int ma_get_alloc_req(const struct ma_state *ms) { - return -EINVAL; + return (int)(((unsigned long)ms->alloc & 0x03)); } -void mtree_destroy(struct maple_tree *mt) +static inline bool ma_is_root(struct maple_node *node) +{ + if (((unsigned long)node->parent & 1) == 1) + return true; + return false; +} +static inline struct maple_node *ma_next_alloc(struct ma_state *ms) +{ + int cnt; + struct maple_node *mn, *smn; + + if (!ms->alloc) + return NULL; + + cnt = ma_get_alloc_cnt(ms); + mn = ma_get_alloc(ms); + cnt--; + if (cnt == 0) { + ms->alloc = NULL; + } else { + smn = (struct maple_node *)mn->slot[cnt - 1]; + mn->slot[cnt - 1] = NULL; + mn = smn; + } + + return mn; +} +static void ma_new_node(struct ma_state *ms, gfp_t gfp) +{ + struct maple_node *mn, *smn; + int req = ma_get_alloc_req(ms); + int allocated = ma_get_alloc_cnt(ms); + int slot; + + if (!req) + return; + + mn = ma_get_alloc(ms); + if (!mn) { + mn = mt_alloc_one(gfp); + if (!mn) + goto list_failed; + printk("Node = %p\n", mn); + req--; + allocated++; + } + + slot = allocated - 1; + while (req > 0) { + smn = mt_alloc_one(gfp); + if (!smn) + goto slot_failed; + smn->parent = NULL; + mn->slot[slot++] = smn; + req--; + allocated++; + } + +slot_failed: + ms->alloc = mn; + ma_set_alloc_req(ms, req); + +list_failed: + if (req > 0) + mas_set_err(ms, -ENOMEM); +} + +/* Private + * Check if there was an error allocating and do the allocation if necessary + * If there are allocations, then free them. + */ +bool mas_nomem(struct ma_state *ms, gfp_t gfp) + __must_hold(ms->tree->lock) +{ + if (ms->node != MA_ERROR(-ENOMEM)) { + struct maple_node *node = ma_get_alloc(ms); + + if (node) { + kfree(node->slot[1]); + kfree(node->slot[0]); + kfree(node); + } + ms->alloc = NULL; + return false; + } + + if (gfpflags_allow_blocking(gfp)) { + mtree_unlock(ms->tree); + ma_new_node(ms, gfp); + mtree_lock(ms->tree); + } else { + ma_new_node(ms, gfp); + } + if (!ma_get_alloc(ms)) + return false; + ms->node = MAS_START; + return true; +} +static struct maple_node *mas_node_cnt(struct ma_state *ms, int count) +{ + int allocated = ma_get_alloc_cnt(ms); + + BUG_ON(count > 3); + + if (allocated < count) { + ma_set_alloc_req(ms, count - allocated); + ma_new_node(ms, GFP_NOWAIT | __GFP_NOWARN); + } + return ms->alloc; +} + +static void *mas_start(struct ma_state *ms) +{ + void *entry; + + if (mas_is_err(ms)) + return NULL; + + if (ms->node == MAS_START) { + entry = ms->tree->ma_root; + if (!xa_is_node(entry)) { + if (ms->index > 0) + return NULL; + ms->node = MAS_ROOT; + } + } else { + entry = mt_mk_node(ms->node, maple_range_64); + } + + return entry; +} + +short ma_data_end_r64(const struct ma_state *ms) +{ + struct maple_range_64 *mr64 = &ms->node->mr64; + short data_end = 0; + + for (data_end = 0; data_end < MAPLE_RANGE64_SLOTS - 1; data_end++) { + if (mr64->pivot[data_end] == 0 || + mr64->pivot[data_end] == ms->max) + return data_end; + } + + if (mr64->slot[data_end+1] != NULL) + data_end++; + + return data_end; +} +static void ma_append(struct ma_state *ms, void *entry) +{ + short idx = ma_data_end_r64(ms) + 2; + struct maple_range_64 *mr64 = &ms->node->mr64; + + if ((ms->index - 1 != mr64->pivot[idx - 1]) || + (ms->index != ms->last)) + idx++; + + if (idx >= MAPLE_RANGE64_SLOTS - 1) + printk("Split this node\n"); + + /* zero the new end */ + mr64->pivot[idx] = 0; + RCU_INIT_POINTER(mr64->slot[idx], NULL); + idx--; + /* Set the entry value */ + RCU_INIT_POINTER(mr64->slot[idx], entry); + mr64->pivot[idx] = ms->last; + idx--; + + /* Create a NULL gap, if necessary */ + if ((ms->index - 1 != mr64->pivot[idx - 1]) || + (ms->index != ms->last)) { + RCU_INIT_POINTER(mr64->slot[idx], NULL); + mr64->pivot[idx] = ms->index - 1; + } +} +static void ma_root_expand(struct ma_state *ms, void *entry) +{ + void *r_entry = ms->tree->ma_root; // root entry + struct maple_node *mn; + + mas_node_cnt(ms, 1); + if (mas_is_err(ms)) + return; + + mn = ma_next_alloc(ms); + ms->node = mn; + mn->parent = (struct maple_node*) + ((unsigned long)ms->tree | MA_ROOT_PARENT); + + + /* + * Insert the existing entry into the new node + * rcu_assign is not necessary as readers are not able to access this + * node. + */ + if (r_entry != NULL) { + RCU_INIT_POINTER(mn->mr64.slot[0], r_entry); + mn->mr64.pivot[0] = 0; + } + ma_append(ms, entry); + + + mn->parent = NULL; + /* swap the new root into the tree */ + rcu_assign_pointer(ms->tree->ma_root, mt_mk_node(mn, maple_leaf_64)); +} + +static void ms_update_limits(struct ma_state *ms, unsigned char slot) +{ + struct maple_range_64 *mr64; + if (slot >= MAPLE_NODE_SLOTS) + return; + + if (mas_is_start(ms)) + return; + + mr64 = &ms->node->mr64; + + if (slot > 0) + ms->min = mr64->pivot[slot - 1] + 1; + + if (slot < MAPLE_RANGE64_SLOTS - 1) + ms->max = mr64->pivot[slot]; +} +static unsigned char mas_walk_r64(struct ma_state *ms, unsigned long val) +{ + struct maple_range_64 *mr64 = &ms->node->mr64; + int i = 0; + + do { + if (i >= MAPLE_RANGE64_SLOTS - 1) + break; // Right-most child. + if (val <= mr64->pivot[i]) + break; + } while (i++ < MAPLE_RANGE64_SLOTS - 1); + + ms_update_limits(ms, i); + ms->node = rcu_dereference(mr64->slot[i]); + return i; +} +void *ma_insert(struct ma_state *ms, void *entry) +{ + void *e_entry = mas_start(ms); // existing entry; + void *p_entry; // Previous entry. + char slot = MAPLE_NODE_SLOTS; + + if (!xa_is_node(e_entry)) { + if (ms->last == 0) { + if (e_entry != NULL) + goto exists; + rcu_assign_pointer(ms->tree->ma_root, entry); + return NULL; + } + ma_root_expand(ms, entry); + return NULL; + } + + /* Find the correct node */ + do { + ms->node = mt_to_node(e_entry); + p_entry = ms->node; + slot = mas_walk_r64(ms, ms->index); + e_entry = ms->node; + } while (xa_is_node(e_entry)); + + if (e_entry) + goto exists; + + /* Do the Insert */ + printk("Inserting %lu\n", ms->index); + + return NULL; +exists: + mas_set_err(ms, -EEXIST); + return NULL; + +} + +/* + * Private + * + * Must hold rcu_read_lock or the write lock. + * + * Find where ms->index is located and return the entry. + * ms->node will point to the node containing the entry. + * + * + */ +void *mas_walk(struct ma_state *ms) { + void *entry = mas_start(ms); + + /* Outside this nodes range, it doesn't exist. */ + if (ms->min > ms->index || + ms->max < ms->index) + return NULL; // FIXME: Retry? + + while (xa_is_node(entry)) { + ms->node = mt_to_node(entry); + mas_walk_r64(ms, ms->index); + entry = ms->node; + } + return entry; } +/* Interface */ void __init maple_tree_init(void) { maple_node_cache = kmem_cache_create("maple node", sizeof(struct maple_node), sizeof(struct maple_node), SLAB_PANIC | SLAB_RECLAIM_ACCOUNT, NULL); } +void mtree_init(struct maple_tree *mt) +{ + spin_lock_init(&mt->ma_lock); + mt->ma_flags = 0; + mt->ma_root = NULL; +} + +EXPORT_SYMBOL(mtree_init); + +void *mtree_load(struct maple_tree *mt, unsigned long index) +{ + void *entry; + MA_STATE(ms, mt, index, index); + rcu_read_lock(); + entry = mas_walk(&ms); + rcu_read_unlock(); + return entry; +} +EXPORT_SYMBOL(mtree_load); +int mtree_insert_range(struct maple_tree *mt, unsigned long first, + unsigned long last, void *entry, gfp_t gfp) +{ + int ret = 0; + + MA_STATE(ms, mt, first, last); + + if (WARN_ON_ONCE(xa_is_internal(entry))) + return -EINVAL; + + if (first > last) + return -EINVAL; + + mtree_lock(ms.tree); +retry: + ma_insert(&ms, entry); + if (mas_nomem(&ms, gfp)) + goto retry; + + mtree_unlock(ms.tree); + if (mas_is_err(&ms)) + return xa_err(ms.node); + + return ret; +} +EXPORT_SYMBOL(mtree_insert_range); +int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry, + gfp_t gfp) +{ + return mtree_insert_range(mt, index, index, entry, gfp); +} +EXPORT_SYMBOL(mtree_insert); +int mtree_erase(struct maple_tree *mt, unsigned long index) +{ + return -EINVAL; +} + +void mtree_destroy(struct maple_tree *mt) +{ +} + #ifdef MT_DEBUG void mt_dump_range(unsigned long min, unsigned long max) @@ -146,7 +522,6 @@ void mt_dump_range64(void *entry, unsigned long min, unsigned long max, last = node->pivot[i]; if (last == 0 && i > 0) break; - if (leaf) mt_dump_entry(node->slot[i], first, last); else @@ -189,6 +564,7 @@ void mt_dump_node(void *entry, unsigned long min, unsigned long max, default: pr_cont(" UNKNOWN TYPE\n"); } + pr_info("dumped node %p\n", entry); } unsigned long mt_max[] = { diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c index db202646d09e..6b7471e283db 100644 --- a/lib/test_maple_tree.c +++ b/lib/test_maple_tree.c @@ -225,14 +225,14 @@ static int maple_tree_seed(void) { unsigned long set[] = {15, 14, 17, 25, 1000, 1001, 1002, 1003, 1005, 0, - 3}; + 3, 2}; unsigned long r[] = {10, 15, 20, 25, 22}; // For range testing void *ptr = &set; mtree_init(&tree); - check_new_node(&tree); - check_double_insert_leaf(&tree); + //check_new_node(&tree); + //check_double_insert_leaf(&tree); check_load(&tree, set[0], NULL); // See if 15 -> NULL check_insert(&tree, set[9], &tree); // Insert 0 @@ -242,6 +242,8 @@ static int maple_tree_seed(void) check_insert(&tree, set[10], ptr); // Insert 3 check_load(&tree, set[9], &tree); // See if 0 -> &tree + check_load(&tree, set[11], NULL); // See if 2 -> NULL + check_load(&tree, set[10], ptr); // See if 3 -> ptr /* Clear out the tree */ @@ -257,9 +259,15 @@ static int maple_tree_seed(void) * a second value, then loads the value again */ check_load(&tree, set[1], NULL); // See if 14 -> NULL + printk("%s: %d\n", __func__, __LINE__); + mt_dump(&tree); check_insert(&tree, set[1], ptr); // insert 14 -> ptr + mt_dump(&tree); + printk("%s: %d\n", __func__, __LINE__); check_load(&tree, set[1], ptr); // See if 14 -> ptr + printk("%s: %d\n", __func__, __LINE__); check_load(&tree, set[0], &tree); // See if 15 -> &tree + printk("%s: %d\n", __func__, __LINE__); /* Tree currently contains: * p[0]: 14 -> (nil) p[1]: 15 -> ptr p[2]: 16 -> &tree p[3]: 0 -> (nil)