maple_tree: Rework walking the tree to combine insert/walk/erase

functions.

This turned out to be a huge deal with many issues resolved for the
following of Thoughts:

1. Nodes can use the last 0x7F bits of the address, so clear them and
such.
2. Root needs to have bit 1 (0x2) set, so be safe around that.
3. mas->node should be encoded (except root needs to remove bit 1 to
avoid being seen as an error).
4. xa_is_node no longer works, so remove BUG_ONs.  Take care that we
know we have a value when we have a slot & an leaf node in mas->node.
5. Store slots in the same place as the alloc requests as their uses do
not overlap.
6. mas_start needs to return a safe root instead of making its own node.
7. Insert was basically rewritten
8. Verify that we are no inserting reserved values
9. Change testcases to not try to insert reserved values.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>

diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index 8fd4c0ebbb7bb4271735f19334daf2581130f872..8176228b60f454a35b8f07a87388cbb640a96dd9 100644 (file)
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -189,7 +189,7 @@ struct ma_state {
 /*
  * Special values for ma_state.node.
  * MAS_START means we have not searched the tree.
- * MAS_ROOT means we have searched the tree and the entry we found leaves in
+ * MAS_ROOT means we have searched the tree and the entry we found lives in
  * the root of the tree (ie it has index 0, length 1 and is the only entry in
  * the tree).
  * MAS_NONE means we have searched the tree and there is no node in the

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index de7a6b2657fe806a6a4fcd26fce45b39968b3d22..ca7d4245a8901f6114e0bef7da1972e1c2c54d70 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -58,15 +58,23 @@ static inline bool mas_is_start(struct ma_state *ms)
 
 static inline struct maple_node *mt_to_node(const void *entry)
 {
-       BUG_ON(!xa_is_node(entry));
        return (struct maple_node *)((unsigned long)entry & ~127);
 }
 
 static inline
 void *mt_mk_node(const struct maple_node *node, enum maple_type type)
 {
-       BUG_ON(xa_is_node(node));
-       return (void *)((unsigned long)node | (type << 3) | 2);
+       return (void *)((unsigned long)node | (type << 3));
+}
+
+static inline struct maple_node *mt_safe_root(const void *entry)
+{
+       return (struct maple_node *)((unsigned long)entry & ~2);
+}
+static inline
+void *mt_mk_root(const struct maple_node *node)
+{
+       return (void *)((unsigned long)node | 2);
 }
 
 static inline bool mas_is_err(struct ma_state *mas)
@@ -144,18 +152,27 @@ static inline int ma_get_alloc_cnt(const struct ma_state *ms)
 
 static inline void ma_set_alloc_req(struct ma_state *ms, int count)
 {
-       ms->alloc = (struct maple_node *)((unsigned long)ms->alloc & ~0x03);
+       ms->alloc = (struct maple_node *)((unsigned long)ms->alloc & ~0x7F);
        ms->alloc = (struct maple_node *)((unsigned long)ms->alloc | count);
 }
 
 static inline int ma_get_alloc_req(const struct ma_state *ms)
 {
-       return (int)(((unsigned long)ms->alloc & 0x03));
+       return (int)(((unsigned long)ms->alloc & 0x7F));
+}
+
+static inline int ma_get_slot(const struct ma_state *ms)
+{
+       return ma_get_alloc_req(ms);
 }
 
+static inline void ma_set_slot(struct ma_state *ms, int slot)
+{
+       ma_set_alloc_req(ms, slot);
+}
 static inline bool ma_is_root(struct maple_node *node)
 {
-       if (((unsigned long)node->parent & 1) == 1)
+       if (((unsigned long)mt_to_node(node)->parent & 1) == 1)
                return true;
        return false;
 }
@@ -276,9 +293,12 @@ static void *mas_start(struct ma_state *ms)
                        if (ms->index > 0)
                                return NULL;
                        ms->node = MAS_ROOT;
+                       ma_set_slot(ms, MAPLE_NODE_SLOTS);
+               } else {
+                       entry = mt_safe_root(entry);
                }
        } else {
-               entry = mt_mk_node(ms->node, maple_range_64);
+               entry = ms->node;
        }
 
        return entry;
@@ -286,7 +306,7 @@ static void *mas_start(struct ma_state *ms)
 
 unsigned char ma_data_end_r64(const struct ma_state *ms)
 {
-       struct maple_range_64 *mr64 = &ms->node->mr64;
+       struct maple_range_64 *mr64 = &mt_to_node(ms->node)->mr64;
        unsigned char data_end = 0;
        unsigned long last = ms->max;
 
@@ -301,57 +321,73 @@ unsigned char ma_data_end_r64(const struct ma_state *ms)
 
        return data_end;
 }
+/* Private
+ *
+ * Insert entry into a node.
+ *
+ * This is done by:
+ * 1. Calculating the range of slot.
+ * 2. Figure out how many slots are needed for the entry. (0, 1, 2)
+ * 3. Copy the data over
+ * 4. Write the entry.
+ *
+ */
 static int _ma_insert(struct ma_state *ms, void *entry, unsigned char slot)
 {
-       unsigned char added = 1;
-       unsigned char o_end = ma_data_end_r64(ms);
-       unsigned char n_end = o_end;
+       struct maple_range_64 *mr64 = &mt_to_node(ms->node)->mr64;
+       int o_end = ma_data_end_r64(ms); // Old end
+       int n_end = o_end;               // New end
        unsigned long max = ms->max;
-       struct maple_range_64 *mr64 = &ms->node->mr64;
+       unsigned long min = ms->min;
+
+       /* Calculate the range of the slot */
+       if (slot < MAPLE_RANGE64_SLOTS - 1 &&  mr64->pivot[slot] != 0)
+               max = mr64->pivot[slot];
 
-       if (o_end < MAPLE_RANGE64_SLOTS - 1)
-               max = mr64->pivot[o_end - 1];
+       if (slot == MAPLE_NODE_SLOTS)
+               slot = o_end;
 
-       if ((ms->index - 1 != max) ||
-          (ms->index != ms->last)) {
-               added++;
+       if (slot > 0)
+               min = mr64->pivot[slot - 1];
+
+       /* Figure out how many slots are needed for the entry. */
+       if (max != ms->last)
+               n_end++;
+
+       if (min != ms->index - 1)
                n_end++;
-       }
 
-       if (n_end >= MAPLE_RANGE64_SLOTS && ms->max != ms->last)
-               pr_info("\n\nSplit this node\n");
 
-       if (o_end == slot) { // Appending
-               /* zero the new end if the pivot exists. */
+       /* Copy the data over */
+       while (o_end >= slot) {
+               RCU_INIT_POINTER(mr64->slot[n_end],
+                               mr64->slot[o_end]);
                if (n_end < MAPLE_RANGE64_SLOTS - 2)
-                       mr64->pivot[n_end + 1] = 0;
-       } else {
-               while (o_end > slot) {
-                       RCU_INIT_POINTER(mr64->slot[n_end],
-                                        mr64->slot[o_end]);
-                       if (n_end < MAPLE_RANGE64_SLOTS - 2)
-                               mr64->pivot[n_end] = mr64->pivot[o_end];
-                       n_end--;
-                       o_end--;
-               }
+                       mr64->pivot[n_end] = mr64->pivot[o_end];
+               n_end--;
+               o_end--;
        }
 
+       // Overwrite the last pivot value
+       if (mr64->pivot[slot] == ms->index)
+               n_end++;
+
+       /* Write the entry */
+
        RCU_INIT_POINTER(mr64->slot[n_end], entry);
-       if ((ms->index - 1 != mr64->pivot[n_end - 1]) ||
-                       (ms->index != ms->last)) {
-               /* Set the entry value */
+
+       if (min != ms->index - 1) {
                mr64->pivot[n_end--] = ms->last;
-               /* Create a NULL gap */
                RCU_INIT_POINTER(mr64->slot[n_end], NULL);
                wmb(); /* slot needs to be written before readers can see. */
                mr64->pivot[n_end] = ms->index - 1;
-       } else {
-               wmb(); /* slot needs to be written before readers can see. */
-               /* Set the entry value */
-               mr64->pivot[n_end] = ms->last;
+               return 2;
        }
-       return added;
+       wmb(); /* slot needs to be written before readers can see. */
+       mr64->pivot[n_end] = ms->last;
+       return 1;
 }
+
 static unsigned char ma_append(struct ma_state *ms, void *entry)
 {
        return _ma_insert(ms, entry, ma_data_end_r64(ms));
@@ -367,7 +403,7 @@ static void ma_root_expand(struct ma_state *ms, void *entry)
                return;
 
        mn = ma_next_alloc(ms);
-       ms->node = mn;
+       ms->node = mt_mk_node(mn, maple_leaf_64);
        mn->parent = (struct maple_node*)
                      ((unsigned long)ms->tree | MA_ROOT_PARENT);
 
@@ -378,12 +414,13 @@ static void ma_root_expand(struct ma_state *ms, void *entry)
 
        ma_append(ms, entry);
        /* swap the new root into the tree */
-       rcu_assign_pointer(ms->tree->ma_root, mt_mk_node(mn, maple_leaf_64));
+       rcu_assign_pointer(ms->tree->ma_root, mt_mk_root(ms->node));
 }
 
 static void mas_update_limits(struct ma_state *ms, unsigned char slot)
 {
        struct maple_range_64 *mr64;
+
        if (slot >= MAPLE_NODE_SLOTS)
                return;
 
@@ -404,7 +441,7 @@ static void mas_update_limits(struct ma_state *ms, unsigned char slot)
  */
 static void *mas_coalesce(struct ma_state *mas)
 {
-       struct maple_range_64 *src = &mas->node->mr64;
+       struct maple_range_64 *src = &mt_to_node(mas->node)->mr64;
        unsigned long last = mas->max;
        struct maple_range_64 *dst;
        struct maple_node *mn;
@@ -451,78 +488,91 @@ static void ma_adopt_children(struct maple_node *parent)
        }
 }
 
-static unsigned char mt_copy_mr64(struct maple_range_64 *dst,
-               unsigned char d_slot, struct maple_range_64 *src,
-               unsigned char s_slot, unsigned char cnt)
+static bool mas_search_slots(struct ma_state *ms, unsigned long val)
 {
-       BUG_ON(cnt >= MAPLE_RANGE64_SLOTS);
-       for (; s_slot < cnt; s_slot++) {
-               if (d_slot > 0 &&
-                   dst->pivot[d_slot - 1] == src->pivot[s_slot]) {
-                       s_slot++;
-               }
-               dst->pivot[d_slot] = src->pivot[s_slot];
-
-               if (s_slot != 0 && src->pivot[s_slot] == 0)
-                       return d_slot;
-
-               if (s_slot > 0 &&
-                   src->slot[s_slot] == NULL &&
-                   src->slot[s_slot - 1] == NULL) {
-                       dst->pivot[--d_slot] = src->pivot[s_slot];
-                       continue;
-               }
-               RCU_INIT_POINTER(dst->slot[d_slot], src->slot[s_slot]);
-               d_slot++;
-       }
-       return d_slot;
-}
-static unsigned char mas_walk_r64(struct ma_state *ms, unsigned long val)
-{
-       struct maple_range_64 *mr64 = &ms->node->mr64;
+       struct maple_range_64 *mr64;
        int i = 0;
 
-
+       mr64 = &mt_to_node(ms->node)->mr64;
        for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) {
-               if (i != 0 && mr64->pivot[i] == 0)
-                       goto not_found;
+               if (i != 0 && mr64->pivot[i] == 0) {
+                       ma_set_slot(ms, MAPLE_NODE_SLOTS);
+                       return false;
+               }
 
                if (val <= mr64->pivot[i])
                        break;
        }
+
        if (i == MAPLE_RANGE64_SLOTS - 2) {
-               // Right-most child.
-               if (ms->max != val)
-                       goto not_found;
-
-               ms->node = rcu_dereference(mr64->slot[++i]);
-               if (!ms->node)
-                       goto not_found;
-               return i;
+               if (val <= mr64->pivot[i])
+                       i++;
        }
 
-       ms->node = rcu_dereference(mr64->slot[i]);
-       return i;
+       ma_set_slot(ms, i);
+       return true;
+}
+void mas_traverse(struct ma_state *mas)
+{
+       unsigned char slot = ma_get_slot(mas);
+
+       mas->node = mt_to_node(mas->node)->slot[slot];
+       mas_update_limits(mas, slot);
+}
+
+bool _mas_walk(struct ma_state *mas)
+{
+       void *p_entry; // Previous entry.
+
+       mas->node = mas_start(mas);
 
-not_found:
-       mas_set_err(ms, -EBADSLT);
-       return i;
+       if (mt_is_leaf(mas->node)) {
+               mas_search_slots(mas, mas->index);
+               return true;
+       }
+
+       do {
+               p_entry = mas->node;
+               if (!mas_search_slots(mas, mas->index)) {
+                       mas->node = p_entry;
+                       return mt_is_leaf(mas->node);
+               }
+               mas_traverse(mas);
+       } while (!mt_is_leaf(mas->node));
 
+       return true;
 }
-void mt_dump_node(void *entry, unsigned long min, unsigned long max,
-               unsigned int depth);
 
+/* Private
+ * We also reserve values with the bottom two bits set to '10' which are
+ * below 4096
+ */
+bool ma_reserved(void *entry)
+{
+       unsigned long ul_entry = (unsigned long)entry;
+
+       if ((ul_entry < 4096) && ((ul_entry & 3) == 2))
+               return true;
+
+       return false;
+}
 void *ma_insert(struct ma_state *mas, void *entry)
 {
-       void *e_entry = mas_start(mas); // existing entry;
        void *p_entry; // Previous entry.
        unsigned char slot = MAPLE_NODE_SLOTS;
        struct maple_node *mn;
        bool leaf;
 
-       if (!xa_is_node(e_entry)) {
+
+       if (ma_reserved(entry))
+               mas_set_err(mas, -EINVAL);
+
+       mas->node = mas_start(mas);
+
+
+       if (!xa_is_node(mas->tree->ma_root)) {
                if (mas->last == 0) {
-                       if (e_entry != NULL)
+                       if (mas->node != NULL)
                                goto exists;
                        rcu_assign_pointer(mas->tree->ma_root, entry);
                        return NULL;
@@ -531,36 +581,26 @@ void *ma_insert(struct ma_state *mas, void *entry)
                return NULL;
        }
 
-       /* Find the correct node */
-       do {
-               mas->node = mt_to_node(e_entry);
-               leaf = mt_is_leaf(e_entry);
-               p_entry = mas->node;
-               mas_update_limits(mas, slot);
-               slot = mas_walk_r64(mas, mas->index);
-               if (mas_is_err(mas)) {
-                       /* Ran off the end of the node. */
-                       mas->node = p_entry;
-                       ma_append(mas, entry);
-                       return NULL;
+       leaf = _mas_walk(mas);
+       slot = ma_get_slot(mas);
+       if (leaf == true && slot != MAPLE_NODE_SLOTS) {
+               if (mt_to_node(mas->node)->slot[slot]) {
+                       mas_set_err(mas, -EEXIST);
+                       goto exists;
                }
-               e_entry = mas->node;
-       } while (xa_is_node(e_entry));
-
-
-       if (e_entry)
-               goto exists;
+       }
 
-       mas->node = p_entry;
        mn = mas_coalesce(mas);
        if (mas_is_err(mas))
-               return NULL;
+               goto error;
+
+       p_entry = mas->node;
 
-       mas->node = mn;
+       mas->node = mt_mk_node(mn, maple_leaf_64);
        /* Do the insert */
        _ma_insert(mas, entry, slot);
        if (mas_is_err(mas))
-               return NULL;
+               goto error;
 
        /* Insert into tree */
        mn->parent = mas->node->parent;
@@ -568,21 +608,21 @@ void *ma_insert(struct ma_state *mas, void *entry)
                ma_adopt_children(mn);
 
        if (ma_is_root(p_entry)) {
-               rcu_assign_pointer(mas->tree->ma_root,
-                                  mt_mk_node(mn, maple_leaf_64));
+               mn->parent = (struct maple_node *)
+                             ((unsigned long)mas->tree | MA_ROOT_PARENT);
+               rcu_assign_pointer(mas->tree->ma_root, mt_mk_root(mas->node));
        } else {
                struct maple_node *parent = mt_parent(mn->parent);
 
                slot = mt_parent_slot(mn);
-               RCU_INIT_POINTER(parent->slot[slot],
-                                mt_mk_node(mn, maple_leaf_64));
+               RCU_INIT_POINTER(parent->slot[slot], mas->node);
        }
-       mt_free(p_entry);
+       mt_free(mt_to_node(p_entry));
        return NULL;
+
+error:
 exists:
-       mas_set_err(mas, -EEXIST);
        return NULL;
-
 }
 
 /*
@@ -595,27 +635,46 @@ exists:
  *
  *
  */
-void *mas_walk(struct ma_state *ms)
+void *mas_walk(struct ma_state *mas)
 {
-       void *entry = mas_start(ms);
+       void *entry = NULL;
        unsigned char slot = MAPLE_NODE_SLOTS;
+       bool leaf = false;
+
+       if (mas->tree->ma_root == NULL)
+               return NULL;
+
+       if (!xa_is_node(mas->tree->ma_root)) {
+               if (mas->last == 0)
+                       return mas->tree->ma_root;
+               return NULL;
+       }
 
        /* Outside this nodes range, it doesn't exist. */
-       if (ms->min > ms->index ||
-           ms->max < ms->index)
+       if (mas->min > mas->index ||
+           mas->max < mas->index)
                return NULL; // FIXME: Retry?
 
-       while (xa_is_node(entry)) {
-               ms->node = mt_to_node(entry);
-               mas_update_limits(ms, slot);
-               slot = mas_walk_r64(ms, ms->index);
-               if (mas_is_err(ms))
-                       return NULL;
-               entry = ms->node;
+       leaf = _mas_walk(mas);
+       slot = ma_get_slot(mas);
+       if (leaf == true && slot != MAPLE_NODE_SLOTS) {
+               struct maple_range_64 mr = mt_to_node(mas->node)->mr64;
+
+               entry = mr.slot[ma_get_slot(mas)];
        }
+       ma_set_slot(mas, 0);
        return entry;
 }
 
+/* Private
+ * Find an entry and erase the entire range
+ */
+int ma_erase(struct ma_state *mas)
+{
+       //bool leaf = _mas_walk(mas);
+       return -EINVAL;
+}
+
 void ma_destroy_walk(struct ma_state *mas)
 {
        struct maple_node *mn = mas->node;
@@ -697,7 +756,14 @@ int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry,
 EXPORT_SYMBOL(mtree_insert);
 int mtree_erase(struct maple_tree *mt, unsigned long index)
 {
-       return -EINVAL;
+       int ret = -EINVAL;
+
+       MA_STATE(mas, mt, index, index);
+
+       mtree_lock(mt);
+       ret = ma_erase(&mas);
+       mtree_unlock(mt);
+       return ret;
 }
 
 void mtree_destroy(struct maple_tree *mt)

diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c
index d364a41fb4b44fb8ddf351395ac205eef20fc819..51d98a9fd06b9b2ceb1fdfef2136afcbede66531 100644 (file)
--- a/lib/test_maple_tree.c
+++ b/lib/test_maple_tree.c
@@ -223,9 +223,9 @@ static DEFINE_MTREE(tree);
 
 static int maple_tree_seed(void)
 {
-       unsigned long set[] = {15, 14, 17, 25, 1000,
+       unsigned long set[] = {5015, 5014, 5017, 25, 1000,
                               1001, 1002, 1003, 1005, 0,
-                              3, 2};
+                              5003, 5002};
        unsigned long r[] = {10, 15, 20, 25, 22}; // For range testing
        void *ptr = &set;
 
@@ -242,15 +242,13 @@ static int maple_tree_seed(void)
 
        check_insert(&tree, set[10], ptr);      // Insert 3
        check_load(&tree, set[9], &tree);       // See if 0 -> &tree
-
        check_load(&tree, set[11], NULL);       // See if 2 -> NULL
-
-
        check_load(&tree, set[10], ptr);       // See if 3 -> ptr
 
        /* Clear out the tree */
        mtree_destroy(&tree);
 
+
        /* Try to insert, insert a dup, and load back what was inserted. */
        mtree_init(&tree);
        check_insert(&tree, set[0], &tree);     // Insert 15