maple_tree: Fix write memory barrier of nodes once dead for RCU mode

During the development of the maple tree, the strategy of freeing
multiple nodes changed and, in the process, the pivots were reused to
store pointers to dead nodes.  To ensure the readers see accurate
pivots, the writers need to mark the nodes as dead and call smp_wmb() to
ensure any readers can identify the node as dead before using the pivot
values.

There were two places where the old method of marking the node as dead
without smp_wmb() were being used, which resulted in RCU readers seeing
the wrong pivot value before seeing the node was dead.  Fix this race
condition by using mte_set_node_dead() which has the smp_wmb() call to
ensure the race is closed.

Add a WARN_ON() to the ma_free_rcu() call to ensure all nodes being
freed are marked as dead to ensure there are no other call paths besides
the two updated paths.

This is necessary for the RCU mode of the maple tree.

Fixes: 54a611b60590 ("Maple Tree: add new data structure")
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 73c8e92e987bdabbc896cf9862803670902a3298..824e2ed2a029beccda02785d227b3460795fdf7d 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -178,7 +178,7 @@ static void mt_free_rcu(struct rcu_head *head)
  */
 static void ma_free_rcu(struct maple_node *node)
 {
-       node->parent = ma_parent_ptr(node);
+       WARN_ON(node->parent != ma_parent_ptr(node));
        call_rcu(&node->rcu, mt_free_rcu);
 }
 
@@ -1772,8 +1772,10 @@ static inline void mas_replace(struct ma_state *mas, bool advanced)
                rcu_assign_pointer(slots[offset], mas->node);
        }
 
-       if (!advanced)
+       if (!advanced) {
+               mte_set_node_dead(old_enode);
                mas_free(mas, old_enode);
+       }
 }
 
 /*
@@ -4214,6 +4216,7 @@ static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas)
 done:
        mas_leaf_set_meta(mas, newnode, dst_pivots, maple_leaf_64, new_end);
        if (in_rcu) {
+               mte_set_node_dead(mas->node);
                mas->node = mt_mk_node(newnode, wr_mas->type);
                mas_replace(mas, false);
        } else {

diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c
index 591c1366a120afd04f49e96c78a988dbac92e51d..44a02b3d972b8a70f70ab27731e709921c5e5623 100644 (file)
--- a/tools/testing/radix-tree/maple.c
+++ b/tools/testing/radix-tree/maple.c
@@ -107,6 +107,7 @@ static noinline void check_new_node(struct maple_tree *mt)
        MT_BUG_ON(mt, mn->slot[1] != NULL);
        MT_BUG_ON(mt, mas_allocated(&mas) != 0);
 
+       mn->parent = ma_parent_ptr(mn);
        ma_free_rcu(mn);
        mas.node = MAS_START;
        mas_nomem(&mas, GFP_KERNEL);
@@ -159,6 +160,7 @@ static noinline void check_new_node(struct maple_tree *mt)
                MT_BUG_ON(mt, mas_allocated(&mas) != i);
                MT_BUG_ON(mt, !mn);
                MT_BUG_ON(mt, not_empty(mn));
+               mn->parent = ma_parent_ptr(mn);
                ma_free_rcu(mn);
        }
 
@@ -191,6 +193,7 @@ static noinline void check_new_node(struct maple_tree *mt)
                MT_BUG_ON(mt, not_empty(mn));
                MT_BUG_ON(mt, mas_allocated(&mas) != i - 1);
                MT_BUG_ON(mt, !mn);
+               mn->parent = ma_parent_ptr(mn);
                ma_free_rcu(mn);
        }
 
@@ -209,6 +212,7 @@ static noinline void check_new_node(struct maple_tree *mt)
                        mn = mas_pop_node(&mas);
                        MT_BUG_ON(mt, not_empty(mn));
                        MT_BUG_ON(mt, mas_allocated(&mas) != j - 1);
+                       mn->parent = ma_parent_ptr(mn);
                        ma_free_rcu(mn);
                }
                MT_BUG_ON(mt, mas_allocated(&mas) != 0);
@@ -232,6 +236,7 @@ static noinline void check_new_node(struct maple_tree *mt)
                        MT_BUG_ON(mt, mas_allocated(&mas) != i - j);
                        mn = mas_pop_node(&mas);
                        MT_BUG_ON(mt, not_empty(mn));
+                       mn->parent = ma_parent_ptr(mn);
                        ma_free_rcu(mn);
                        MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1);
                }
@@ -268,6 +273,7 @@ static noinline void check_new_node(struct maple_tree *mt)
                        mn = mas_pop_node(&mas); /* get the next node. */
                        MT_BUG_ON(mt, mn == NULL);
                        MT_BUG_ON(mt, not_empty(mn));
+                       mn->parent = ma_parent_ptr(mn);
                        ma_free_rcu(mn);
                }
                MT_BUG_ON(mt, mas_allocated(&mas) != 0);
@@ -293,6 +299,7 @@ static noinline void check_new_node(struct maple_tree *mt)
                        mn = mas_pop_node(&mas2); /* get the next node. */
                        MT_BUG_ON(mt, mn == NULL);
                        MT_BUG_ON(mt, not_empty(mn));
+                       mn->parent = ma_parent_ptr(mn);
                        ma_free_rcu(mn);
                }
                MT_BUG_ON(mt, mas_allocated(&mas2) != 0);
@@ -333,10 +340,12 @@ static noinline void check_new_node(struct maple_tree *mt)
        MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2);
        mn = mas_pop_node(&mas);
        MT_BUG_ON(mt, not_empty(mn));
+       mn->parent = ma_parent_ptr(mn);
        ma_free_rcu(mn);
        for (i = 1; i <= MAPLE_ALLOC_SLOTS + 1; i++) {
                mn = mas_pop_node(&mas);
                MT_BUG_ON(mt, not_empty(mn));
+               mn->parent = ma_parent_ptr(mn);
                ma_free_rcu(mn);
        }
        MT_BUG_ON(mt, mas_allocated(&mas) != 0);
@@ -374,6 +383,7 @@ static noinline void check_new_node(struct maple_tree *mt)
                mas_node_count(&mas, i); /* Request */
                mas_nomem(&mas, GFP_KERNEL); /* Fill request */
                mn = mas_pop_node(&mas); /* get the next node. */
+               mn->parent = ma_parent_ptr(mn);
                ma_free_rcu(mn);
                mas_destroy(&mas);
 
@@ -381,10 +391,13 @@ static noinline void check_new_node(struct maple_tree *mt)
                mas_node_count(&mas, i); /* Request */
                mas_nomem(&mas, GFP_KERNEL); /* Fill request */
                mn = mas_pop_node(&mas); /* get the next node. */
+               mn->parent = ma_parent_ptr(mn);
                ma_free_rcu(mn);
                mn = mas_pop_node(&mas); /* get the next node. */
+               mn->parent = ma_parent_ptr(mn);
                ma_free_rcu(mn);
                mn = mas_pop_node(&mas); /* get the next node. */
+               mn->parent = ma_parent_ptr(mn);
                ma_free_rcu(mn);
                mas_destroy(&mas);
        }
@@ -35368,6 +35381,7 @@ static noinline void check_prealloc(struct maple_tree *mt)
        MT_BUG_ON(mt, allocated != 1 + height * 3);
        mn = mas_pop_node(&mas);
        MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1);
+       mn->parent = ma_parent_ptr(mn);
        ma_free_rcu(mn);
        MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
        mas_destroy(&mas);
@@ -35385,6 +35399,7 @@ static noinline void check_prealloc(struct maple_tree *mt)
        mas_destroy(&mas);
        allocated = mas_allocated(&mas);
        MT_BUG_ON(mt, allocated != 0);
+       mn->parent = ma_parent_ptr(mn);
        ma_free_rcu(mn);
 
        MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
@@ -35755,6 +35770,7 @@ void farmer_tests(void)
        tree.ma_root = mt_mk_node(node, maple_leaf_64);
        mt_dump(&tree, mt_dump_dec);
 
+       node->parent = ma_parent_ptr(node);
        ma_free_rcu(node);
 
        /* Check things that will make lockdep angry */