mm: Convert cgroup writeback to XArray

We're still under the protection of the cgwb_lock as well as the xa_lock.
It could probably be removed or reduced in scope in a few places, but
I'll leave that to someone who understands this code better than I do.

Signed-off-by: Matthew Wilcox <willy@infradead.org>

diff --git a/include/linux/backing-dev-defs.h b/include/linux/backing-dev-defs.h
index 6a1a8a314d85321f0852df56ddb60f86168753a2..92aa3a71d4be2ce0b9e679df77b9e87823467abf 100644 (file)
--- a/include/linux/backing-dev-defs.h
+++ b/include/linux/backing-dev-defs.h
@@ -187,7 +187,7 @@ struct backing_dev_info {
        struct bdi_writeback wb;  /* the root writeback info for this bdi */
        struct list_head wb_list; /* list of all wbs */
 #ifdef CONFIG_CGROUP_WRITEBACK
-       struct radix_tree_root cgwb_tree; /* radix tree of active cgroup wbs */
+       struct xarray cgwb_xa;          /* radix tree of active cgroup wbs */
        struct rb_root cgwb_congested_tree; /* their congested states */
        struct mutex cgwb_release_mutex;  /* protect shutdown of wb structs */
        struct rw_semaphore wb_switch_rwsem; /* no cgwb switch while syncing */

diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
index 35b31d176f7475273704192bf9dc448ea09b3cd3..30c9e01ca938150b067424a563a0f0d1d744d73a 100644 (file)
--- a/include/linux/backing-dev.h
+++ b/include/linux/backing-dev.h
@@ -273,7 +273,7 @@ static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi
        if (!memcg_css->parent)
                return &bdi->wb;
 
-       wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
+       wb = xa_load(&bdi->cgwb_xa, memcg_css->id);
 
        /*
         * %current's blkcg equals the effective blkcg of its memcg.  No

diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index e8e89158adec6cb17e941289d4e1e8cfdd829c57..73f455d95006266a062b240d2ecba33d1c460dc2 100644 (file)
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -381,8 +381,8 @@ static void wb_exit(struct bdi_writeback *wb)
 #include <linux/memcontrol.h>
 
 /*
- * cgwb_lock protects bdi->cgwb_tree, bdi->cgwb_congested_tree,
- * blkcg->cgwb_list, and memcg->cgwb_list.  bdi->cgwb_tree is also RCU
+ * cgwb_lock protects bdi->cgwb_xa, bdi->cgwb_congested_tree,
+ * blkcg->cgwb_list, and memcg->cgwb_list.  bdi->cgwb_xa is also RCU
  * protected.
  */
 static DEFINE_SPINLOCK(cgwb_lock);
@@ -507,7 +507,7 @@ static void cgwb_kill(struct bdi_writeback *wb)
 {
        lockdep_assert_held(&cgwb_lock);
 
-       WARN_ON(!radix_tree_delete(&wb->bdi->cgwb_tree, wb->memcg_css->id));
+       WARN_ON(xa_erase(&wb->bdi->cgwb_xa, wb->memcg_css->id) != wb);
        list_del(&wb->memcg_node);
        list_del(&wb->blkcg_node);
        percpu_ref_kill(&wb->refcnt);
@@ -539,7 +539,7 @@ static int cgwb_create(struct backing_dev_info *bdi,
 
        /* look up again under lock and discard on blkcg mismatch */
        spin_lock_irqsave(&cgwb_lock, flags);
-       wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
+       wb = xa_load(&bdi->cgwb_xa, memcg_css->id);
        if (wb && wb->blkcg_css != blkcg_css) {
                cgwb_kill(wb);
                wb = NULL;
@@ -583,7 +583,7 @@ static int cgwb_create(struct backing_dev_info *bdi,
        if (test_bit(WB_registered, &bdi->wb.state) &&
            blkcg_cgwb_list->next && memcg_cgwb_list->next) {
                /* we might have raced another instance of this function */
-               ret = radix_tree_insert(&bdi->cgwb_tree, memcg_css->id, wb);
+               ret = xa_insert(&bdi->cgwb_xa, memcg_css->id, wb, GFP_ATOMIC);
                if (!ret) {
                        list_add_tail_rcu(&wb->bdi_node, &bdi->wb_list);
                        list_add(&wb->memcg_node, memcg_cgwb_list);
@@ -595,7 +595,7 @@ static int cgwb_create(struct backing_dev_info *bdi,
        }
        spin_unlock_irqrestore(&cgwb_lock, flags);
        if (ret) {
-               if (ret == -EEXIST)
+               if (ret == -EBUSY)
                        ret = 0;
                goto err_fprop_exit;
        }
@@ -651,7 +651,7 @@ struct bdi_writeback *wb_get_create(struct backing_dev_info *bdi,
 
        do {
                rcu_read_lock();
-               wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
+               wb = xa_load(&bdi->cgwb_xa, memcg_css->id);
                if (wb) {
                        struct cgroup_subsys_state *blkcg_css;
 
@@ -673,7 +673,7 @@ static int cgwb_bdi_init(struct backing_dev_info *bdi)
 {
        int ret;
 
-       INIT_RADIX_TREE(&bdi->cgwb_tree, GFP_ATOMIC);
+       xa_init_flags(&bdi->cgwb_xa, XA_FLAGS_LOCK_IRQ);
        bdi->cgwb_congested_tree = RB_ROOT;
        mutex_init(&bdi->cgwb_release_mutex);
        init_rwsem(&bdi->wb_switch_rwsem);
@@ -688,15 +688,14 @@ static int cgwb_bdi_init(struct backing_dev_info *bdi)
 
 static void cgwb_bdi_unregister(struct backing_dev_info *bdi)
 {
-       struct radix_tree_iter iter;
-       void **slot;
        struct bdi_writeback *wb;
+       unsigned long index;
 
        WARN_ON(test_bit(WB_registered, &bdi->wb.state));
 
        spin_lock_irq(&cgwb_lock);
-       radix_tree_for_each_slot(slot, &bdi->cgwb_tree, &iter, 0)
-               cgwb_kill(*slot);
+       xa_for_each(&bdi->cgwb_xa, index, wb)
+               cgwb_kill(wb);
        spin_unlock_irq(&cgwb_lock);
 
        mutex_lock(&bdi->cgwb_release_mutex);