From: Christoph Hellwig Date: Fri, 9 Sep 2022 12:18:02 +0000 (+0200) Subject: blk-cgroup: move the cgroup information to struct gendisk X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=refs%2Fheads%2Fblk-cgroup-gendisk;p=users%2Fhch%2Fblock.git blk-cgroup: move the cgroup information to struct gendisk cgroup information only makes sense on a live gendisk that allows file system I/O (which includes the raw block device). So move over the cgroup related members. Signed-off-by: Christoph Hellwig --- diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 1351fab3755e..ae9195164f50 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -984,7 +984,7 @@ void bfq_end_wr_async(struct bfq_data *bfqd) { struct blkcg_gq *blkg; - list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) { + list_for_each_entry(blkg, &bfqd->queue->disk->blkg_list, q_node) { struct bfq_group *bfqg = blkg_to_bfqg(blkg); bfq_end_wr_async_queues(bfqd, bfqg); @@ -1278,7 +1278,7 @@ struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node) if (ret) return NULL; - return blkg_to_bfqg(bfqd->queue->root_blkg); + return blkg_to_bfqg(bfqd->queue->disk->root_blkg); } struct blkcg_policy blkcg_policy_bfq = { diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 2fd232a79d9b..6503da4910e9 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -76,10 +76,10 @@ static struct cgroup_subsys_state *blkcg_css(void) return task_css(current, io_cgrp_id); } -static bool blkcg_policy_enabled(struct request_queue *q, +static bool blkcg_policy_enabled(struct gendisk *disk, const struct blkcg_policy *pol) { - return pol && test_bit(pol->plid, q->blkcg_pols); + return pol && test_bit(pol->plid, disk->blkcg_pols); } static void blkg_free_workfn(struct work_struct *work) @@ -243,7 +243,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, struct blkcg_policy *pol = blkcg_policy[i]; struct blkg_policy_data *pd; - if (!blkcg_policy_enabled(disk->queue, pol)) + if (!blkcg_policy_enabled(disk, pol)) continue; /* alloc per-policy data and attach it to blkg */ @@ -328,7 +328,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg); if (likely(!ret)) { hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); - list_add(&blkg->q_node, &disk->queue->blkg_list); + list_add(&blkg->q_node, &disk->blkg_list); for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkcg_policy *pol = blkcg_policy[i]; @@ -395,7 +395,7 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, while (true) { struct blkcg *pos = blkcg; struct blkcg *parent = blkcg_parent(blkcg); - struct blkcg_gq *ret_blkg = q->root_blkg; + struct blkcg_gq *ret_blkg = disk->root_blkg; while (parent) { blkg = blkg_lookup(parent, disk); @@ -470,7 +470,7 @@ static void blkg_destroy_all(struct gendisk *disk) restart: spin_lock_irq(&q->queue_lock); - list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { + list_for_each_entry_safe(blkg, n, &disk->blkg_list, q_node) { struct blkcg *blkcg = blkg->blkcg; spin_lock(&blkcg->lock); @@ -489,7 +489,7 @@ restart: } } - q->root_blkg = NULL; + disk->root_blkg = NULL; spin_unlock_irq(&q->queue_lock); } @@ -566,7 +566,7 @@ void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, rcu_read_lock(); hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { spin_lock_irq(&blkg->disk->queue->queue_lock); - if (blkcg_policy_enabled(blkg->disk->queue, pol)) + if (blkcg_policy_enabled(blkg->disk, pol)) total += prfill(sf, blkg->pd[pol->plid], data); spin_unlock_irq(&blkg->disk->queue->queue_lock); } @@ -674,7 +674,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, rcu_read_lock(); spin_lock_irq(&q->queue_lock); - if (!blkcg_policy_enabled(q, pol)) { + if (!blkcg_policy_enabled(disk, pol)) { ret = -EOPNOTSUPP; goto fail_unlock; } @@ -719,7 +719,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, rcu_read_lock(); spin_lock_irq(&q->queue_lock); - if (!blkcg_policy_enabled(q, pol)) { + if (!blkcg_policy_enabled(disk, pol)) { blkg_free(new_blkg); ret = -EOPNOTSUPP; goto fail_preloaded; @@ -887,7 +887,7 @@ static void blkcg_fill_root_iostats(void) class_dev_iter_init(&iter, &block_class, NULL, &disk_type); while ((dev = class_dev_iter_next(&iter))) { struct block_device *bdev = dev_to_bdev(dev); - struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg; + struct blkcg_gq *blkg = bdev->bd_disk->root_blkg; struct blkg_iostat tmp; int cpu; unsigned long flags; @@ -1233,7 +1233,7 @@ int blkcg_init_disk(struct gendisk *disk) bool preloaded; int ret; - INIT_LIST_HEAD(&q->blkg_list); + INIT_LIST_HEAD(&disk->blkg_list); new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL); if (!new_blkg) @@ -1247,7 +1247,7 @@ int blkcg_init_disk(struct gendisk *disk) blkg = blkg_create(&blkcg_root, disk, new_blkg); if (IS_ERR(blkg)) goto err_unlock; - q->root_blkg = blkg; + disk->root_blkg = blkg; spin_unlock_irq(&q->queue_lock); if (preloaded) @@ -1359,7 +1359,7 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) struct blkcg_gq *blkg, *pinned_blkg = NULL; int ret; - if (blkcg_policy_enabled(q, pol)) + if (blkcg_policy_enabled(disk, pol)) return 0; if (queue_is_mq(q)) @@ -1368,7 +1368,7 @@ retry: spin_lock_irq(&q->queue_lock); /* blkg_list is pushed at the head, reverse walk to allocate parents first */ - list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) { + list_for_each_entry_reverse(blkg, &disk->blkg_list, q_node) { struct blkg_policy_data *pd; if (blkg->pd[pol->plid]) @@ -1412,10 +1412,10 @@ retry: /* all allocated, init in the same order */ if (pol->pd_init_fn) - list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) + list_for_each_entry_reverse(blkg, &disk->blkg_list, q_node) pol->pd_init_fn(blkg->pd[pol->plid]); - __set_bit(pol->plid, q->blkcg_pols); + __set_bit(pol->plid, disk->blkcg_pols); ret = 0; spin_unlock_irq(&q->queue_lock); @@ -1431,7 +1431,7 @@ out: enomem: /* alloc failed, nothing's initialized yet, free everything */ spin_lock_irq(&q->queue_lock); - list_for_each_entry(blkg, &q->blkg_list, q_node) { + list_for_each_entry(blkg, &disk->blkg_list, q_node) { struct blkcg *blkcg = blkg->blkcg; spin_lock(&blkcg->lock); @@ -1461,7 +1461,7 @@ void blkcg_deactivate_policy(struct gendisk *disk, struct request_queue *q = disk->queue; struct blkcg_gq *blkg; - if (!blkcg_policy_enabled(q, pol)) + if (!blkcg_policy_enabled(disk, pol)) return; if (queue_is_mq(q)) @@ -1469,9 +1469,9 @@ void blkcg_deactivate_policy(struct gendisk *disk, spin_lock_irq(&q->queue_lock); - __clear_bit(pol->plid, q->blkcg_pols); + __clear_bit(pol->plid, disk->blkcg_pols); - list_for_each_entry(blkg, &q->blkg_list, q_node) { + list_for_each_entry(blkg, &disk->blkg_list, q_node) { struct blkcg *blkcg = blkg->blkcg; spin_lock(&blkcg->lock); @@ -1883,7 +1883,7 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, * Associate @bio with the blkg found by combining the css's blkg and the * request_queue of the @bio. An association failure is handled by walking up * the blkg tree. Therefore, the blkg associated can be anything between @blkg - * and q->root_blkg. This situation only happens when a cgroup is dying and + * and disk->root_blkg. This situation only happens when a cgroup is dying and * then the remaining bios will spill to the closest alive blkg. * * A reference will be taken on the blkg and will be released when @bio is @@ -1898,8 +1898,8 @@ void bio_associate_blkg_from_css(struct bio *bio, if (css && css->parent) { bio->bi_blkg = blkg_tryget_closest(bio, css); } else { - blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg); - bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg; + blkg_get(bio->bi_bdev->bd_disk->root_blkg); + bio->bi_blkg = bio->bi_bdev->bd_disk->root_blkg; } } EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css); diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index 589e3bb13411..8d85afe1d80b 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -239,7 +239,7 @@ static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, WARN_ON_ONCE(!rcu_read_lock_held()); if (blkcg == &blkcg_root) - return disk->queue->root_blkg; + return disk->root_blkg; blkg = rcu_dereference(blkcg->blkg_hint); if (blkg && blkg->disk == disk) diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index f9c98e2987f2..13fb06e38235 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -666,7 +666,7 @@ static void blkiolatency_timer_fn(struct timer_list *t) rcu_read_lock(); blkg_for_each_descendant_pre(blkg, pos_css, - blkiolat->rqos.disk->queue->root_blkg) { + blkiolat->rqos.disk->root_blkg) { struct iolatency_grp *iolat; struct child_latency_info *lat_info; unsigned long flags; diff --git a/block/blk-throttle.c b/block/blk-throttle.c index f105224a3f84..e25ac077da81 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -457,7 +457,8 @@ static void blk_throtl_update_limit_valid(struct throtl_data *td) bool low_valid = false; rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, + td->queue->disk->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); if (tg->bps[READ][LIMIT_LOW] || tg->bps[WRITE][LIMIT_LOW] || @@ -1150,7 +1151,7 @@ static void throtl_pending_timer_fn(struct timer_list *t) spin_lock_irq(&q->queue_lock); - if (!q->root_blkg) + if (!q->disk->root_blkg) goto out_unlock; if (throtl_can_upgrade(td, NULL)) @@ -1292,7 +1293,8 @@ static void tg_conf_updated(struct throtl_grp *tg, bool global) * blk-throttle. */ blkg_for_each_descendant_pre(blkg, pos_css, - global ? tg->td->queue->root_blkg : tg_to_blkg(tg)) { + global ? tg->td->queue->disk->root_blkg : + tg_to_blkg(tg)) { struct throtl_grp *this_tg = blkg_to_tg(blkg); struct throtl_grp *parent_tg; @@ -1685,7 +1687,7 @@ void blk_throtl_cancel_bios(struct gendisk *disk) * path need RCU protection and to prevent warning from lockdep. */ rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, q->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, disk->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_service_queue *sq = &tg->service_queue; @@ -1824,7 +1826,8 @@ static bool throtl_can_upgrade(struct throtl_data *td, return false; rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, + td->queue->disk->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); if (tg == this_tg) @@ -1870,7 +1873,8 @@ static void throtl_upgrade_state(struct throtl_data *td) td->low_upgrade_time = jiffies; td->scale = 0; rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { + blkg_for_each_descendant_post(blkg, pos_css, + td->queue->disk->root_blkg) { struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_service_queue *sq = &tg->service_queue; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 84b13fdd34a7..a17c1e283bc5 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -163,6 +163,11 @@ struct gendisk { struct timer_rand_state *random; atomic_t sync_io; /* RAID */ struct disk_events *ev; +#ifdef CONFIG_BLK_CGROUP + DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); + struct blkcg_gq *root_blkg; + struct list_head blkg_list; +#endif #ifdef CONFIG_BLK_DEV_INTEGRITY struct kobject integrity_kobj; #endif /* CONFIG_BLK_DEV_INTEGRITY */ @@ -482,11 +487,6 @@ struct request_queue { struct blk_mq_tags *sched_shared_tags; struct list_head icq_list; -#ifdef CONFIG_BLK_CGROUP - DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); - struct blkcg_gq *root_blkg; - struct list_head blkg_list; -#endif struct queue_limits limits;