block: cancel all throttled bios in del_gendisk()

Throttled bios can't be issued after del_gendisk() is done, thus
it's better to cancel them immediately rather than waiting for
throttle is done.

For example, if user thread is throttled with low bps while it's
issuing large io, and the device is deleted. The user thread will
wait for a long time for io to return.

Signed-off-by: Yu Kuai <yukuai3@huawei.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20220318130144.1066064-4-ming.lei@redhat.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 880701d0106fb4b17ac60efb29b6253871da01b4..469c483719bea8309a92a14a6c2c39cb9cc5bd15 100644 (file)
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -874,7 +874,8 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio,
               bio != throtl_peek_queued(&tg->service_queue.queued[rw]));
 
        /* If tg->bps = -1, then BW is unlimited */
-       if (bps_limit == U64_MAX && iops_limit == UINT_MAX) {
+       if ((bps_limit == U64_MAX && iops_limit == UINT_MAX) ||
+           tg->flags & THROTL_TG_CANCELING) {
                if (wait)
                        *wait = 0;
                return true;
@@ -1776,6 +1777,39 @@ static bool throtl_hierarchy_can_upgrade(struct throtl_grp *tg)
        return false;
 }
 
+void blk_throtl_cancel_bios(struct request_queue *q)
+{
+       struct cgroup_subsys_state *pos_css;
+       struct blkcg_gq *blkg;
+
+       spin_lock_irq(&q->queue_lock);
+       /*
+        * queue_lock is held, rcu lock is not needed here technically.
+        * However, rcu lock is still held to emphasize that following
+        * path need RCU protection and to prevent warning from lockdep.
+        */
+       rcu_read_lock();
+       blkg_for_each_descendant_post(blkg, pos_css, q->root_blkg) {
+               struct throtl_grp *tg = blkg_to_tg(blkg);
+               struct throtl_service_queue *sq = &tg->service_queue;
+
+               /*
+                * Set the flag to make sure throtl_pending_timer_fn() won't
+                * stop until all throttled bios are dispatched.
+                */
+               blkg_to_tg(blkg)->flags |= THROTL_TG_CANCELING;
+               /*
+                * Update disptime after setting the above flag to make sure
+                * throtl_select_dispatch() won't exit without dispatching.
+                */
+               tg_update_disptime(tg);
+
+               throtl_schedule_pending_timer(sq, jiffies + 1);
+       }
+       rcu_read_unlock();
+       spin_unlock_irq(&q->queue_lock);
+}
+
 static bool throtl_can_upgrade(struct throtl_data *td,
        struct throtl_grp *this_tg)
 {

diff --git a/block/blk-throttle.h b/block/blk-throttle.h
index b23a9f3abb8242ae5270528082c88c523f74c6a3..c1b60299612723ebc4cef93a0574a905d76ec780 100644 (file)
--- a/block/blk-throttle.h
+++ b/block/blk-throttle.h
@@ -56,6 +56,7 @@ enum tg_state_flags {
        THROTL_TG_PENDING       = 1 << 0,       /* on parent's pending tree */
        THROTL_TG_WAS_EMPTY     = 1 << 1,       /* bio_lists[] became non-empty */
        THROTL_TG_HAS_IOPS_LIMIT = 1 << 2,      /* tg has iops limit */
+       THROTL_TG_CANCELING     = 1 << 3,       /* starts to cancel bio */
 };
 
 enum {
@@ -162,11 +163,13 @@ static inline int blk_throtl_init(struct request_queue *q) { return 0; }
 static inline void blk_throtl_exit(struct request_queue *q) { }
 static inline void blk_throtl_register_queue(struct request_queue *q) { }
 static inline bool blk_throtl_bio(struct bio *bio) { return false; }
+static inline void blk_throtl_cancel_bios(struct request_queue *q) { }
 #else /* CONFIG_BLK_DEV_THROTTLING */
 int blk_throtl_init(struct request_queue *q);
 void blk_throtl_exit(struct request_queue *q);
 void blk_throtl_register_queue(struct request_queue *q);
 bool __blk_throtl_bio(struct bio *bio);
+void blk_throtl_cancel_bios(struct request_queue *q);
 static inline bool blk_throtl_bio(struct bio *bio)
 {
        struct throtl_grp *tg = blkg_to_tg(bio->bi_blkg);

diff --git a/block/genhd.c b/block/genhd.c
index 56f66c6fee943360a0f144927c2a7d5e161ab35c..8244f09e058dd468d6e24610d73c16934c2e28c6 100644 (file)
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -25,6 +25,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/badblocks.h>
 #include <linux/part_stat.h>
+#include "blk-throttle.h"
 
 #include "blk.h"
 #include "blk-mq-sched.h"
@@ -627,6 +628,8 @@ void del_gendisk(struct gendisk *disk)
 
        blk_mq_freeze_queue_wait(q);
 
+       blk_throtl_cancel_bios(disk->queue);
+
        blk_sync_queue(q);
        blk_flush_integrity();
        /*