blk-mq-sched: change ->dispatch_requests() to ->dispatch_request()

When we invoke dispatch_requests(), the scheduler empties everything
into the passed in list. This isn't always a good thing, since it
means that we remove items that we could have potentially merged
with.

Change the function to dispatch single requests at the time. If
we do that, we can backoff exactly at the point where the device
can't consume more IO, and leave the rest with the scheduler for
better merging and future dispatch decision making.

Signed-off-by: Jens Axboe <axboe@fb.com>
Reviewed-by: Omar Sandoval <osandov@fb.com>
Tested-by: Hannes Reinecke <hare@suse.com>

diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index fcc0e893d6870a76ab0c338a4018e09438ad542b..c27613de80c5dd1b1c088eaa54fc1af653a96f1a 100644 (file)
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -201,15 +201,22 @@ void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
         * leave them there for as long as we can. Mark the hw queue as
         * needing a restart in that case.
         */
-       if (list_empty(&rq_list)) {
-               if (e && e->type->ops.mq.dispatch_requests)
-                       e->type->ops.mq.dispatch_requests(hctx, &rq_list);
-               else
-                       blk_mq_flush_busy_ctxs(hctx, &rq_list);
-       } else
+       if (!list_empty(&rq_list)) {
                blk_mq_sched_mark_restart(hctx);
-
-       blk_mq_dispatch_rq_list(hctx, &rq_list);
+               blk_mq_dispatch_rq_list(hctx, &rq_list);
+       } else if (!e || !e->type->ops.mq.dispatch_request) {
+               blk_mq_flush_busy_ctxs(hctx, &rq_list);
+               blk_mq_dispatch_rq_list(hctx, &rq_list);
+       } else {
+               do {
+                       struct request *rq;
+
+                       rq = e->type->ops.mq.dispatch_request(hctx);
+                       if (!rq)
+                               break;
+                       list_add(&rq->queuelist, &rq_list);
+               } while (blk_mq_dispatch_rq_list(hctx, &rq_list));
+       }
 }
 
 void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 21795c6575bc61fff24e900188950d61eb0da28f..301ae29fd229a992839e5ea81f4115e23243b53b 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -998,7 +998,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list)
         */
        if (!list_empty(list)) {
                spin_lock(&hctx->lock);
-               list_splice(list, &hctx->dispatch);
+               list_splice_init(list, &hctx->dispatch);
                spin_unlock(&hctx->lock);
 
                /*

diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index a01986d7b6fb962c899b06873f46e51a2249c43b..d93ec713fa623d648d8ab7cba22e236d36f51b16 100644 (file)
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -287,14 +287,16 @@ done:
        return rq;
 }
 
-static void dd_dispatch_requests(struct blk_mq_hw_ctx *hctx,
-                                struct list_head *rq_list)
+static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
 {
        struct deadline_data *dd = hctx->queue->elevator->elevator_data;
+       struct request *rq;
 
        spin_lock(&dd->lock);
-       blk_mq_sched_move_to_dispatch(hctx, rq_list, __dd_dispatch_request);
+       rq = __dd_dispatch_request(hctx);
        spin_unlock(&dd->lock);
+
+       return rq;
 }
 
 static void dd_exit_queue(struct elevator_queue *e)
@@ -517,7 +519,7 @@ static struct elv_fs_entry deadline_attrs[] = {
 static struct elevator_type mq_deadline = {
        .ops.mq = {
                .insert_requests        = dd_insert_requests,
-               .dispatch_requests      = dd_dispatch_requests,
+               .dispatch_request       = dd_dispatch_request,
                .next_request           = elv_rb_latter_request,
                .former_request         = elv_rb_former_request,
                .bio_merge              = dd_bio_merge,

diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index ecb96fd67c6d477c042173a14c0765582201db5a..b5825c4f06f7fe3f5e7fb9bb714cf99e73f6eced 100644 (file)
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -92,7 +92,7 @@ struct elevator_mq_ops {
        struct request *(*get_request)(struct request_queue *, unsigned int, struct blk_mq_alloc_data *);
        void (*put_request)(struct request *);
        void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool);
-       void (*dispatch_requests)(struct blk_mq_hw_ctx *, struct list_head *);
+       struct request *(*dispatch_request)(struct blk_mq_hw_ctx *);
        bool (*has_work)(struct blk_mq_hw_ctx *);
        void (*completed_request)(struct blk_mq_hw_ctx *, struct request *);
        void (*started_request)(struct request *);