block: move call of scheduler's ->completed_request() hook

Commit 4bc6339a583c ("block: move blk_stat_add() to
__blk_mq_end_request()") consolidated some calls using ktime_get() so
we'd only need to call it once. Kyber's ->completed_request() hook also
calls ktime_get(), so let's move it to the same place, too.

Signed-off-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h
index 4e028ee4243014ff8eed44627f8e0cc5068217ca..8a9544203173fac13a718f22e746128246dde4c7 100644 (file)
--- a/block/blk-mq-sched.h
+++ b/block/blk-mq-sched.h
@@ -49,12 +49,12 @@ blk_mq_sched_allow_merge(struct request_queue *q, struct request *rq,
        return true;
 }
 
-static inline void blk_mq_sched_completed_request(struct request *rq)
+static inline void blk_mq_sched_completed_request(struct request *rq, u64 now)
 {
        struct elevator_queue *e = rq->q->elevator;
 
        if (e && e->type->ops.mq.completed_request)
-               e->type->ops.mq.completed_request(rq);
+               e->type->ops.mq.completed_request(rq, now);
 }
 
 static inline void blk_mq_sched_started_request(struct request *rq)

diff --git a/block/blk-mq.c b/block/blk-mq.c
index d384ab700afde2eca3f016f71e31963f9884ca9e..1e72d53e8f2de3844b8b0c5f9fdd93e2c145f0e9 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -528,6 +528,9 @@ inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
                blk_stat_add(rq, now);
        }
 
+       if (rq->internal_tag != -1)
+               blk_mq_sched_completed_request(rq, now);
+
        blk_account_io_done(rq, now);
 
        if (rq->end_io) {
@@ -564,8 +567,6 @@ static void __blk_mq_complete_request(struct request *rq)
 
        if (!blk_mq_mark_complete(rq))
                return;
-       if (rq->internal_tag != -1)
-               blk_mq_sched_completed_request(rq);
 
        if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
                rq->q->softirq_done_fn(rq);

diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c
index a1660bafc912452a37730ac0ed2bf02f6c8a99bc..95d062c07c61e7f64111c85f6d4095f47b79f71a 100644 (file)
--- a/block/kyber-iosched.c
+++ b/block/kyber-iosched.c
@@ -558,12 +558,12 @@ static void kyber_finish_request(struct request *rq)
        rq_clear_domain_token(kqd, rq);
 }
 
-static void kyber_completed_request(struct request *rq)
+static void kyber_completed_request(struct request *rq, u64 now)
 {
        struct request_queue *q = rq->q;
        struct kyber_queue_data *kqd = q->elevator->elevator_data;
        unsigned int sched_domain;
-       u64 now, latency, target;
+       u64 latency, target;
 
        /*
         * Check if this request met our latency goal. If not, quickly gather
@@ -585,7 +585,6 @@ static void kyber_completed_request(struct request *rq)
        if (blk_stat_is_active(kqd->cb))
                return;
 
-       now = ktime_get_ns();
        if (now < rq->io_start_time_ns)
                return;
 

diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index a02deea301857185d2902a6a3c3c0dfaba24edb8..015bb59c0331d64ef2e5b4cf0edbdb53b054c333 100644 (file)
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -111,7 +111,7 @@ struct elevator_mq_ops {
        void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool);
        struct request *(*dispatch_request)(struct blk_mq_hw_ctx *);
        bool (*has_work)(struct blk_mq_hw_ctx *);
-       void (*completed_request)(struct request *);
+       void (*completed_request)(struct request *, u64);
        void (*started_request)(struct request *);
        void (*requeue_request)(struct request *);
        struct request *(*former_request)(struct request_queue *, struct request *);