blk-mq: remove REQ_ATOM_STARTED

After the recent updates to use generation number and state based
synchronization, we can easily replace REQ_ATOM_STARTED usages by
adding an extra state to distinguish completed but not yet freed
state.

Add MQ_RQ_COMPLETE and replace REQ_ATOM_STARTED usages with
blk_mq_rq_state() tests.  REQ_ATOM_STARTED no longer has any users
left and is removed.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c
index b56a4f35720d8a46e8a5daf3f2c63a51475308a3..8adc83786256c33039c85e72ab9b4f7d187db5d7 100644 (file)
--- a/block/blk-mq-debugfs.c
+++ b/block/blk-mq-debugfs.c
@@ -271,7 +271,6 @@ static const char *const cmd_flag_name[] = {
 #define RQF_NAME(name) [ilog2((__force u32)RQF_##name)] = #name
 static const char *const rqf_name[] = {
        RQF_NAME(SORTED),
-       RQF_NAME(STARTED),
        RQF_NAME(QUEUED),
        RQF_NAME(SOFTBARRIER),
        RQF_NAME(FLUSH_SEQ),
@@ -295,7 +294,6 @@ static const char *const rqf_name[] = {
 #define RQAF_NAME(name) [REQ_ATOM_##name] = #name
 static const char *const rqaf_name[] = {
        RQAF_NAME(COMPLETE),
-       RQAF_NAME(STARTED),
        RQAF_NAME(POLL_SLEPT),
 };
 #undef RQAF_NAME
@@ -409,7 +407,7 @@ static void hctx_show_busy_rq(struct request *rq, void *data, bool reserved)
        const struct show_busy_params *params = data;
 
        if (blk_mq_map_queue(rq->q, rq->mq_ctx->cpu) == params->hctx &&
-           test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
+           blk_mq_rq_state(rq) != MQ_RQ_IDLE)
                __blk_mq_debugfs_rq_show(params->m,
                                         list_entry_rq(&rq->queuelist));
 }

diff --git a/block/blk-mq.c b/block/blk-mq.c
index d1000c6cbec6d9b02ecf38b29be502d092b99cdf..275812909d77da50f224377ca36416ab47b8fbae 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -483,7 +483,6 @@ void blk_mq_free_request(struct request *rq)
                blk_put_rl(blk_rq_rl(rq));
 
        blk_mq_rq_update_state(rq, MQ_RQ_IDLE);
-       clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
        clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
        if (rq->tag != -1)
                blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
@@ -531,6 +530,7 @@ static void __blk_mq_complete_request(struct request *rq)
        int cpu;
 
        WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT);
+       blk_mq_rq_update_state(rq, MQ_RQ_COMPLETE);
 
        if (rq->internal_tag != -1)
                blk_mq_sched_completed_request(rq);
@@ -642,7 +642,7 @@ EXPORT_SYMBOL(blk_mq_complete_request);
 
 int blk_mq_request_started(struct request *rq)
 {
-       return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
+       return blk_mq_rq_state(rq) != MQ_RQ_IDLE;
 }
 EXPORT_SYMBOL_GPL(blk_mq_request_started);
 
@@ -661,7 +661,6 @@ void blk_mq_start_request(struct request *rq)
        }
 
        WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
-       WARN_ON_ONCE(test_bit(REQ_ATOM_STARTED, &rq->atomic_flags));
 
        /*
         * Mark @rq in-flight which also advances the generation number,
@@ -683,8 +682,6 @@ void blk_mq_start_request(struct request *rq)
        write_seqcount_end(&rq->gstate_seq);
        preempt_enable();
 
-       set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
-
        if (q->dma_drain_size && blk_rq_bytes(rq)) {
                /*
                 * Make sure space for the drain appears.  We know we can do
@@ -697,13 +694,9 @@ void blk_mq_start_request(struct request *rq)
 EXPORT_SYMBOL(blk_mq_start_request);
 
 /*
- * When we reach here because queue is busy, REQ_ATOM_COMPLETE
- * flag isn't set yet, so there may be race with timeout handler,
- * but given rq->deadline is just set in .queue_rq() under
- * this situation, the race won't be possible in reality because
- * rq->timeout should be set as big enough to cover the window
- * between blk_mq_start_request() called from .queue_rq() and
- * clearing REQ_ATOM_STARTED here.
+ * When we reach here because queue is busy, it's safe to change the state
+ * to IDLE without checking @rq->aborted_gstate because we should still be
+ * holding the RCU read lock and thus protected against timeout.
  */
 static void __blk_mq_requeue_request(struct request *rq)
 {
@@ -715,7 +708,7 @@ static void __blk_mq_requeue_request(struct request *rq)
        wbt_requeue(q->rq_wb, &rq->issue_stat);
        blk_mq_sched_requeue_request(rq);
 
-       if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
+       if (blk_mq_rq_state(rq) != MQ_RQ_IDLE) {
                blk_mq_rq_update_state(rq, MQ_RQ_IDLE);
                if (q->dma_drain_size && blk_rq_bytes(rq))
                        rq->nr_phys_segments--;
@@ -822,18 +815,6 @@ static void blk_mq_rq_timed_out(struct request *req, bool reserved)
        const struct blk_mq_ops *ops = req->q->mq_ops;
        enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
 
-       /*
-        * We know that complete is set at this point. If STARTED isn't set
-        * anymore, then the request isn't active and the "timeout" should
-        * just be ignored. This can happen due to the bitflag ordering.
-        * Timeout first checks if STARTED is set, and if it is, assumes
-        * the request is active. But if we race with completion, then
-        * both flags will get cleared. So check here again, and ignore
-        * a timeout event with a request that isn't active.
-        */
-       if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
-               return;
-
        req->rq_flags |= RQF_MQ_TIMEOUT_EXPIRED;
 
        if (ops->timeout)
@@ -869,8 +850,7 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
 
        might_sleep();
 
-       if ((rq->rq_flags & RQF_MQ_TIMEOUT_EXPIRED) ||
-           !test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
+       if (rq->rq_flags & RQF_MQ_TIMEOUT_EXPIRED)
                return;
 
        /* read coherent snapshots of @rq->state_gen and @rq->deadline */
@@ -3022,8 +3002,7 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
 
        hrtimer_init_sleeper(&hs, current);
        do {
-               if (test_bit(REQ_ATOM_STARTED, &rq->atomic_flags) &&
-                   blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
+               if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
                        break;
                set_current_state(TASK_UNINTERRUPTIBLE);
                hrtimer_start_expires(&hs.timer, mode);

diff --git a/block/blk-mq.h b/block/blk-mq.h
index 6b2d61629d4879f4d5aec713afd8417d8ab674d4..8591a54d989b163284f14d420524af6dc008cc28 100644 (file)
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -34,6 +34,7 @@ struct blk_mq_ctx {
 enum mq_rq_state {
        MQ_RQ_IDLE              = 0,
        MQ_RQ_IN_FLIGHT         = 1,
+       MQ_RQ_COMPLETE          = 2,
 
        MQ_RQ_STATE_BITS        = 2,
        MQ_RQ_STATE_MASK        = (1 << MQ_RQ_STATE_BITS) - 1,

diff --git a/block/blk.h b/block/blk.h
index 9cb2739edb6af9be99ead3b3459e62b5182ca1ee..a68dbe312ea36377064af0da0628437edb60fab8 100644 (file)
--- a/block/blk.h
+++ b/block/blk.h
@@ -124,7 +124,6 @@ void blk_account_io_done(struct request *req);
  */
 enum rq_atomic_flags {
        REQ_ATOM_COMPLETE = 0,
-       REQ_ATOM_STARTED,
 
        REQ_ATOM_POLL_SLEPT,
 };