block: update cached timestamp post schedule/preemption

Mark the task as having a cached timestamp when set assign it, so we
can efficiently check if it needs updating post being scheduled back in.
This covers both the actual schedule out case, which would've flushed
the plug, and the preemption case which doesn't touch the plugged
requests (for many reasons, one of them being then we'd need to have
preemption disabled around plug state manipulation).

Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-core.c b/block/blk-core.c
index 13b449df5ba02bdf18fe685c4d413f9639acda5f..314c3065891a510b51fd5aa495ce0d83f513655e 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1183,6 +1183,8 @@ void __blk_flush_plug(struct blk_plug *plug, bool from_schedule)
         */
        if (unlikely(!rq_list_empty(plug->cached_rq)))
                blk_mq_free_plug_rqs(plug);
+
+       current->flags &= ~PF_BLOCK_TS;
 }
 
 /**

diff --git a/block/blk.h b/block/blk.h
index 14bbc4b780f275cd175e26f38f9f97278fb54ddd..913c93838a01bf254e546319c979c9d912917cc2 100644 (file)
--- a/block/blk.h
+++ b/block/blk.h
@@ -529,8 +529,10 @@ static inline u64 blk_time_get_ns(void)
         * a valid timestamp" separately, just accept that we'll do an extra
         * ktime_get_ns() if we just happen to get 0 as the current time.
         */
-       if (!plug->cur_ktime)
+       if (!plug->cur_ktime) {
                plug->cur_ktime = ktime_get_ns();
+               current->flags |= PF_BLOCK_TS;
+       }
        return plug->cur_ktime;
 }
 

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 996d2ad756ff782e8f2e65cc63f588325d2f0301..d7cac3de65b31bc0cb5a5a62c6896ce69925f5ec 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -973,6 +973,18 @@ static inline void blk_flush_plug(struct blk_plug *plug, bool async)
                __blk_flush_plug(plug, async);
 }
 
+/*
+ * tsk == current here
+ */
+static inline void blk_plug_invalidate_ts(struct task_struct *tsk)
+{
+       struct blk_plug *plug = tsk->plug;
+
+       if (plug)
+               plug->cur_ktime = 0;
+       current->flags &= ~PF_BLOCK_TS;
+}
+
 int blkdev_issue_flush(struct block_device *bdev);
 long nr_blockdev_pages(void);
 #else /* CONFIG_BLOCK */
@@ -996,6 +1008,10 @@ static inline void blk_flush_plug(struct blk_plug *plug, bool async)
 {
 }
 
+static inline void blk_plug_invalidate_ts(struct task_struct *tsk)
+{
+}
+
 static inline int blkdev_issue_flush(struct block_device *bdev)
 {
        return 0;

diff --git a/include/linux/sched.h b/include/linux/sched.h
index ffe8f618ab869729bd6c888a8a05e45d46a6c7c2..15b7cb478d166a2298c1139ba09827e304358df5 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1642,7 +1642,7 @@ extern struct pid *cad_pid;
 #define PF_NO_SETAFFINITY      0x04000000      /* Userland is not allowed to meddle with cpus_mask */
 #define PF_MCE_EARLY           0x08000000      /* Early kill for mce process policy */
 #define PF_MEMALLOC_PIN                0x10000000      /* Allocation context constrained to zones which allow long term pinning. */
-#define PF__HOLE__20000000     0x20000000
+#define PF_BLOCK_TS            0x20000000      /* plug has ts that needs updating */
 #define PF__HOLE__40000000     0x40000000
 #define PF_SUSPEND_TASK                0x80000000      /* This thread called freeze_processes() and should not be frozen */
 

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9116bcc903467fe0d5854e3deb06b8d334cf85eb..083f2258182d899c44d5436ad8ff7097c24b3ebd 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6787,10 +6787,12 @@ static inline void sched_submit_work(struct task_struct *tsk)
 
 static void sched_update_worker(struct task_struct *tsk)
 {
-       if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
+       if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) {
+               if (tsk->flags & PF_BLOCK_TS)
+                       blk_plug_invalidate_ts(tsk);
                if (tsk->flags & PF_WQ_WORKER)
                        wq_worker_running(tsk);
-               else
+               else if (tsk->flags & PF_IO_WORKER)
                        io_wq_worker_running(tsk);
        }
 }