block: add polled wakeup task helper

If we're polling for IO on a device that doesn't use interrupts, then
IO completion loop (and wake of task) is done by submitting task itself.
If that is the case, then we don't need to enter the wake_up_process()
function, we can simply mark ourselves as TASK_RUNNING.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/block_dev.c b/fs/block_dev.c
index c039abfb205203e1cf250d49e8a2c5887a23b310..9fe56672cfe51a0f645e632c2bc7189e2f5c640d 100644 (file)
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -181,7 +181,7 @@ static void blkdev_bio_end_io_simple(struct bio *bio)
        struct task_struct *waiter = bio->bi_private;
 
        WRITE_ONCE(bio->bi_private, NULL);
-       wake_up_process(waiter);
+       blk_wake_io_task(waiter);
 }
 
 static ssize_t
@@ -305,7 +305,7 @@ static void blkdev_bio_end_io(struct bio *bio)
                        struct task_struct *waiter = dio->waiter;
 
                        WRITE_ONCE(dio->waiter, NULL);
-                       wake_up_process(waiter);
+                       blk_wake_io_task(waiter);
                }
        }
 

diff --git a/fs/iomap.c b/fs/iomap.c
index f61d13dfdf09583403e8402962ce373d68c03d19..b0462b363badff9dc3896cf8c478c6afd1ed4bec 100644 (file)
--- a/fs/iomap.c
+++ b/fs/iomap.c
@@ -1525,7 +1525,7 @@ static void iomap_dio_bio_end_io(struct bio *bio)
                if (dio->wait_for_completion) {
                        struct task_struct *waiter = dio->submit.waiter;
                        WRITE_ONCE(dio->submit.waiter, NULL);
-                       wake_up_process(waiter);
+                       blk_wake_io_task(waiter);
                } else if (dio->flags & IOMAP_DIO_WRITE) {
                        struct inode *inode = file_inode(dio->iocb->ki_filp);
 

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 41aaa05e42c1e2798b54d77fda7b80d5cc75bd83..91c44f7a7f62b700056e3cf17a08ef82ffdb954e 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1772,4 +1772,17 @@ static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
 
 #endif /* CONFIG_BLOCK */
 
+static inline void blk_wake_io_task(struct task_struct *waiter)
+{
+       /*
+        * If we're polling, the task itself is doing the completions. For
+        * that case, we don't need to signal a wakeup, it's enough to just
+        * mark us as RUNNING.
+        */
+       if (waiter == current)
+               __set_current_state(TASK_RUNNING);
+       else
+               wake_up_process(waiter);
+}
+
 #endif

diff --git a/mm/page_io.c b/mm/page_io.c
index d4d1c89bcdddcef43dfa04fa02926f21fba228fb..57572ff46016dde4945622ead210593d183e3806 100644 (file)
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -140,7 +140,7 @@ out:
        unlock_page(page);
        WRITE_ONCE(bio->bi_private, NULL);
        bio_put(bio);
-       wake_up_process(waiter);
+       blk_wake_io_task(waiter);
        put_task_struct(waiter);
 }