block: add a struct io_comp_batch argument to fops->iopoll()

struct io_comp_batch contains a list head and a completion handler, which
will allow completions to more effciently completed batches of IO.

For now, no functional changes in this patch, we just define the
io_comp_batch structure and add the argument to the file_operations iopoll
handler.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-core.c b/block/blk-core.c
index 20b6cc06461a60372bf13013c2906d66f2ed0ae1..d0c2e11411d0382c848c6c00c3f7bbf1780ab250 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1078,7 +1078,7 @@ EXPORT_SYMBOL(submit_bio);
  * Note: the caller must either be the context that submitted @bio, or
  * be in a RCU critical section to prevent freeing of @bio.
  */
-int bio_poll(struct bio *bio, unsigned int flags)
+int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags)
 {
        struct request_queue *q = bio->bi_bdev->bd_disk->queue;
        blk_qc_t cookie = READ_ONCE(bio->bi_cookie);
@@ -1096,7 +1096,7 @@ int bio_poll(struct bio *bio, unsigned int flags)
        if (WARN_ON_ONCE(!queue_is_mq(q)))
                ret = 0;        /* not yet implemented, should not happen */
        else
-               ret = blk_mq_poll(q, cookie, flags);
+               ret = blk_mq_poll(q, cookie, iob, flags);
        blk_queue_exit(q);
        return ret;
 }
@@ -1106,7 +1106,8 @@ EXPORT_SYMBOL_GPL(bio_poll);
  * Helper to implement file_operations.iopoll.  Requires the bio to be stored
  * in iocb->private, and cleared before freeing the bio.
  */
-int iocb_bio_iopoll(struct kiocb *kiocb, unsigned int flags)
+int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
+                   unsigned int flags)
 {
        struct bio *bio;
        int ret = 0;
@@ -1134,7 +1135,7 @@ int iocb_bio_iopoll(struct kiocb *kiocb, unsigned int flags)
        rcu_read_lock();
        bio = READ_ONCE(kiocb->private);
        if (bio && bio->bi_bdev)
-               ret = bio_poll(bio, flags);
+               ret = bio_poll(bio, iob, flags);
        rcu_read_unlock();
 
        return ret;

diff --git a/block/blk-exec.c b/block/blk-exec.c
index 55f0cd34b37b9906cbb05866f461627e44ddb7f4..1b8b47f6e79bbccb612c53c8d047d0af9d57cfb7 100644 (file)
--- a/block/blk-exec.c
+++ b/block/blk-exec.c
@@ -77,7 +77,7 @@ static bool blk_rq_is_poll(struct request *rq)
 static void blk_rq_poll_completion(struct request *rq, struct completion *wait)
 {
        do {
-               bio_poll(rq->bio, 0);
+               bio_poll(rq->bio, NULL, 0);
                cond_resched();
        } while (!completion_done(wait));
 }

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 74505b545dd332bd955022e451fa67dea1560bec..79c25b64e8b090f0f153404f12ed591923a6d9d1 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -4174,14 +4174,14 @@ static bool blk_mq_poll_hybrid(struct request_queue *q, blk_qc_t qc)
 }
 
 static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
-               unsigned int flags)
+                              struct io_comp_batch *iob, unsigned int flags)
 {
        struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, cookie);
        long state = get_current_state();
        int ret;
 
        do {
-               ret = q->mq_ops->poll(hctx);
+               ret = q->mq_ops->poll(hctx, iob);
                if (ret > 0) {
                        __set_current_state(TASK_RUNNING);
                        return ret;
@@ -4201,14 +4201,15 @@ static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
        return 0;
 }
 
-int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, unsigned int flags)
+int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, struct io_comp_batch *iob,
+               unsigned int flags)
 {
        if (!(flags & BLK_POLL_NOSLEEP) &&
            q->poll_nsec != BLK_MQ_POLL_CLASSIC) {
                if (blk_mq_poll_hybrid(q, cookie))
                        return 1;
        }
-       return blk_mq_poll_classic(q, cookie, flags);
+       return blk_mq_poll_classic(q, cookie, iob, flags);
 }
 
 unsigned int blk_mq_rq_cpu(struct request *rq)

diff --git a/block/blk-mq.h b/block/blk-mq.h
index 1b91a3fdaa01816dd26dfadd22c5207141211fe8..ebf67f4d4f2ed09bd9f674dd0b514f30652ac8c8 100644 (file)
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -31,7 +31,8 @@ struct blk_mq_ctx {
 } ____cacheline_aligned_in_smp;
 
 void blk_mq_submit_bio(struct bio *bio);
-int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, unsigned int flags);
+int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, struct io_comp_batch *iob,
+               unsigned int flags);
 void blk_mq_exit_queue(struct request_queue *q);
 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
 void blk_mq_wake_waiters(struct request_queue *q);

diff --git a/block/fops.c b/block/fops.c
index 1d4f862950bbbe0b85e38e2e6f8d1dd5b8f80b53..2c43e493e37c656f943252073b081ada0035672b 100644 (file)
--- a/block/fops.c
+++ b/block/fops.c
@@ -105,7 +105,7 @@ static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,
                set_current_state(TASK_UNINTERRUPTIBLE);
                if (!READ_ONCE(bio.bi_private))
                        break;
-               if (!(iocb->ki_flags & IOCB_HIPRI) || !bio_poll(&bio, 0))
+               if (!(iocb->ki_flags & IOCB_HIPRI) || !bio_poll(&bio, NULL, 0))
                        blk_io_schedule();
        }
        __set_current_state(TASK_RUNNING);
@@ -291,7 +291,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
                if (!READ_ONCE(dio->waiter))
                        break;
 
-               if (!do_poll || !bio_poll(bio, 0))
+               if (!do_poll || !bio_poll(bio, NULL, 0))
                        blk_io_schedule();
        }
        __set_current_state(TASK_RUNNING);

diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c
index bd4a41afbbfc983e5decf8f38a20ae785d5f0d16..0ec0191d41969ba22e1ee8e9e16804e20be7edc3 100644 (file)
--- a/drivers/block/rnbd/rnbd-clt.c
+++ b/drivers/block/rnbd/rnbd-clt.c
@@ -1176,7 +1176,7 @@ static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx,
        return ret;
 }
 
-static int rnbd_rdma_poll(struct blk_mq_hw_ctx *hctx)
+static int rnbd_rdma_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 {
        struct rnbd_queue *q = hctx->driver_data;
        struct rnbd_clt_dev *dev = q->dev;

diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 896328271471b2e334de3f887a3b37e6d7464e24..bb0482dfab3c0b6fb96960aaea02d65aca5a4268 100644 (file)
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -1092,7 +1092,7 @@ static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
        enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
 }
 
-static int nvme_poll(struct blk_mq_hw_ctx *hctx)
+static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 {
        struct nvme_queue *nvmeq = hctx->driver_data;
        bool found;
@@ -1274,7 +1274,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
         * Did we miss an interrupt?
         */
        if (test_bit(NVMEQ_POLLED, &nvmeq->flags))
-               nvme_poll(req->mq_hctx);
+               nvme_poll(req->mq_hctx, NULL);
        else
                nvme_poll_irqdisable(nvmeq);
 

diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 40317e1b918353913b9b90093e252261ba77d2c7..1624da3702d43286376cfea9a6e730a1534b730f 100644 (file)
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -2106,7 +2106,7 @@ unmap_qe:
        return ret;
 }
 
-static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx)
+static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 {
        struct nvme_rdma_queue *queue = hctx->driver_data;
 

diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 3c1c29dd30207cd6fbf337a67aa72e8ef34cfddd..9ce3458ee1dd1468b4f689939c14910e804ac61e 100644 (file)
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -2429,7 +2429,7 @@ static int nvme_tcp_map_queues(struct blk_mq_tag_set *set)
        return 0;
 }
 
-static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx)
+static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 {
        struct nvme_tcp_queue *queue = hctx->driver_data;
        struct sock *sk = queue->sock->sk;

diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 33fd9a01330ce4607d7016d31e1a80734e63a752..30f7d0b4eb732c3726d570c4affac9deeefc37c0 100644 (file)
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -1784,7 +1784,7 @@ static void scsi_mq_exit_request(struct blk_mq_tag_set *set, struct request *rq,
 }
 
 
-static int scsi_mq_poll(struct blk_mq_hw_ctx *hctx)
+static int scsi_mq_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 {
        struct Scsi_Host *shost = hctx->driver_data;
 

diff --git a/fs/io_uring.c b/fs/io_uring.c
index c5066146b8de39228121e73dd3cf7a14f8f27dd4..cd77a137f2d8658fd4b922e18fa20ca2ddcfdc76 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2483,7 +2483,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
                if (!list_empty(&done))
                        break;
 
-               ret = kiocb->ki_filp->f_op->iopoll(kiocb, poll_flags);
+               ret = kiocb->ki_filp->f_op->iopoll(kiocb, NULL, poll_flags);
                if (unlikely(ret < 0))
                        return ret;
                else if (ret)

diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index 8efab177011dd9317c79ee07476fae64e3d16d63..83ecfba53abe1c7467464e683d0cb92a35f1488d 100644 (file)
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -630,7 +630,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
                                break;
 
                        if (!dio->submit.poll_bio ||
-                           !bio_poll(dio->submit.poll_bio, 0))
+                           !bio_poll(dio->submit.poll_bio, NULL, 0))
                                blk_io_schedule();
                }
                __set_current_state(TASK_RUNNING);

diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 9fb8618fb957ddd875570bb70ab4399b0e52b4d2..4c79439af2f29c142f6d96f8f2a1ee27ef5fb132 100644 (file)
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -532,7 +532,7 @@ struct blk_mq_ops {
        /**
         * @poll: Called to poll for completion of a specific tag.
         */
-       int (*poll)(struct blk_mq_hw_ctx *);
+       int (*poll)(struct blk_mq_hw_ctx *, struct io_comp_batch *);
 
        /**
         * @complete: Mark the request as complete.

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index b0a322172965591d671d7b36fde37ac30793826b..fd9771a1da096f6ba4c920633d3e5403aaada218 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -569,8 +569,9 @@ blk_status_t errno_to_blk_status(int errno);
 #define BLK_POLL_ONESHOT               (1 << 0)
 /* do not sleep to wait for the expected completion time */
 #define BLK_POLL_NOSLEEP               (1 << 1)
-int bio_poll(struct bio *bio, unsigned int flags);
-int iocb_bio_iopoll(struct kiocb *kiocb, unsigned int flags);
+int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags);
+int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
+                       unsigned int flags);
 
 static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
 {
@@ -1298,6 +1299,14 @@ int fsync_bdev(struct block_device *bdev);
 int freeze_bdev(struct block_device *bdev);
 int thaw_bdev(struct block_device *bdev);
 
+struct io_comp_batch {
+       struct request *req_list;
+       bool need_ts;
+       void (*complete)(struct io_comp_batch *);
+};
+
+#define DEFINE_IO_COMP_BATCH(name)     struct io_comp_batch name = { }
+
 #define rq_list_add(listptr, rq)       do {            \
        (rq)->rq_next = *(listptr);                     \
        *(listptr) = rq;                                \

diff --git a/include/linux/fs.h b/include/linux/fs.h
index f595f4097cb7575671989fc8b9eb5c780b864495..31029a91f440e23ffc211aa84be0853a2266263c 100644 (file)
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -48,6 +48,7 @@
 struct backing_dev_info;
 struct bdi_writeback;
 struct bio;
+struct io_comp_batch;
 struct export_operations;
 struct fiemap_extent_info;
 struct hd_geometry;
@@ -2071,7 +2072,8 @@ struct file_operations {
        ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
        ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
        ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
-       int (*iopoll)(struct kiocb *kiocb, unsigned int flags);
+       int (*iopoll)(struct kiocb *kiocb, struct io_comp_batch *,
+                       unsigned int flags);
        int (*iterate) (struct file *, struct dir_context *);
        int (*iterate_shared) (struct file *, struct dir_context *);
        __poll_t (*poll) (struct file *, struct poll_table_struct *);

diff --git a/mm/page_io.c b/mm/page_io.c
index a68faab5b3105bf19a8339f626c0d06cfcc62c5f..6010fb07f23195f4c9cf46abb846dd9bb9f9c5a6 100644 (file)
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -424,7 +424,7 @@ int swap_readpage(struct page *page, bool synchronous)
                if (!READ_ONCE(bio->bi_private))
                        break;
 
-               if (!bio_poll(bio, 0))
+               if (!bio_poll(bio, NULL, 0))
                        blk_io_schedule();
        }
        __set_current_state(TASK_RUNNING);