io_uring/rw: implement vectored registered rw

Implement registered buffer vectored reads with new opcodes
IORING_OP_WRITEV_FIXED and IORING_OP_READV_FIXED.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/d7c89eb481e870f598edc91cc66ff4d1e4ae3788.1741362889.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 3d99bf9bbf611002d06ae78952dc06e6af89277d..9e5eec7490bb579df09eaf60898d0f3bb3197a14 100644 (file)
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -281,6 +281,8 @@ enum io_uring_op {
        IORING_OP_LISTEN,
        IORING_OP_RECV_ZC,
        IORING_OP_EPOLL_WAIT,
+       IORING_OP_READV_FIXED,
+       IORING_OP_WRITEV_FIXED,
 
        /* this goes last, obviously */
        IORING_OP_LAST,

diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index db77df513d55b1f9aeaf138d7b7008c277a56b85..7fd173197b1e0621b217474df869bb564969c1ec 100644 (file)
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -540,6 +540,35 @@ const struct io_issue_def io_issue_defs[] = {
                .prep                   = io_eopnotsupp_prep,
 #endif
        },
+       [IORING_OP_READV_FIXED] = {
+               .needs_file             = 1,
+               .unbound_nonreg_file    = 1,
+               .pollin                 = 1,
+               .plug                   = 1,
+               .audit_skip             = 1,
+               .ioprio                 = 1,
+               .iopoll                 = 1,
+               .iopoll_queue           = 1,
+               .vectored               = 1,
+               .async_size             = sizeof(struct io_async_rw),
+               .prep                   = io_prep_readv_fixed,
+               .issue                  = io_read,
+       },
+       [IORING_OP_WRITEV_FIXED] = {
+               .needs_file             = 1,
+               .hash_reg_file          = 1,
+               .unbound_nonreg_file    = 1,
+               .pollout                = 1,
+               .plug                   = 1,
+               .audit_skip             = 1,
+               .ioprio                 = 1,
+               .iopoll                 = 1,
+               .iopoll_queue           = 1,
+               .vectored               = 1,
+               .async_size             = sizeof(struct io_async_rw),
+               .prep                   = io_prep_writev_fixed,
+               .issue                  = io_write,
+       },
 };
 
 const struct io_cold_def io_cold_defs[] = {
@@ -775,6 +804,16 @@ const struct io_cold_def io_cold_defs[] = {
        [IORING_OP_EPOLL_WAIT] = {
                .name                   = "EPOLL_WAIT",
        },
+       [IORING_OP_READV_FIXED] = {
+               .name                   = "READV_FIXED",
+               .cleanup                = io_readv_writev_cleanup,
+               .fail                   = io_rw_fail,
+       },
+       [IORING_OP_WRITEV_FIXED] = {
+               .name                   = "WRITEV_FIXED",
+               .cleanup                = io_readv_writev_cleanup,
+               .fail                   = io_rw_fail,
+       },
 };
 
 const char *io_uring_get_opcode(u8 opcode)

diff --git a/io_uring/rw.c b/io_uring/rw.c
index ad7f647d48e9c6a5c5e437e0ddf364beacb84f74..4c4229f41aaa1ad0d44d64038c8fb97bd4b2113d 100644 (file)
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -381,6 +381,57 @@ int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
        return __io_prep_rw(req, sqe, ITER_SOURCE);
 }
 
+static int io_rw_prep_reg_vec(struct io_kiocb *req, int ddir)
+{
+       struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+       struct io_async_rw *io = req->async_data;
+       const struct iovec __user *uvec;
+       size_t uvec_segs = rw->len;
+       struct iovec *iov;
+       int iovec_off, ret;
+       void *res;
+
+       if (uvec_segs > io->vec.nr) {
+               ret = io_vec_realloc(&io->vec, uvec_segs);
+               if (ret)
+                       return ret;
+               req->flags |= REQ_F_NEED_CLEANUP;
+       }
+       /* pad iovec to the right */
+       iovec_off = io->vec.nr - uvec_segs;
+       iov = io->vec.iovec + iovec_off;
+       uvec = u64_to_user_ptr(rw->addr);
+       res = iovec_from_user(uvec, uvec_segs, uvec_segs, iov,
+                             io_is_compat(req->ctx));
+       if (IS_ERR(res))
+               return PTR_ERR(res);
+
+       ret = io_import_reg_vec(ddir, &io->iter, req, &io->vec,
+                               uvec_segs, iovec_off, 0);
+       iov_iter_save_state(&io->iter, &io->iter_state);
+       return ret;
+}
+
+int io_prep_readv_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+{
+       int ret;
+
+       ret = __io_prep_rw(req, sqe, ITER_DEST);
+       if (unlikely(ret))
+               return ret;
+       return io_rw_prep_reg_vec(req, ITER_DEST);
+}
+
+int io_prep_writev_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+{
+       int ret;
+
+       ret = __io_prep_rw(req, sqe, ITER_SOURCE);
+       if (unlikely(ret))
+               return ret;
+       return io_rw_prep_reg_vec(req, ITER_SOURCE);
+}
+
 /*
  * Multishot read is prepared just like a normal read/write request, only
  * difference is that we set the MULTISHOT flag.

diff --git a/io_uring/rw.h b/io_uring/rw.h
index 529fd2f96a7fb123506f233efd0d9047bc49e176..81d6d9a8cf6937ace677aaf1e27071858b728fc6 100644 (file)
--- a/io_uring/rw.h
+++ b/io_uring/rw.h
@@ -32,6 +32,8 @@ struct io_async_rw {
 
 int io_prep_read_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe);
+int io_prep_readv_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe);
+int io_prep_writev_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 int io_prep_readv(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 int io_prep_writev(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 int io_prep_read(struct io_kiocb *req, const struct io_uring_sqe *sqe);