io_uring/cmd: add iovec cache for commands

Add iou_vec to commands and wire caching for it, but don't expose it to
users just yet. We need the vec cleared on initial alloc, but since
we can't place it at the beginning at the moment, zero the entire
async_data. It's cached, and the performance effects only the initial
allocation, and it might be not a bad idea since we're exposing those
bits to outside drivers.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/c0f2145b75791bc6106eb4e72add2cf6a2c72a7a.1742579999.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 5eb9be063a7c90d24c341554c7f91c49795032e6..e1128b9551aa870fc23be161ee005172850b93e0 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -289,7 +289,7 @@ static void io_free_alloc_caches(struct io_ring_ctx *ctx)
        io_alloc_cache_free(&ctx->apoll_cache, kfree);
        io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
        io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free);
-       io_alloc_cache_free(&ctx->cmd_cache, kfree);
+       io_alloc_cache_free(&ctx->cmd_cache, io_cmd_cache_free);
        io_alloc_cache_free(&ctx->msg_cache, kfree);
        io_futex_cache_free(ctx);
        io_rsrc_cache_free(ctx);
@@ -335,7 +335,8 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
                            sizeof(struct io_async_rw),
                            offsetof(struct io_async_rw, clear));
        ret |= io_alloc_cache_init(&ctx->cmd_cache, IO_ALLOC_CACHE_MAX,
-                           sizeof(struct io_async_cmd), 0);
+                           sizeof(struct io_async_cmd),
+                           sizeof(struct io_async_cmd));
        spin_lock_init(&ctx->msg_lock);
        ret |= io_alloc_cache_init(&ctx->msg_cache, IO_ALLOC_CACHE_MAX,
                            sizeof(struct io_kiocb), 0);

diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index e4aa61a414fbe95e8469b7ecdcd4dcf38ebe7f0c..489384c0438bd8b07be9cb8467fe6e0f82227cd5 100644 (file)
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -755,6 +755,7 @@ const struct io_cold_def io_cold_defs[] = {
        },
        [IORING_OP_URING_CMD] = {
                .name                   = "URING_CMD",
+               .cleanup                = io_uring_cmd_cleanup,
        },
        [IORING_OP_SEND_ZC] = {
                .name                   = "SEND_ZC",

diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 7c126ee497ea8f28cc270661c1ff249c66b15051..6a21cdaaf49577dd5a64e01d05ce185b913b5570 100644 (file)
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -16,6 +16,14 @@
 #include "rsrc.h"
 #include "uring_cmd.h"
 
+void io_cmd_cache_free(const void *entry)
+{
+       struct io_async_cmd *ac = (struct io_async_cmd *)entry;
+
+       io_vec_free(&ac->vec);
+       kfree(ac);
+}
+
 static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)
 {
        struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
@@ -29,13 +37,23 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)
 
        if (issue_flags & IO_URING_F_UNLOCKED)
                return;
+
+       io_alloc_cache_vec_kasan(&ac->vec);
+       if (ac->vec.nr > IO_VEC_CACHE_SOFT_CAP)
+               io_vec_free(&ac->vec);
+
        if (io_alloc_cache_put(&req->ctx->cmd_cache, cache)) {
                ioucmd->sqe = NULL;
                req->async_data = NULL;
-               req->flags &= ~REQ_F_ASYNC_DATA;
+               req->flags &= ~(REQ_F_ASYNC_DATA|REQ_F_NEED_CLEANUP);
        }
 }
 
+void io_uring_cmd_cleanup(struct io_kiocb *req)
+{
+       io_req_uring_cleanup(req, 0);
+}
+
 bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
                                   struct io_uring_task *tctx, bool cancel_all)
 {

diff --git a/io_uring/uring_cmd.h b/io_uring/uring_cmd.h
index 2ec3a8785534631339e2fe1ada966b90eb68a02c..b45ec7cffcd10b1e4682f34126f3b3dbc701ab7b 100644 (file)
--- a/io_uring/uring_cmd.h
+++ b/io_uring/uring_cmd.h
@@ -1,13 +1,18 @@
 // SPDX-License-Identifier: GPL-2.0
 
 #include <linux/io_uring/cmd.h>
+#include <linux/io_uring_types.h>
 
 struct io_async_cmd {
        struct io_uring_cmd_data        data;
+       struct iou_vec                  vec;
 };
 
 int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags);
 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
+void io_uring_cmd_cleanup(struct io_kiocb *req);
 
 bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
                                   struct io_uring_task *tctx, bool cancel_all);
+
+void io_cmd_cache_free(const void *entry);