io_uring/rsrc: split io_kiocb node type assignments

Currently the io_rsrc_node assignment in io_kiocb is an array of two
pointers, as two nodes may be assigned to a request - one file node,
and one buffer node. However, the buffer node can co-exist with the
provided buffers, as currently it's not supported to use both provided
and registered buffers at the same time.

This crucially brings struct io_kiocb down to 4 cache lines again, as
before it spilled into the 5th cacheline.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index d52fec533c516a299917be26a67696d6a111f988..01e7fb9fcfe2a22db6c93959270bf07d3574c963 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -475,6 +475,7 @@ enum {
        REQ_F_BL_EMPTY_BIT,
        REQ_F_BL_NO_RECYCLE_BIT,
        REQ_F_BUFFERS_COMMIT_BIT,
+       REQ_F_BUF_NODE_BIT,
 
        /* not a real bit, just to check we're not overflowing the space */
        __REQ_F_LAST_BIT,
@@ -553,6 +554,8 @@ enum {
        REQ_F_BL_NO_RECYCLE     = IO_REQ_FLAG(REQ_F_BL_NO_RECYCLE_BIT),
        /* buffer ring head needs incrementing on put */
        REQ_F_BUFFERS_COMMIT    = IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT),
+       /* buf node is valid */
+       REQ_F_BUF_NODE          = IO_REQ_FLAG(REQ_F_BUF_NODE_BIT),
 };
 
 typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);
@@ -633,6 +636,8 @@ struct io_kiocb {
                 * REQ_F_BUFFER_RING is set.
                 */
                struct io_buffer_list   *buf_list;
+
+               struct io_rsrc_node     *buf_node;
        };
 
        union {
@@ -642,7 +647,7 @@ struct io_kiocb {
                __poll_t apoll_events;
        };
 
-       struct io_rsrc_node             *rsrc_nodes[2];
+       struct io_rsrc_node             *file_node;
 
        atomic_t                        refs;
        bool                            cancel_seq_set;

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index f08ea7fd59986a18fd1f3abdf76bd92c6f63eceb..5bab8a3b04566a79dad94984345abefe4e412e49 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -948,8 +948,8 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res)
 static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
 {
        req->ctx = ctx;
-       req->rsrc_nodes[IORING_RSRC_FILE] = NULL;
-       req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
+       req->buf_node = NULL;
+       req->file_node = NULL;
        req->link = NULL;
        req->async_data = NULL;
        /* not necessary, but safer to zero */
@@ -1882,7 +1882,7 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
        io_ring_submit_lock(ctx, issue_flags);
        node = io_rsrc_node_lookup(&ctx->file_table.data, fd);
        if (node) {
-               io_req_assign_rsrc_node(req, node);
+               io_req_assign_rsrc_node(&req->file_node, node);
                req->flags |= io_slot_flags(node);
                file = io_slot_file(node);
        }

diff --git a/io_uring/net.c b/io_uring/net.c
index 2f7b334ed70885e69e94ecf434a564e870f00a67..2ccc2b4094310f93dcb732e77ca8eff5ddaab0ef 100644 (file)
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -1348,7 +1348,8 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags)
                io_ring_submit_lock(ctx, issue_flags);
                node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index);
                if (node) {
-                       io_req_assign_rsrc_node(sr->notif, node);
+                       io_req_assign_rsrc_node(&sr->notif->buf_node, node);
+                       sr->notif->flags |= REQ_F_BUF_NODE;
                        ret = 0;
                }
                io_ring_submit_unlock(ctx, issue_flags);

diff --git a/io_uring/nop.c b/io_uring/nop.c
index 149dbdc53607a038c8520d6a52247ccf29f48f72..bc22bcc739f37d8e258ec77fbf0ab10c36048d15 100644 (file)
--- a/io_uring/nop.c
+++ b/io_uring/nop.c
@@ -67,7 +67,8 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags)
                io_ring_submit_lock(ctx, issue_flags);
                node = io_rsrc_node_lookup(&ctx->buf_table, nop->buffer);
                if (node) {
-                       io_req_assign_rsrc_node(req, node);
+                       io_req_assign_rsrc_node(&req->buf_node, node);
+                       req->flags |= REQ_F_BUF_NODE;
                        ret = 0;
                }
                io_ring_submit_unlock(ctx, issue_flags);

diff --git a/io_uring/notif.c b/io_uring/notif.c
index 4f02e969cf08c0eba24422ade878a4b059a67807..8dfbb0bd8e4d575f14898a5c7c45318c53dcd98f 100644 (file)
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -117,8 +117,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
        notif->file = NULL;
        notif->task = current;
        io_get_task_refs(1);
-       notif->rsrc_nodes[IORING_RSRC_FILE] = NULL;
-       notif->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
+       notif->file_node = NULL;
+       notif->buf_node = NULL;
 
        nd = io_notif_to_data(notif);
        nd->zc_report = false;

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 9a8fac31fa49d9bb7e5f8c85e0d397242b8b5e24..bc3a863b14bb97f7d6fdba5ae3b283840765b197 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -95,10 +95,14 @@ static inline bool io_reset_rsrc_node(struct io_rsrc_data *data, int index)
 
 static inline void io_req_put_rsrc_nodes(struct io_kiocb *req)
 {
-       io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_FILE]);
-       io_put_rsrc_node(req->rsrc_nodes[IORING_RSRC_BUFFER]);
-       req->rsrc_nodes[IORING_RSRC_FILE] = NULL;
-       req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
+       if (req->file_node) {
+               io_put_rsrc_node(req->file_node);
+               req->file_node = NULL;
+       }
+       if (req->flags & REQ_F_BUF_NODE) {
+               io_put_rsrc_node(req->buf_node);
+               req->buf_node = NULL;
+       }
 }
 
 static inline struct io_ring_ctx *io_rsrc_node_ctx(struct io_rsrc_node *node)
@@ -111,11 +115,11 @@ static inline int io_rsrc_node_type(struct io_rsrc_node *node)
        return node->ctx_ptr & IORING_RSRC_TYPE_MASK;
 }
 
-static inline void io_req_assign_rsrc_node(struct io_kiocb *req,
+static inline void io_req_assign_rsrc_node(struct io_rsrc_node **dst_node,
                                           struct io_rsrc_node *node)
 {
        node->refs++;
-       req->rsrc_nodes[io_rsrc_node_type(node)] = node;
+       *dst_node = node;
 }
 
 int io_files_update(struct io_kiocb *req, unsigned int issue_flags);

diff --git a/io_uring/rw.c b/io_uring/rw.c
index 1ea6be2ccc903a83bb6f7e5958e351e8af6b89ed..144730344c0f47a149e90454a91168b2efb86744 100644 (file)
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -341,7 +341,8 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe
        node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
        if (!node)
                return -EFAULT;
-       io_req_assign_rsrc_node(req, node);
+       io_req_assign_rsrc_node(&req->buf_node, node);
+       req->flags |= REQ_F_BUF_NODE;
 
        io = req->async_data;
        ret = io_import_fixed(ddir, &io->iter, node->buf, rw->addr, rw->len);

diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 535909a38e76333e7ec8f9f56bd46ae23f734b8a..88a73d21fc0bf09f752a8ef742df42acac9fb395 100644 (file)
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -219,7 +219,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
                 * being called. This prevents destruction of the mapped buffer
                 * we'll need at actual import time.
                 */
-               io_req_assign_rsrc_node(req, node);
+               io_req_assign_rsrc_node(&req->buf_node, node);
        }
        ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
 
@@ -275,7 +275,7 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
                              struct iov_iter *iter, void *ioucmd)
 {
        struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
-       struct io_rsrc_node *node = req->rsrc_nodes[IORING_RSRC_BUFFER];
+       struct io_rsrc_node *node = req->buf_node;
 
        /* Must have had rsrc_node assigned at prep time */
        if (node)