io_uring: track link's head and tail during submit

Explicitly save not only a link's head in io_submit_sqe[s]() but the
tail as well. That's in preparation for keeping linked requests in a
singly linked list.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 10cfb6d179942910fb0ef4177cc196203f355669..0441185ac51009dd5b01c4e7a64e762f45d1e3da 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -6536,8 +6536,13 @@ static inline void io_queue_link_head(struct io_kiocb *req,
                io_queue_sqe(req, NULL, cs);
 }
 
+struct io_submit_link {
+       struct io_kiocb *head;
+       struct io_kiocb *last;
+};
+
 static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
-                        struct io_kiocb **link, struct io_comp_state *cs)
+                        struct io_submit_link *link, struct io_comp_state *cs)
 {
        struct io_ring_ctx *ctx = req->ctx;
        int ret;
@@ -6549,8 +6554,8 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
         * submitted sync once the chain is complete. If none of those
         * conditions are true (normal request), then just queue it.
         */
-       if (*link) {
-               struct io_kiocb *head = *link;
+       if (link->head) {
+               struct io_kiocb *head = link->head;
 
                /*
                 * Taking sequential execution of a link, draining both sides
@@ -6571,11 +6576,12 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
                }
                trace_io_uring_link(ctx, req, head);
                list_add_tail(&req->link_list, &head->link_list);
+               link->last = req;
 
                /* last request of a link, enqueue the link */
                if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
                        io_queue_link_head(head, cs);
-                       *link = NULL;
+                       link->head = NULL;
                }
        } else {
                if (unlikely(ctx->drain_next)) {
@@ -6589,7 +6595,8 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
                        ret = io_req_defer_prep(req, sqe);
                        if (unlikely(ret))
                                req->flags |= REQ_F_FAIL_LINK;
-                       *link = req;
+                       link->head = req;
+                       link->last = req;
                } else {
                        io_queue_sqe(req, sqe, cs);
                }
@@ -6769,7 +6776,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
 static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
 {
        struct io_submit_state state;
-       struct io_kiocb *link = NULL;
+       struct io_submit_link link;
        int i, submitted = 0;
 
        /* if we have a backlog and couldn't flush it all, return BUSY */
@@ -6789,6 +6796,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
        refcount_add(nr, &current->usage);
 
        io_submit_state_start(&state, ctx, nr);
+       link.head = NULL;
 
        for (i = 0; i < nr; i++) {
                const struct io_uring_sqe *sqe;
@@ -6834,8 +6842,8 @@ fail_req:
                percpu_counter_sub(&tctx->inflight, unused);
                put_task_struct_many(current, unused);
        }
-       if (link)
-               io_queue_link_head(link, &state.comp);
+       if (link.head)
+               io_queue_link_head(link.head, &state.comp);
        io_submit_state_end(&state);
 
         /* Commit SQ ring head once we've consumed and submitted all SQEs */