io_uring: refactor io_req_find_next

Move the fast path from io_req_find_next() into callers. It prepares us
for further changes.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/10bd0e564472dde0c7f8d90ae317d05356cd565a.1647897811.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index f61490aee3d3ca0ae95e5eebd61149935551553e..1859fff53ddfd7d0d63ee3f85ec839cf5f73f88e 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2370,8 +2370,6 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
 {
        struct io_kiocb *nxt;
 
-       if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))))
-               return NULL;
        /*
         * If LINK is set, we have dependent requests in this chain. If we
         * didn't fail this request, queue the first one up, moving any other
@@ -2598,10 +2596,12 @@ static void io_req_task_queue_reissue(struct io_kiocb *req)
 
 static inline void io_queue_next(struct io_kiocb *req)
 {
-       struct io_kiocb *nxt = io_req_find_next(req);
+       if (unlikely(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))) {
+               struct io_kiocb *nxt = io_req_find_next(req);
 
-       if (nxt)
-               io_req_task_queue(nxt);
+               if (nxt)
+                       io_req_task_queue(nxt);
+       }
 }
 
 static void io_free_req(struct io_kiocb *req)
@@ -2695,7 +2695,8 @@ static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
        struct io_kiocb *nxt = NULL;
 
        if (req_ref_put_and_test(req)) {
-               nxt = io_req_find_next(req);
+               if (unlikely(req->flags & (REQ_F_LINK|REQ_F_HARDLINK)))
+                       nxt = io_req_find_next(req);
                __io_free_req(req);
        }
        return nxt;