io_uring: optimise out req->opcode reloading

Looking at the assembly, the compiler decided to reload req->opcode in
io_op_defs[opcode].needs_file instead of one it had in a register, so
store it in a temp variable so it can be optimised out. Also move the
personality block later, it's better for spilling/etc. as it only
depends on @sqe, which we're keeping anyway.

By the way, zero req->opcode if it over IORING_OP_LAST, not a problem,
at the moment but is safer.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/6ba869f5f8b7b0f991c87fdf089f0abf87cbe06b.1633532552.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 04d6e35ea0df73f9d5fe186cbef38954e463958e..7918a320104d16c2dd5051f8dcce346b677a173d 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -6975,9 +6975,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
 {
        unsigned int sqe_flags;
        int personality;
+       u8 opcode;
 
        /* req is partially pre-initialised, see io_preinit_req() */
-       req->opcode = READ_ONCE(sqe->opcode);
+       req->opcode = opcode = READ_ONCE(sqe->opcode);
        /* same numerical values with corresponding REQ_F_*, safe to copy */
        req->flags = sqe_flags = READ_ONCE(sqe->flags);
        req->user_data = READ_ONCE(sqe->user_data);
@@ -6985,14 +6986,16 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
        req->fixed_rsrc_refs = NULL;
        req->task = current;
 
-       if (unlikely(req->opcode >= IORING_OP_LAST))
+       if (unlikely(opcode >= IORING_OP_LAST)) {
+               req->opcode = 0;
                return -EINVAL;
+       }
        if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) {
                /* enforce forwards compatibility on users */
                if (sqe_flags & ~SQE_VALID_FLAGS)
                        return -EINVAL;
                if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
-                   !io_op_defs[req->opcode].buffer_select)
+                   !io_op_defs[opcode].buffer_select)
                        return -EOPNOTSUPP;
                if (sqe_flags & IOSQE_IO_DRAIN)
                        io_init_req_drain(req);
@@ -7011,23 +7014,14 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
                }
        }
 
-       personality = READ_ONCE(sqe->personality);
-       if (personality) {
-               req->creds = xa_load(&ctx->personalities, personality);
-               if (!req->creds)
-                       return -EINVAL;
-               get_cred(req->creds);
-               req->flags |= REQ_F_CREDS;
-       }
-
-       if (io_op_defs[req->opcode].needs_file) {
+       if (io_op_defs[opcode].needs_file) {
                struct io_submit_state *state = &ctx->submit_state;
 
                /*
                 * Plug now if we have more than 2 IO left after this, and the
                 * target is potentially a read/write to block based storage.
                 */
-               if (state->need_plug && io_op_defs[req->opcode].plug) {
+               if (state->need_plug && io_op_defs[opcode].plug) {
                        state->plug_started = true;
                        state->need_plug = false;
                        blk_start_plug(&state->plug);
@@ -7039,6 +7033,15 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
                        return -EBADF;
        }
 
+       personality = READ_ONCE(sqe->personality);
+       if (personality) {
+               req->creds = xa_load(&ctx->personalities, personality);
+               if (!req->creds)
+                       return -EINVAL;
+               get_cred(req->creds);
+               req->flags |= REQ_F_CREDS;
+       }
+
        return io_req_prep(req, sqe);
 }