io_uring: avoid ring quiesce while registering async eventfd

This is done using the RCU data structure (io_ev_fd). eventfd_async is
moved from io_ring_ctx to io_ev_fd which is RCU protected hence avoiding
ring quiesce which is much more expensive than an RCU lock. The place
where eventfd_async is read is already under rcu_read_lock so there is no
extra RCU read-side critical section needed.

Signed-off-by: Usama Arif <usama.arif@bytedance.com>
Link: https://lore.kernel.org/r/20220204145117.1186568-4-usama.arif@bytedance.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 6c35e88be4ea576bb0b7ea49788bf20951ca7d64..961866b24b4f2614660091b0d2a9675ba33f7b2a 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -328,6 +328,7 @@ struct io_submit_state {
 
 struct io_ev_fd {
        struct eventfd_ctx      *cq_ev_fd;
+       unsigned int            eventfd_async: 1;
        struct rcu_head         rcu;
 };
 
@@ -340,7 +341,6 @@ struct io_ring_ctx {
                unsigned int            flags;
                unsigned int            compat: 1;
                unsigned int            drain_next: 1;
-               unsigned int            eventfd_async: 1;
                unsigned int            restricted: 1;
                unsigned int            off_timeout_used: 1;
                unsigned int            drain_active: 1;
@@ -1756,7 +1756,7 @@ static void io_eventfd_signal(struct io_ring_ctx *ctx)
        if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
                goto out;
 
-       if (!ctx->eventfd_async || io_wq_current_is_worker())
+       if (!ev_fd->eventfd_async || io_wq_current_is_worker())
                eventfd_signal(ev_fd->cq_ev_fd, 1);
 
 out:
@@ -9385,7 +9385,8 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
        return done ? done : err;
 }
 
-static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
+static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
+                              unsigned int eventfd_async)
 {
        struct io_ev_fd *ev_fd;
        __s32 __user *fds = arg;
@@ -9409,6 +9410,7 @@ static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
                kfree(ev_fd);
                return ret;
        }
+       ev_fd->eventfd_async = eventfd_async;
 
        rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
        return ret;
@@ -11014,6 +11016,7 @@ static bool io_register_op_must_quiesce(int op)
        case IORING_UNREGISTER_FILES:
        case IORING_REGISTER_FILES_UPDATE:
        case IORING_REGISTER_EVENTFD:
+       case IORING_REGISTER_EVENTFD_ASYNC:
        case IORING_UNREGISTER_EVENTFD:
        case IORING_REGISTER_PROBE:
        case IORING_REGISTER_PERSONALITY:
@@ -11114,17 +11117,16 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
                ret = io_register_files_update(ctx, arg, nr_args);
                break;
        case IORING_REGISTER_EVENTFD:
-       case IORING_REGISTER_EVENTFD_ASYNC:
                ret = -EINVAL;
                if (nr_args != 1)
                        break;
-               ret = io_eventfd_register(ctx, arg);
-               if (ret)
+               ret = io_eventfd_register(ctx, arg, 0);
+               break;
+       case IORING_REGISTER_EVENTFD_ASYNC:
+               ret = -EINVAL;
+               if (nr_args != 1)
                        break;
-               if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
-                       ctx->eventfd_async = 1;
-               else
-                       ctx->eventfd_async = 0;
+               ret = io_eventfd_register(ctx, arg, 1);
                break;
        case IORING_UNREGISTER_EVENTFD:
                ret = -EINVAL;