io_uring/rsrc: unify file and buffer resource tables

For files, there's nr_user_files/file_table/file_data, and buffers have
nr_user_bufs/user_bufs/buf_data. There's no reason why file_table and
file_data can't be the same thing, and ditto for the buffer side. That
gets rid of more io_ring_ctx state that's in two spots rather than just
being in one spot, as it should be. Put all the registered file data in
one locations, and ditto on the buffer front.

This also avoids having both io_rsrc_data->nodes being an allocated
array, and ->user_bufs[] or ->file_table.nodes. There's no reason to
have this information duplicated. Keep it in one spot, io_rsrc_data,
along with how many resources are available.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 696f2a05a98b26bb6f03defcce7ca8164229b481..77fd508d043ae6b29e2b142071bdc14b6dc7d3ed 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -55,8 +55,13 @@ struct io_wq_work {
        int cancel_seq;
 };
 
+struct io_rsrc_data {
+       unsigned int                    nr;
+       struct io_rsrc_node             **nodes;
+};
+
 struct io_file_table {
-       struct io_rsrc_node **nodes;
+       struct io_rsrc_data data;
        unsigned long *bitmap;
        unsigned int alloc_hint;
 };
@@ -276,9 +281,7 @@ struct io_ring_ctx {
                struct io_wq_work_list  iopoll_list;
 
                struct io_file_table    file_table;
-               struct io_rsrc_node     **user_bufs;
-               unsigned                nr_user_files;
-               unsigned                nr_user_bufs;
+               struct io_rsrc_data     buf_table;
 
                struct io_submit_state  submit_state;
 
@@ -366,10 +369,6 @@ struct io_ring_ctx {
        struct wait_queue_head          poll_wq;
        struct io_restriction           restrictions;
 
-       /* slow path rsrc auxilary data, used by update/register */
-       struct io_rsrc_data             *file_data;
-       struct io_rsrc_data             *buf_data;
-
        u32                     pers_next;
        struct xarray           personalities;
 

diff --git a/io_uring/cancel.c b/io_uring/cancel.c
index cc3475b22ae5d62c6031c7edee72f66933d2044a..3a299630702587b5e336dd54b0a4407100c6b1f7 100644 (file)
--- a/io_uring/cancel.c
+++ b/io_uring/cancel.c
@@ -240,9 +240,9 @@ static int __io_sync_cancel(struct io_uring_task *tctx,
        /* fixed must be grabbed every time since we drop the uring_lock */
        if ((cd->flags & IORING_ASYNC_CANCEL_FD) &&
            (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
-               if (unlikely(fd >= ctx->nr_user_files))
+               if (unlikely(fd >= ctx->file_table.data.nr))
                        return -EBADF;
-               fd = array_index_nospec(fd, ctx->nr_user_files);
+               fd = array_index_nospec(fd, ctx->file_table.data.nr);
                cd->file = io_file_from_index(&ctx->file_table, fd);
                if (!cd->file)
                        return -EBADF;

diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c
index 064a79475c5f575e25340fccaf88b79816076468..e3f5e9fe556245a693e933b8fbe7967a4973fff8 100644 (file)
--- a/io_uring/fdinfo.c
+++ b/io_uring/fdinfo.c
@@ -165,8 +165,8 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file)
        seq_printf(m, "SqThreadCpu:\t%d\n", sq_cpu);
        seq_printf(m, "SqTotalTime:\t%llu\n", sq_total_time);
        seq_printf(m, "SqWorkTime:\t%llu\n", sq_work_time);
-       seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
-       for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
+       seq_printf(m, "UserFiles:\t%u\n", ctx->file_table.data.nr);
+       for (i = 0; has_lock && i < ctx->file_table.data.nr; i++) {
                struct file *f = io_file_from_index(&ctx->file_table, i);
 
                if (f)
@@ -174,9 +174,9 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file)
                else
                        seq_printf(m, "%5u: <none>\n", i);
        }
-       seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
-       for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
-               struct io_mapped_ubuf *buf = ctx->user_bufs[i]->buf;
+       seq_printf(m, "UserBufs:\t%u\n", ctx->buf_table.nr);
+       for (i = 0; has_lock && i < ctx->buf_table.nr; i++) {
+               struct io_mapped_ubuf *buf = ctx->buf_table.nodes[i]->buf;
 
                seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, buf->len);
        }

diff --git a/io_uring/filetable.c b/io_uring/filetable.c
index 1b12a9a1cc16bef2284bada6054ebba4f5e167a0..c1f9f9550446cee679914a60f8a3ae75645edcd4 100644 (file)
--- a/io_uring/filetable.c
+++ b/io_uring/filetable.c
@@ -38,25 +38,19 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx)
 
 bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
 {
-       table->nodes = kvmalloc_array(nr_files, sizeof(struct io_src_node *),
-                                       GFP_KERNEL_ACCOUNT | __GFP_ZERO);
-       if (unlikely(!table->nodes))
+       if (io_rsrc_data_alloc(&table->data, nr_files))
                return false;
-
        table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
-       if (unlikely(!table->bitmap)) {
-               kvfree(table->nodes);
-               return false;
-       }
-
-       return true;
+       if (table->bitmap)
+               return true;
+       io_rsrc_data_free(&table->data);
+       return false;
 }
 
 void io_free_file_tables(struct io_file_table *table)
 {
-       kvfree(table->nodes);
+       io_rsrc_data_free(&table->data);
        bitmap_free(table->bitmap);
-       table->nodes = NULL;
        table->bitmap = NULL;
 }
 
@@ -68,22 +62,22 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
 
        if (io_is_uring_fops(file))
                return -EBADF;
-       if (!ctx->file_data)
+       if (!ctx->file_table.data.nr)
                return -ENXIO;
-       if (slot_index >= ctx->nr_user_files)
+       if (slot_index >= ctx->file_table.data.nr)
                return -EINVAL;
 
        node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
        if (!node)
                return -ENOMEM;
 
-       slot_index = array_index_nospec(slot_index, ctx->nr_user_files);
-       if (ctx->file_table.nodes[slot_index])
-               io_put_rsrc_node(ctx->file_table.nodes[slot_index]);
+       slot_index = array_index_nospec(slot_index, ctx->file_table.data.nr);
+       if (ctx->file_table.data.nodes[slot_index])
+               io_put_rsrc_node(ctx->file_table.data.nodes[slot_index]);
        else
                io_file_bitmap_set(&ctx->file_table, slot_index);
 
-       ctx->file_table.nodes[slot_index] = node;
+       ctx->file_table.data.nodes[slot_index] = node;
        io_fixed_file_set(node, file);
        return 0;
 }
@@ -129,16 +123,16 @@ int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
 
 int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
 {
-       if (unlikely(!ctx->file_data))
+       if (unlikely(!ctx->file_table.data.nr))
                return -ENXIO;
-       if (offset >= ctx->nr_user_files)
+       if (offset >= ctx->file_table.data.nr)
                return -EINVAL;
 
-       offset = array_index_nospec(offset, ctx->nr_user_files);
-       if (!ctx->file_table.nodes[offset])
+       offset = array_index_nospec(offset, ctx->file_table.data.nr);
+       if (!ctx->file_table.data.nodes[offset])
                return -EBADF;
-       io_put_rsrc_node(ctx->file_table.nodes[offset]);
-       ctx->file_table.nodes[offset] = NULL;
+       io_put_rsrc_node(ctx->file_table.data.nodes[offset]);
+       ctx->file_table.data.nodes[offset] = NULL;
        io_file_bitmap_clear(&ctx->file_table, offset);
        return 0;
 }
@@ -153,7 +147,7 @@ int io_register_file_alloc_range(struct io_ring_ctx *ctx,
                return -EFAULT;
        if (check_add_overflow(range.off, range.len, &end))
                return -EOVERFLOW;
-       if (range.resv || end > ctx->nr_user_files)
+       if (range.resv || end > ctx->file_table.data.nr)
                return -EINVAL;
 
        io_file_table_set_alloc_range(ctx, range.off, range.len);

diff --git a/io_uring/filetable.h b/io_uring/filetable.h
index 47616079abaaa9c53ff43eca076cc34f78ba4148..664c31502dbba74b5d9aef6bde164a1024c58783 100644 (file)
--- a/io_uring/filetable.h
+++ b/io_uring/filetable.h
@@ -52,7 +52,7 @@ static inline struct file *io_slot_file(struct io_rsrc_node *node)
 static inline struct file *io_file_from_index(struct io_file_table *table,
                                              int index)
 {
-       struct io_rsrc_node *node = table->nodes[index];
+       struct io_rsrc_node *node = table->data.nodes[index];
 
        if (node)
                return io_slot_file(node);

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 60c947114fa39e0ae24d8dcbeb6c72316e09041e..78df515fb3a72119ba5efbff30f27788cb9fbb38 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1879,11 +1879,10 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
        struct file *file = NULL;
 
        io_ring_submit_lock(ctx, issue_flags);
-
-       if (unlikely((unsigned int)fd >= ctx->nr_user_files))
+       if (unlikely((unsigned int)fd >= ctx->file_table.data.nr))
                goto out;
-       fd = array_index_nospec(fd, ctx->nr_user_files);
-       node = ctx->file_table.nodes[fd];
+       fd = array_index_nospec(fd, ctx->file_table.data.nr);
+       node = ctx->file_table.data.nodes[fd];
        if (node) {
                io_req_assign_rsrc_node(req, node);
                req->flags |= io_slot_flags(node);

diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c
index edea1ffd501c5a0c5b4e93d251f4cc40413c567e..b90ab3b8f5e0e868b7b03570293befb7477e9eab 100644 (file)
--- a/io_uring/msg_ring.c
+++ b/io_uring/msg_ring.c
@@ -180,8 +180,8 @@ static struct file *io_msg_grab_file(struct io_kiocb *req, unsigned int issue_fl
        int idx = msg->src_fd;
 
        io_ring_submit_lock(ctx, issue_flags);
-       if (likely(idx < ctx->nr_user_files)) {
-               idx = array_index_nospec(idx, ctx->nr_user_files);
+       if (likely(idx < ctx->file_table.data.nr)) {
+               idx = array_index_nospec(idx, ctx->file_table.data.nr);
                file = io_file_from_index(&ctx->file_table, idx);
                if (file)
                        get_file(file);

diff --git a/io_uring/net.c b/io_uring/net.c
index ce1156551d10bb11c1a3eaea93cbb746d9f068b8..3e1f31574abbeb44a4356dfba401a89979d93842 100644 (file)
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -1347,9 +1347,9 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags)
 
                ret = -EFAULT;
                io_ring_submit_lock(ctx, issue_flags);
-               if (sr->buf_index < ctx->nr_user_bufs) {
-                       idx = array_index_nospec(sr->buf_index, ctx->nr_user_bufs);
-                       node = ctx->user_bufs[idx];
+               if (sr->buf_index < ctx->buf_table.nr) {
+                       idx = array_index_nospec(sr->buf_index, ctx->buf_table.nr);
+                       node = ctx->buf_table.nodes[idx];
                        io_req_assign_rsrc_node(sr->notif, node);
                        ret = 0;
                }

diff --git a/io_uring/nop.c b/io_uring/nop.c
index de91600a3bc6811fe838af7cc1738a27a3a9a48a..0dac01127de5bb4988c46d2abd569b9228d730e5 100644 (file)
--- a/io_uring/nop.c
+++ b/io_uring/nop.c
@@ -66,9 +66,9 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags)
 
                ret = -EFAULT;
                io_ring_submit_lock(ctx, issue_flags);
-               if (nop->buffer < ctx->nr_user_bufs) {
-                       idx = array_index_nospec(nop->buffer, ctx->nr_user_bufs);
-                       node = READ_ONCE(ctx->user_bufs[idx]);
+               if (nop->buffer < ctx->buf_table.nr) {
+                       idx = array_index_nospec(nop->buffer, ctx->buf_table.nr);
+                       node = READ_ONCE(ctx->buf_table.nodes[idx]);
                        io_req_assign_rsrc_node(req, node);
                        ret = 0;
                }

diff --git a/io_uring/register.c b/io_uring/register.c
index 1eb686eaa310f216cd3a73660c1473c451ea96e5..45edfc57963a27cb6d257f3fa66d4ca63fac69d1 100644 (file)
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -937,7 +937,8 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
        mutex_lock(&ctx->uring_lock);
        ret = __io_uring_register(ctx, opcode, arg, nr_args);
        mutex_unlock(&ctx->uring_lock);
-       trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret);
+       trace_io_uring_register(ctx, opcode, ctx->file_table.data.nr,
+                               ctx->buf_table.nr, ret);
        if (!use_registered_ring)
                fput(file);
        return ret;

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 69a9cd82460d01463c72d10348b03d144cdc2cde..49a6ab5f3ae96f886d29ba1389fe95cfea0e6800 100644 (file)
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -142,39 +142,28 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type)
        return node;
 }
 
-static void io_rsrc_data_free(struct io_rsrc_data *data)
+__cold void io_rsrc_data_free(struct io_rsrc_data *data)
 {
-       int i;
-
-       for (i = 0; i < data->nr; i++) {
-               struct io_rsrc_node *node = data->nodes[i];
-
-               if (node)
-                       io_put_rsrc_node(node);
+       if (!data->nr)
+               return;
+       while (data->nr--) {
+               if (data->nodes[data->nr])
+                       io_put_rsrc_node(data->nodes[data->nr]);
        }
        kvfree(data->nodes);
-       kfree(data);
+       data->nodes = NULL;
+       data->nr = 0;
 }
 
-__cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, unsigned nr,
-                                    struct io_rsrc_data **pdata)
+__cold int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr)
 {
-       struct io_rsrc_data *data;
-
-       data = kzalloc(sizeof(*data), GFP_KERNEL);
-       if (!data)
-               return -ENOMEM;
-
        data->nodes = kvmalloc_array(nr, sizeof(struct io_rsrc_node *),
-                                       GFP_KERNEL | __GFP_ZERO);
-       if (!data->nodes) {
-               kfree(data);
-               return -ENOMEM;
+                                       GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+       if (data->nodes) {
+               data->nr = nr;
+               return 0;
        }
-
-       data->nr = nr;
-       *pdata = data;
-       return 0;
+       return -ENOMEM;
 }
 
 static int __io_sqe_files_update(struct io_ring_ctx *ctx,
@@ -186,9 +175,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
        int fd, i, err = 0;
        unsigned int done;
 
-       if (!ctx->file_data)
+       if (!ctx->file_table.data.nr)
                return -ENXIO;
-       if (up->offset + nr_args > ctx->nr_user_files)
+       if (up->offset + nr_args > ctx->file_table.data.nr)
                return -EINVAL;
 
        for (done = 0; done < nr_args; done++) {
@@ -206,10 +195,10 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
                if (fd == IORING_REGISTER_FILES_SKIP)
                        continue;
 
-               i = array_index_nospec(up->offset + done, ctx->nr_user_files);
-               if (ctx->file_table.nodes[i]) {
-                       io_put_rsrc_node(ctx->file_table.nodes[i]);
-                       ctx->file_table.nodes[i] = NULL;
+               i = array_index_nospec(up->offset + done, ctx->file_table.data.nr);
+               if (ctx->file_table.data.nodes[i]) {
+                       io_put_rsrc_node(ctx->file_table.data.nodes[i]);
+                       ctx->file_table.data.nodes[i] = NULL;
                        io_file_bitmap_clear(&ctx->file_table, i);
                }
                if (fd != -1) {
@@ -234,7 +223,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
                                fput(file);
                                break;
                        }
-                       ctx->file_table.nodes[i] = node;
+                       ctx->file_table.data.nodes[i] = node;
                        if (tag)
                                node->tag = tag;
                        io_fixed_file_set(node, file);
@@ -256,9 +245,9 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
        __u32 done;
        int i, err;
 
-       if (!ctx->buf_data)
+       if (!ctx->buf_table.nr)
                return -ENXIO;
-       if (up->offset + nr_args > ctx->nr_user_bufs)
+       if (up->offset + nr_args > ctx->buf_table.nr)
                return -EINVAL;
 
        for (done = 0; done < nr_args; done++) {
@@ -282,16 +271,16 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
                        err = -EINVAL;
                        break;
                }
-               i = array_index_nospec(up->offset + done, ctx->nr_user_bufs);
                node = io_sqe_buffer_register(ctx, iov, &last_hpage);
                if (IS_ERR(node)) {
                        err = PTR_ERR(node);
                        break;
                }
-               if (ctx->user_bufs[i])
-                       io_put_rsrc_node(ctx->user_bufs[i]);
+               i = array_index_nospec(up->offset + done, ctx->buf_table.nr);
+               if (ctx->buf_table.nodes[i])
+                       io_put_rsrc_node(ctx->buf_table.nodes[i]);
 
-               ctx->user_bufs[i] = node;
+               ctx->buf_table.nodes[i] = node;
                if (tag)
                        node->tag = tag;
                if (ctx->compat)
@@ -409,7 +398,7 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req,
        struct file *file;
        int ret, fd;
 
-       if (!req->ctx->file_data)
+       if (!req->ctx->file_table.data.nr)
                return -ENXIO;
 
        for (done = 0; done < up->nr_args; done++) {
@@ -494,35 +483,13 @@ void io_free_rsrc_node(struct io_rsrc_node *node)
        kfree(node);
 }
 
-static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
-{
-       int i;
-
-       lockdep_assert_held(&ctx->uring_lock);
-
-       for (i = 0; i < ctx->nr_user_files; i++) {
-               struct io_rsrc_node *node = ctx->file_table.nodes[i];
-
-               if (node) {
-                       io_put_rsrc_node(node);
-                       io_file_bitmap_clear(&ctx->file_table, i);
-                       ctx->file_table.nodes[i] = NULL;
-               }
-       }
-
-       io_free_file_tables(&ctx->file_table);
-       io_file_table_set_alloc_range(ctx, 0, 0);
-       io_rsrc_data_free(ctx->file_data);
-       ctx->file_data = NULL;
-       ctx->nr_user_files = 0;
-}
-
 int io_sqe_files_unregister(struct io_ring_ctx *ctx)
 {
-       if (!ctx->file_data)
+       if (!ctx->file_table.data.nr)
                return -ENXIO;
 
-       __io_sqe_files_unregister(ctx);
+       io_free_file_tables(&ctx->file_table);
+       io_file_table_set_alloc_range(ctx, 0, 0);
        return 0;
 }
 
@@ -534,7 +501,7 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
        int fd, ret;
        unsigned i;
 
-       if (ctx->file_data)
+       if (ctx->file_table.data.nr)
                return -EBUSY;
        if (!nr_args)
                return -EINVAL;
@@ -542,17 +509,10 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
                return -EMFILE;
        if (nr_args > rlimit(RLIMIT_NOFILE))
                return -EMFILE;
-       ret = io_rsrc_data_alloc(ctx, nr_args, &ctx->file_data);
-       if (ret)
-               return ret;
-
-       if (!io_alloc_file_tables(&ctx->file_table, nr_args)) {
-               io_rsrc_data_free(ctx->file_data);
-               ctx->file_data = NULL;
+       if (!io_alloc_file_tables(&ctx->file_table, nr_args))
                return -ENOMEM;
-       }
 
-       for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
+       for (i = 0; i < nr_args; i++) {
                struct io_rsrc_node *node;
                u64 tag = 0;
 
@@ -589,44 +549,24 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
                }
                if (tag)
                        node->tag = tag;
-               ctx->file_table.nodes[i] = node;
+               ctx->file_table.data.nodes[i] = node;
                io_fixed_file_set(node, file);
                io_file_bitmap_set(&ctx->file_table, i);
        }
 
        /* default it to the whole table */
-       io_file_table_set_alloc_range(ctx, 0, ctx->nr_user_files);
+       io_file_table_set_alloc_range(ctx, 0, ctx->file_table.data.nr);
        return 0;
 fail:
-       __io_sqe_files_unregister(ctx);
+       io_sqe_files_unregister(ctx);
        return ret;
 }
 
-static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
-{
-       unsigned int i;
-
-       lockdep_assert_held(&ctx->uring_lock);
-
-       for (i = 0; i < ctx->nr_user_bufs; i++) {
-               if (ctx->user_bufs[i]) {
-                       io_put_rsrc_node(ctx->user_bufs[i]);
-                       ctx->user_bufs[i] = NULL;
-               }
-       }
-       kvfree(ctx->user_bufs);
-       ctx->user_bufs = NULL;
-       io_rsrc_data_free(ctx->buf_data);
-       ctx->buf_data = NULL;
-       ctx->nr_user_bufs = 0;
-}
-
 int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
 {
-       if (!ctx->buf_data)
+       if (!ctx->buf_table.nr)
                return -ENXIO;
-
-       __io_sqe_buffers_unregister(ctx);
+       io_rsrc_data_free(&ctx->buf_table);
        return 0;
 }
 
@@ -653,8 +593,8 @@ static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
        }
 
        /* check previously registered pages */
-       for (i = 0; i < ctx->nr_user_bufs; i++) {
-               struct io_rsrc_node *node = ctx->user_bufs[i];
+       for (i = 0; i < ctx->buf_table.nr; i++) {
+               struct io_rsrc_node *node = ctx->buf_table.nodes[i];
                struct io_mapped_ubuf *imu = node->buf;
 
                for (j = 0; j < imu->nr_bvecs; j++) {
@@ -805,6 +745,9 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
        struct io_imu_folio_data data;
        bool coalesced;
 
+       if (!iov->iov_base)
+               return rsrc_empty_node;
+
        node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
        if (!node)
                return ERR_PTR(-ENOMEM);
@@ -864,40 +807,29 @@ done:
        return node;
 }
 
-static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
-{
-       ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
-       return ctx->user_bufs ? 0 : -ENOMEM;
-}
-
 int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
                            unsigned int nr_args, u64 __user *tags)
 {
        struct page *last_hpage = NULL;
-       struct io_rsrc_data *data;
+       struct io_rsrc_data data;
        struct iovec fast_iov, *iov = &fast_iov;
        const struct iovec __user *uvec;
        int i, ret;
 
        BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16));
 
-       if (ctx->user_bufs)
+       if (ctx->buf_table.nr)
                return -EBUSY;
        if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
                return -EINVAL;
-       ret = io_rsrc_data_alloc(ctx, nr_args, &data);
+       ret = io_rsrc_data_alloc(&data, nr_args);
        if (ret)
                return ret;
-       ret = io_buffers_map_alloc(ctx, nr_args);
-       if (ret) {
-               io_rsrc_data_free(data);
-               return ret;
-       }
 
        if (!arg)
                memset(iov, 0, sizeof(*iov));
 
-       for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
+       for (i = 0; i < nr_args; i++) {
                struct io_rsrc_node *node;
                u64 tag = 0;
 
@@ -935,14 +867,12 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
                }
                if (tag)
                        node->tag = tag;
-               ctx->user_bufs[i] = node;
+               data.nodes[i] = node;
        }
 
-       WARN_ON_ONCE(ctx->buf_data);
-
-       ctx->buf_data = data;
+       ctx->buf_table = data;
        if (ret)
-               __io_sqe_buffers_unregister(ctx);
+               io_sqe_buffers_unregister(ctx);
        return ret;
 }
 
@@ -1009,8 +939,7 @@ int io_import_fixed(int ddir, struct iov_iter *iter,
 
 static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx)
 {
-       struct io_rsrc_node **user_bufs;
-       struct io_rsrc_data *data;
+       struct io_rsrc_data data;
        int i, ret, nbufs;
 
        /*
@@ -1021,43 +950,37 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
 
        mutex_lock(&src_ctx->uring_lock);
        ret = -ENXIO;
-       nbufs = src_ctx->nr_user_bufs;
+       nbufs = src_ctx->buf_table.nr;
        if (!nbufs)
                goto out_unlock;
-       ret = io_rsrc_data_alloc(ctx, nbufs, &data);
+       ret = io_rsrc_data_alloc(&data, nbufs);
        if (ret)
                goto out_unlock;
 
-       ret = -ENOMEM;
-       user_bufs = kvmalloc_array(nbufs, sizeof(struct io_rsrc_node *),
-                                       GFP_KERNEL | __GFP_ZERO);
-       if (!user_bufs)
-               goto out_free_data;
-
        for (i = 0; i < nbufs; i++) {
-               struct io_rsrc_node *src_node = src_ctx->user_bufs[i];
+               struct io_rsrc_node *src_node = src_ctx->buf_table.nodes[i];
                struct io_rsrc_node *dst_node;
 
                if (src_node == rsrc_empty_node) {
                        dst_node = rsrc_empty_node;
                } else {
                        dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
-                       if (!dst_node)
+                       if (!dst_node) {
+                               ret = -ENOMEM;
                                goto out_put_free;
+                       }
 
                        refcount_inc(&src_node->buf->refs);
                        dst_node->buf = src_node->buf;
                }
-               user_bufs[i] = dst_node;
+               data.nodes[i] = dst_node;
        }
 
        /* Have a ref on the bufs now, drop src lock and re-grab our own lock */
        mutex_unlock(&src_ctx->uring_lock);
        mutex_lock(&ctx->uring_lock);
-       if (!ctx->user_bufs) {
-               ctx->user_bufs = user_bufs;
-               ctx->buf_data = data;
-               ctx->nr_user_bufs = nbufs;
+       if (!ctx->buf_table.nr) {
+               ctx->buf_table = data;
                return 0;
        }
 
@@ -1068,12 +991,10 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
        i = nbufs;
 out_put_free:
        while (i--) {
-               io_buffer_unmap(src_ctx, user_bufs[i]);
-               kfree(user_bufs[i]);
+               io_buffer_unmap(src_ctx, data.nodes[i]);
+               kfree(data.nodes[i]);
        }
-       kvfree(user_bufs);
-out_free_data:
-       io_rsrc_data_free(data);
+       io_rsrc_data_free(&data);
 out_unlock:
        mutex_unlock(&src_ctx->uring_lock);
        mutex_lock(&ctx->uring_lock);
@@ -1094,7 +1015,7 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)
        struct file *file;
        int ret;
 
-       if (ctx->user_bufs || ctx->nr_user_bufs)
+       if (ctx->buf_table.nr)
                return -EBUSY;
        if (copy_from_user(&buf, arg, sizeof(buf)))
                return -EFAULT;

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 48f712488f6b2fca4c6944fe6482eaa8bed10b6b..569ea9ce1405387136a63b5ea247f1c012b68d54 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -13,11 +13,6 @@ enum {
        IORING_RSRC_BUFFER              = 1,
 };
 
-struct io_rsrc_data {
-       unsigned int                    nr;
-       struct io_rsrc_node             **nodes;
-};
-
 struct io_rsrc_node {
        struct io_ring_ctx              *ctx;
        int                             refs;
@@ -50,6 +45,8 @@ struct io_imu_folio_data {
 
 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type);
 void io_free_rsrc_node(struct io_rsrc_node *node);
+void io_rsrc_data_free(struct io_rsrc_data *data);
+int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr);
 
 int io_import_fixed(int ddir, struct iov_iter *iter,
                           struct io_mapped_ubuf *imu,

diff --git a/io_uring/rw.c b/io_uring/rw.c
index 65491f4f2c7e3e0817f4e58d4ac3e93d305c7f4b..28fff18ebb19dc76334856385e0b5c8ad68948f2 100644 (file)
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -339,10 +339,10 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe
        if (unlikely(ret))
                return ret;
 
-       if (unlikely(req->buf_index >= ctx->nr_user_bufs))
+       if (unlikely(req->buf_index >= ctx->buf_table.nr))
                return -EFAULT;
-       index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);
-       node = ctx->user_bufs[index];
+       index = array_index_nospec(req->buf_index, ctx->buf_table.nr);
+       node = ctx->buf_table.nodes[index];
        io_req_assign_rsrc_node(req, node);
 
        io = req->async_data;

diff --git a/io_uring/splice.c b/io_uring/splice.c
index f78afb575ae6c805cad0e99e07b1b5398d949089..aaaddb66e90ac8fa51123ddc3a5339a44f76eb34 100644 (file)
--- a/io_uring/splice.c
+++ b/io_uring/splice.c
@@ -66,10 +66,10 @@ static struct file *io_splice_get_file(struct io_kiocb *req,
                return io_file_get_normal(req, sp->splice_fd_in);
 
        io_ring_submit_lock(ctx, issue_flags);
-       if (unlikely(sp->splice_fd_in >= ctx->nr_user_files))
+       if (unlikely(sp->splice_fd_in >= ctx->file_table.data.nr))
                goto out;
-       sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->nr_user_files);
-       node = ctx->file_table.nodes[sp->splice_fd_in];
+       sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->file_table.data.nr);
+       node = ctx->file_table.data.nodes[sp->splice_fd_in];
        if (node) {
                node->refs++;
                sp->rsrc_node = node;

diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 0899c71008aeaa5a9660d2855943469d1aba06c0..17d5f5004702f1afdff7d738f5c68295defddbe6 100644 (file)
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -212,15 +212,15 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
                u16 index;
 
                index = READ_ONCE(sqe->buf_index);
-               if (unlikely(index >= ctx->nr_user_bufs))
+               if (unlikely(index >= ctx->buf_table.nr))
                        return -EFAULT;
-               req->buf_index = array_index_nospec(index, ctx->nr_user_bufs);
+               req->buf_index = array_index_nospec(index, ctx->buf_table.nr);
                /*
                 * Pi node upfront, prior to io_uring_cmd_import_fixed()
                 * being called. This prevents destruction of the mapped buffer
                 * we'll need at actual import time.
                 */
-               io_req_assign_rsrc_node(req, ctx->user_bufs[req->buf_index]);
+               io_req_assign_rsrc_node(req, ctx->buf_table.nodes[req->buf_index]);
        }
        ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);