io_uring/register: cache old SQ/CQ head reading for copies

The SQ and CQ ring heads are read twice - once for verifying that it's
within bounds, and once inside the loops copying SQE and CQE entries.
This is technically incorrect, in case the values could get modified
in between verifying them and using them in the copy loop. While this
won't lead to anything truly nefarious, it may cause longer loop times
for the copies than expected.

Read the ring head values once, and use the verified value in the copy
loops.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/register.c b/io_uring/register.c
index ffcbc840032e9394c0de807a2191570603cfedcb..371aec87e078c82921ca4e3838866536750231da 100644 (file)
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -405,8 +405,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 {
        struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
        size_t size, sq_array_offset;
+       unsigned i, tail, old_head;
        struct io_uring_params p;
-       unsigned i, tail;
        void *ptr;
        int ret;
 
@@ -518,9 +518,10 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
         */
        n.sq_sqes = ptr;
        tail = READ_ONCE(o.rings->sq.tail);
-       if (tail - READ_ONCE(o.rings->sq.head) > p.sq_entries)
+       old_head = READ_ONCE(o.rings->sq.head);
+       if (tail - old_head > p.sq_entries)
                goto overflow;
-       for (i = READ_ONCE(o.rings->sq.head); i < tail; i++) {
+       for (i = old_head; i < tail; i++) {
                unsigned src_head = i & (ctx->sq_entries - 1);
                unsigned dst_head = i & (p.sq_entries - 1);
 
@@ -530,7 +531,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
        WRITE_ONCE(n.rings->sq.tail, READ_ONCE(o.rings->sq.tail));
 
        tail = READ_ONCE(o.rings->cq.tail);
-       if (tail - READ_ONCE(o.rings->cq.head) > p.cq_entries) {
+       old_head = READ_ONCE(o.rings->cq.head);
+       if (tail - old_head > p.cq_entries) {
 overflow:
                /* restore old rings, and return -EOVERFLOW via cleanup path */
                ctx->rings = o.rings;
@@ -539,7 +541,7 @@ overflow:
                ret = -EOVERFLOW;
                goto out;
        }
-       for (i = READ_ONCE(o.rings->cq.head); i < tail; i++) {
+       for (i = old_head; i < tail; i++) {
                unsigned src_head = i & (ctx->cq_entries - 1);
                unsigned dst_head = i & (p.cq_entries - 1);