io_uring/kbuf: flag request if buffer pool is empty after buffer pick

Normally we do an extra roundtrip for retries even if the buffer pool has
depleted, as we don't check that upfront. Rather than add this check, have
the buffer selection methods mark the request with REQ_F_BL_EMPTY if the
used buffer group is out of buffers after this selection. This is very
cheap to do once we're all the way inside there anyway, and it gives the
caller a chance to make better decisions on how to proceed.

For example, recv/recvmsg multishot could check this flag when it
decides whether to keep receiving or not.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index bd7071aeec5d016e1e4a02897c63d5cf816a0aac..d8111d64812b8b808a8079cfda703cf87c30bad8 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -480,6 +480,7 @@ enum {
        REQ_F_POLL_NO_LAZY_BIT,
        REQ_F_CANCEL_SEQ_BIT,
        REQ_F_CAN_POLL_BIT,
+       REQ_F_BL_EMPTY_BIT,
 
        /* not a real bit, just to check we're not overflowing the space */
        __REQ_F_LAST_BIT,
@@ -556,6 +557,8 @@ enum {
        REQ_F_CANCEL_SEQ        = IO_REQ_FLAG(REQ_F_CANCEL_SEQ_BIT),
        /* file is pollable */
        REQ_F_CAN_POLL          = IO_REQ_FLAG(REQ_F_CAN_POLL_BIT),
+       /* buffer list was empty after selection of buffer */
+       REQ_F_BL_EMPTY          = IO_REQ_FLAG(REQ_F_BL_EMPTY_BIT),
 };
 
 typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);

diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index ee866d646997932ebf0bba236ccfb7761b26e8ea..3d257ed9031b0918ab640a033e002123f5d605de 100644 (file)
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -139,6 +139,8 @@ static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
                list_del(&kbuf->list);
                if (*len == 0 || *len > kbuf->len)
                        *len = kbuf->len;
+               if (list_empty(&bl->buf_list))
+                       req->flags |= REQ_F_BL_EMPTY;
                req->flags |= REQ_F_BUFFER_SELECTED;
                req->kbuf = kbuf;
                req->buf_index = kbuf->bid;
@@ -152,12 +154,16 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
                                          unsigned int issue_flags)
 {
        struct io_uring_buf_ring *br = bl->buf_ring;
+       __u16 tail, head = bl->head;
        struct io_uring_buf *buf;
-       __u16 head = bl->head;
 
-       if (unlikely(smp_load_acquire(&br->tail) == head))
+       tail = smp_load_acquire(&br->tail);
+       if (unlikely(tail == head))
                return NULL;
 
+       if (head + 1 == tail)
+               req->flags |= REQ_F_BL_EMPTY;
+
        head &= bl->mask;
        /* mmaped buffers are always contig */
        if (bl->is_mmap || head < IO_BUFFER_LIST_BUF_PER_PAGE) {