io_uring/napi: ensure napi polling is aborted when work is available

While testing io_uring NAPI with DEFER_TASKRUN, I ran into slowdowns and
stalls in packet delivery. Turns out that while
io_napi_busy_loop_should_end() aborts appropriately on regular
task_work, it does not abort if we have local task_work pending.

Move io_has_work() into the private io_uring.h header, and gate whether
we should continue polling on that as well. This makes NAPI polling on
send/receive work as designed with IORING_SETUP_DEFER_TASKRUN as well.

Fixes: 8d0c12a80cde ("io-uring: add napi busy poll support")
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index b25c2217c322497d313f249abeb412f7895c77e5..844a7524ed916c86080aaa4018cb0a24acd42267 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -122,11 +122,6 @@
 #define IO_COMPL_BATCH                 32
 #define IO_REQ_ALLOC_BATCH             8
 
-enum {
-       IO_CHECK_CQ_OVERFLOW_BIT,
-       IO_CHECK_CQ_DROPPED_BIT,
-};
-
 struct io_defer_entry {
        struct list_head        list;
        struct io_kiocb         *req;
@@ -2479,12 +2474,6 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
        return ret;
 }
 
-static inline bool io_has_work(struct io_ring_ctx *ctx)
-{
-       return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||
-              !llist_empty(&ctx->work_llist);
-}
-
 static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
                            int wake_flags, void *key)
 {

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 1ca99522811b40ddb9050add2087eb374aea9805..6426ee382276b5f298c68a450ceb894205eb374d 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -448,4 +448,15 @@ static inline bool io_file_can_poll(struct io_kiocb *req)
        }
        return false;
 }
+
+enum {
+       IO_CHECK_CQ_OVERFLOW_BIT,
+       IO_CHECK_CQ_DROPPED_BIT,
+};
+
+static inline bool io_has_work(struct io_ring_ctx *ctx)
+{
+       return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||
+              !llist_empty(&ctx->work_llist);
+}
 #endif

diff --git a/io_uring/napi.c b/io_uring/napi.c
index 8ec016899539021cfa885375b63858447875dc87..b234adda7dfd2d49f5b673827c0764336602f0af 100644 (file)
--- a/io_uring/napi.c
+++ b/io_uring/napi.c
@@ -122,7 +122,7 @@ static bool io_napi_busy_loop_should_end(void *data,
 
        if (signal_pending(current))
                return true;
-       if (io_should_wake(iowq))
+       if (io_should_wake(iowq) || io_has_work(iowq->ctx))
                return true;
        if (io_napi_busy_loop_timeout(start_time, iowq->napi_busy_poll_to))
                return true;