io_uring: make io_double_put_req() use normal completion path

If we don't use the normal completion path, we may skip killing links
that should be errored and freed. Add __io_double_put_req() for use
within the completion path itself, other calls should just use
io_double_put_req().

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index d877c7f6368eec301e50b7da6eb95608001a7ebb..1da103df2bfaef50e029a375563d41652453b613 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -383,6 +383,7 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res);
 static void __io_free_req(struct io_kiocb *req);
 static void io_put_req(struct io_kiocb *req);
 static void io_double_put_req(struct io_kiocb *req);
+static void __io_double_put_req(struct io_kiocb *req);
 
 static struct kmem_cache *req_cachep;
 
@@ -916,7 +917,7 @@ static void io_fail_links(struct io_kiocb *req)
                        io_link_cancel_timeout(link);
                } else {
                        io_cqring_fill_event(link, -ECANCELED);
-                       io_double_put_req(link);
+                       __io_double_put_req(link);
                }
        }
 
@@ -990,13 +991,24 @@ static void io_put_req(struct io_kiocb *req)
                io_free_req(req);
 }
 
-static void io_double_put_req(struct io_kiocb *req)
+/*
+ * Must only be used if we don't need to care about links, usually from
+ * within the completion handling itself.
+ */
+static void __io_double_put_req(struct io_kiocb *req)
 {
        /* drop both submit and complete references */
        if (refcount_sub_and_test(2, &req->refs))
                __io_free_req(req);
 }
 
+static void io_double_put_req(struct io_kiocb *req)
+{
+       /* drop both submit and complete references */
+       if (refcount_sub_and_test(2, &req->refs))
+               io_free_req(req);
+}
+
 static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
 {
        struct io_rings *rings = ctx->rings;