io-wq: briefly spin for new work after finishing work

To avoid going to sleep only to get woken shortly thereafter, spin
briefly for new work upon completion of work.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index 6b663ddceb4277ceb6f5b023a5ae7684958c54b8..90c4978781fb5fb48885a682ab89db72f3c780c6 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -491,26 +491,46 @@ next:
        } while (1);
 }
 
+static inline void io_worker_spin_for_work(struct io_wqe *wqe)
+{
+       int i = 0;
+
+       while (++i < 1000) {
+               if (io_wqe_run_queue(wqe))
+                       break;
+               if (need_resched())
+                       break;
+               cpu_relax();
+       }
+}
+
 static int io_wqe_worker(void *data)
 {
        struct io_worker *worker = data;
        struct io_wqe *wqe = worker->wqe;
        struct io_wq *wq = wqe->wq;
+       bool did_work;
 
        io_worker_start(wqe, worker);
 
+       did_work = false;
        while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) {
                set_current_state(TASK_INTERRUPTIBLE);
+loop:
+               if (did_work)
+                       io_worker_spin_for_work(wqe);
                spin_lock_irq(&wqe->lock);
                if (io_wqe_run_queue(wqe)) {
                        __set_current_state(TASK_RUNNING);
                        io_worker_handle_work(worker);
-                       continue;
+                       did_work = true;
+                       goto loop;
                }
+               did_work = false;
                /* drops the lock on success, retry */
                if (__io_worker_idle(wqe, worker)) {
                        __release(&wqe->lock);
-                       continue;
+                       goto loop;
                }
                spin_unlock_irq(&wqe->lock);
                if (signal_pending(current))

diff --git a/fs/io-wq.h b/fs/io-wq.h
index 7c333a28e2a7e9874f27c85492b943b5d33fc935..fb993b2bd0ef019e76438bc5483b2995ba49fc8a 100644 (file)
--- a/fs/io-wq.h
+++ b/fs/io-wq.h
@@ -35,7 +35,8 @@ static inline void wq_list_add_tail(struct io_wq_work_node *node,
                                    struct io_wq_work_list *list)
 {
        if (!list->first) {
-               list->first = list->last = node;
+               list->last = node;
+               WRITE_ONCE(list->first, node);
        } else {
                list->last->next = node;
                list->last = node;
@@ -47,7 +48,7 @@ static inline void wq_node_del(struct io_wq_work_list *list,
                               struct io_wq_work_node *prev)
 {
        if (node == list->first)
-               list->first = node->next;
+               WRITE_ONCE(list->first, node->next);
        if (node == list->last)
                list->last = prev;
        if (prev)
@@ -58,7 +59,7 @@ static inline void wq_node_del(struct io_wq_work_list *list,
 #define wq_list_for_each(pos, prv, head)                       \
        for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)
 
-#define wq_list_empty(list)    ((list)->first == NULL)
+#define wq_list_empty(list)    (READ_ONCE((list)->first) == NULL)
 #define INIT_WQ_LIST(list)     do {                            \
        (list)->first = NULL;                                   \
        (list)->last = NULL;                                    \