ring-buffer: Use wait_event_interruptible() in ring_buffer_wait()

Convert ring_buffer_wait() over to wait_event_interruptible(). The default
condition is to execute the wait loop inside __wait_event() just once.

This does not change the ring_buffer_wait() prototype yet, but
restructures the code so that it can take a "cond" and "data" parameter
and will call wait_event_interruptible() with a helper function as the
condition.

The helper function (rb_wait_cond) takes the cond function and data
parameters. It will first check if the buffer hit the watermark defined by
the "full" parameter and then call the passed in condition parameter. If
either are true, it returns true.

If rb_wait_cond() does not return true, it will set the appropriate
"waiters_pending" flag and returns false.

Link: https://lore.kernel.org/linux-trace-kernel/CAHk-=wgsNgewHFxZAJiAQznwPMqEtQmi1waeS2O1v6L4c_Um5A@mail.gmail.com/
Link: https://lore.kernel.org/linux-trace-kernel/20240312121703.399598519@goodmis.org
Cc: stable@vger.kernel.org
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: linke li <lilinke99@qq.com>
Cc: Rabin Vincent <rabin@rab.in>
Fixes: f3ddb74ad0790 ("tracing: Wake up ring buffer waiters on closing of the file")
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>

diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h
index fa802db216f94f8aa7349b02216f536dfa494279..338a33db1577e5d150725da256a53ee8e9e536f2 100644 (file)
--- a/include/linux/ring_buffer.h
+++ b/include/linux/ring_buffer.h
@@ -98,6 +98,7 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k
        __ring_buffer_alloc((size), (flags), &__key);   \
 })
 
+typedef bool (*ring_buffer_cond_fn)(void *data);
 int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full);
 __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu,
                          struct file *filp, poll_table *poll_table, int full);

diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 8c3730a8866231259225fc7561bbf5c72a0157ce..f4c34b7c7e1e766ed148eeb4215c71de06c87cf8 100644 (file)
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -843,43 +843,15 @@ static bool rb_watermark_hit(struct trace_buffer *buffer, int cpu, int full)
        return ret;
 }
 
-/**
- * ring_buffer_wait - wait for input to the ring buffer
- * @buffer: buffer to wait on
- * @cpu: the cpu buffer to wait on
- * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS
- *
- * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
- * as data is added to any of the @buffer's cpu buffers. Otherwise
- * it will wait for data to be added to a specific cpu buffer.
- */
-int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full)
+static inline bool
+rb_wait_cond(struct rb_irq_work *rbwork, struct trace_buffer *buffer,
+            int cpu, int full, ring_buffer_cond_fn cond, void *data)
 {
-       struct ring_buffer_per_cpu *cpu_buffer;
-       DEFINE_WAIT(wait);
-       struct rb_irq_work *work;
-       int ret = 0;
-
-       /*
-        * Depending on what the caller is waiting for, either any
-        * data in any cpu buffer, or a specific buffer, put the
-        * caller on the appropriate wait queue.
-        */
-       if (cpu == RING_BUFFER_ALL_CPUS) {
-               work = &buffer->irq_work;
-               /* Full only makes sense on per cpu reads */
-               full = 0;
-       } else {
-               if (!cpumask_test_cpu(cpu, buffer->cpumask))
-                       return -ENODEV;
-               cpu_buffer = buffer->buffers[cpu];
-               work = &cpu_buffer->irq_work;
-       }
+       if (rb_watermark_hit(buffer, cpu, full))
+               return true;
 
-       if (full)
-               prepare_to_wait(&work->full_waiters, &wait, TASK_INTERRUPTIBLE);
-       else
-               prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE);
+       if (cond(data))
+               return true;
 
        /*
         * The events can happen in critical sections where
@@ -902,27 +874,75 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full)
         * a task has been queued. It's OK for spurious wake ups.
         */
        if (full)
-               work->full_waiters_pending = true;
+               rbwork->full_waiters_pending = true;
        else
-               work->waiters_pending = true;
+               rbwork->waiters_pending = true;
 
-       if (rb_watermark_hit(buffer, cpu, full))
-               goto out;
+       return false;
+}
 
-       if (signal_pending(current)) {
-               ret = -EINTR;
-               goto out;
+/*
+ * The default wait condition for ring_buffer_wait() is to just to exit the
+ * wait loop the first time it is woken up.
+ */
+static bool rb_wait_once(void *data)
+{
+       long *once = data;
+
+       /* wait_event() actually calls this twice before scheduling*/
+       if (*once > 1)
+               return true;
+
+       (*once)++;
+       return false;
+}
+
+/**
+ * ring_buffer_wait - wait for input to the ring buffer
+ * @buffer: buffer to wait on
+ * @cpu: the cpu buffer to wait on
+ * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS
+ *
+ * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
+ * as data is added to any of the @buffer's cpu buffers. Otherwise
+ * it will wait for data to be added to a specific cpu buffer.
+ */
+int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full)
+{
+       struct ring_buffer_per_cpu *cpu_buffer;
+       struct wait_queue_head *waitq;
+       ring_buffer_cond_fn cond;
+       struct rb_irq_work *rbwork;
+       void *data;
+       long once = 0;
+       int ret = 0;
+
+       cond = rb_wait_once;
+       data = &once;
+
+       /*
+        * Depending on what the caller is waiting for, either any
+        * data in any cpu buffer, or a specific buffer, put the
+        * caller on the appropriate wait queue.
+        */
+       if (cpu == RING_BUFFER_ALL_CPUS) {
+               rbwork = &buffer->irq_work;
+               /* Full only makes sense on per cpu reads */
+               full = 0;
+       } else {
+               if (!cpumask_test_cpu(cpu, buffer->cpumask))
+                       return -ENODEV;
+               cpu_buffer = buffer->buffers[cpu];
+               rbwork = &cpu_buffer->irq_work;
        }
 
-       schedule();
- out:
        if (full)
-               finish_wait(&work->full_waiters, &wait);
+               waitq = &rbwork->full_waiters;
        else
-               finish_wait(&work->waiters, &wait);
+               waitq = &rbwork->waiters;
 
-       if (!ret && !rb_watermark_hit(buffer, cpu, full) && signal_pending(current))
-               ret = -EINTR;
+       ret = wait_event_interruptible((*waitq),
+                               rb_wait_cond(rbwork, buffer, cpu, full, cond, data));
 
        return ret;
 }