smp: Optimize send_call_function_single_ipi()

Just like the ttwu_queue_remote() IPI, make use of _TIF_POLLING_NRFLAG
to avoid sending IPIs to idle CPUs.

[ mingo: Fix UP build bug. ]

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20200526161907.953304789@infradead.org

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 2cacc1e44a84d754577c6effb5b8dd8ea682cbd5..fa0d4990618e89f5883f9c48b62398404c3d23e2 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2296,6 +2296,16 @@ static void wake_csd_func(void *info)
        sched_ttwu_pending();
 }
 
+void send_call_function_single_ipi(int cpu)
+{
+       struct rq *rq = cpu_rq(cpu);
+
+       if (!set_nr_if_polling(rq->idle))
+               arch_send_call_function_single_ipi(cpu);
+       else
+               trace_sched_wake_idle_without_ipi(cpu);
+}
+
 /*
  * Queue a task on the target CPUs wake_list and wake the CPU via IPI if
  * necessary. The wakee CPU on receipt of the IPI will queue the task

diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index b743bf38f08fbd4c56aa4b82f138278b29794f3c..387fd75ee6f727c64124efcb45f83a2e8d654583 100644 (file)
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -289,6 +289,11 @@ static void do_idle(void)
         */
        smp_mb__after_atomic();
 
+       /*
+        * RCU relies on this call to be done outside of an RCU read-side
+        * critical section.
+        */
+       flush_smp_call_function_from_idle();
        sched_ttwu_pending();
        schedule_idle();
 

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 3c163cb5493f9cc34e2ffd8f60c4141016a2fdf8..75b062999c4345a5528a8ee60d8b295c1ae39c73 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1506,11 +1506,12 @@ static inline void unregister_sched_domain_sysctl(void)
 }
 #endif
 
-#else
+extern void flush_smp_call_function_from_idle(void);
 
+#else /* !CONFIG_SMP: */
+static inline void flush_smp_call_function_from_idle(void) { }
 static inline void sched_ttwu_pending(void) { }
-
-#endif /* CONFIG_SMP */
+#endif
 
 #include "stats.h"
 #include "autogroup.h"

diff --git a/kernel/smp.c b/kernel/smp.c
index f720e38e880df795b31da4e4ce1a3768fda56fdf..9f1181375141eb58468fc308a9677c7f0e4a426f 100644 (file)
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -135,6 +135,8 @@ static __always_inline void csd_unlock(call_single_data_t *csd)
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
 
+extern void send_call_function_single_ipi(int cpu);
+
 /*
  * Insert a previously allocated call_single_data_t element
  * for execution on the given CPU. data must already have
@@ -178,7 +180,7 @@ static int generic_exec_single(int cpu, call_single_data_t *csd,
         * equipped to do the right thing...
         */
        if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
-               arch_send_call_function_single_ipi(cpu);
+               send_call_function_single_ipi(cpu);
 
        return 0;
 }
@@ -278,6 +280,18 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline)
        }
 }
 
+void flush_smp_call_function_from_idle(void)
+{
+       unsigned long flags;
+
+       if (llist_empty(this_cpu_ptr(&call_single_queue)))
+               return;
+
+       local_irq_save(flags);
+       flush_smp_call_function_queue(true);
+       local_irq_restore(flags);
+}
+
 /*
  * smp_call_function_single - Run a function on a specific CPU
  * @func: The function to run. This must be fast and non-blocking.