tick/nohz: Change signal tick dependency to wake up CPUs of member tasks

Rather than waking up all nohz_full CPUs on the system, only wake up
the target CPUs of member threads of the signal.

Reduces interruptions to nohz_full CPUs.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210512232924.150322-8-frederic@kernel.org

diff --git a/include/linux/tick.h b/include/linux/tick.h
index 2258984a0e8a79b57e83a0738df971c8110e9fa8..0bb80a7f05b9576217520aaac2a25227c38889e5 100644 (file)
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -211,7 +211,7 @@ extern void tick_nohz_dep_set_task(struct task_struct *tsk,
                                   enum tick_dep_bits bit);
 extern void tick_nohz_dep_clear_task(struct task_struct *tsk,
                                     enum tick_dep_bits bit);
-extern void tick_nohz_dep_set_signal(struct signal_struct *signal,
+extern void tick_nohz_dep_set_signal(struct task_struct *tsk,
                                     enum tick_dep_bits bit);
 extern void tick_nohz_dep_clear_signal(struct signal_struct *signal,
                                       enum tick_dep_bits bit);
@@ -256,11 +256,11 @@ static inline void tick_dep_clear_task(struct task_struct *tsk,
        if (tick_nohz_full_enabled())
                tick_nohz_dep_clear_task(tsk, bit);
 }
-static inline void tick_dep_set_signal(struct signal_struct *signal,
+static inline void tick_dep_set_signal(struct task_struct *tsk,
                                       enum tick_dep_bits bit)
 {
        if (tick_nohz_full_enabled())
-               tick_nohz_dep_set_signal(signal, bit);
+               tick_nohz_dep_set_signal(tsk, bit);
 }
 static inline void tick_dep_clear_signal(struct signal_struct *signal,
                                         enum tick_dep_bits bit)
@@ -288,7 +288,7 @@ static inline void tick_dep_set_task(struct task_struct *tsk,
                                     enum tick_dep_bits bit) { }
 static inline void tick_dep_clear_task(struct task_struct *tsk,
                                       enum tick_dep_bits bit) { }
-static inline void tick_dep_set_signal(struct signal_struct *signal,
+static inline void tick_dep_set_signal(struct task_struct *tsk,
                                       enum tick_dep_bits bit) { }
 static inline void tick_dep_clear_signal(struct signal_struct *signal,
                                         enum tick_dep_bits bit) { }

diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 3bb96a8b49c9bd8888e1aaa1791470fe169f4a8f..29a5e54e6e105465a5b7d2770bd6bbcaff1452a7 100644 (file)
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -523,7 +523,7 @@ static void arm_timer(struct k_itimer *timer, struct task_struct *p)
        if (CPUCLOCK_PERTHREAD(timer->it_clock))
                tick_dep_set_task(p, TICK_DEP_BIT_POSIX_TIMER);
        else
-               tick_dep_set_signal(p->signal, TICK_DEP_BIT_POSIX_TIMER);
+               tick_dep_set_signal(p, TICK_DEP_BIT_POSIX_TIMER);
 }
 
 /*
@@ -1358,7 +1358,7 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clkid,
        if (*newval < *nextevt)
                *nextevt = *newval;
 
-       tick_dep_set_signal(tsk->signal, TICK_DEP_BIT_POSIX_TIMER);
+       tick_dep_set_signal(tsk, TICK_DEP_BIT_POSIX_TIMER);
 }
 
 static int do_cpu_nanosleep(const clockid_t which_clock, int flags,

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index b90ca6635ea4c6775ec71a2c4a0bc6595be9a9b0..acbe6722cf756448d849b7c7c271d67300dccfc9 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -444,9 +444,20 @@ EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task);
  * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
  * per process timers.
  */
-void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit)
+void tick_nohz_dep_set_signal(struct task_struct *tsk,
+                             enum tick_dep_bits bit)
 {
-       tick_nohz_dep_set_all(&sig->tick_dep_mask, bit);
+       int prev;
+       struct signal_struct *sig = tsk->signal;
+
+       prev = atomic_fetch_or(BIT(bit), &sig->tick_dep_mask);
+       if (!prev) {
+               struct task_struct *t;
+
+               lockdep_assert_held(&tsk->sighand->siglock);
+               __for_each_thread(sig, t)
+                       tick_nohz_kick_task(t);
+       }
 }
 
 void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)