nohz: Make nohz API agnostic against idle ticks cputime accounting

When the timer tick fires, it accounts the new jiffy as either part
of system, user or idle time. This is how we record the cputime
statistics.

But when the tick is stopped from the idle task, we still need
to record the number of jiffies spent tickless until we restart
the tick and fall back to traditional tick-based cputime accounting.

To do this, we take a snapshot of jiffies when the tick is stopped
and compute the difference against the new value of jiffies when
the tick is restarted. Then we account this whole difference to
the idle cputime.

However we are preparing to be able to stop the tick from other places
than idle. So this idle time accounting needs to be performed from
the callers of nohz APIs, not from the nohz APIs themselves because
we now want them to be agnostic against places that stop/restart tick.

Therefore, we pull the tickless idle time accounting out of generic
nohz helpers up to idle entry/exit callers.

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Alessio Igor Bogani <abogani@kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Avi Kivity <avi@redhat.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Daniel Lezcano <daniel.lezcano@linaro.org>
Cc: Geoff Levand <geoff@infradead.org>
Cc: Gilad Ben Yossef <gilad@benyossef.com>
Cc: Hakan Akkan <hakanakkan@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Kevin Hilman <khilman@ti.com>
Cc: Max Krasnyansky <maxk@qualcomm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Stephen Hemminger <shemminger@vyatta.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sven-Thorsten Dietrich <thebigcorporation@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 81409bba24257de34199d287d57f589ca8aafd39..911834b33b8a457fb81e3c9ccc7f4597c4a7a7de 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -402,7 +402,6 @@ static void tick_nohz_stop_sched_tick(struct tick_sched *ts, ktime_t now)
 
                        ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);
                        ts->tick_stopped = 1;
-                       ts->idle_jiffies = last_jiffies;
                }
 
                ts->idle_sleeps++;
@@ -445,9 +444,13 @@ out:
 static void __tick_nohz_idle_enter(struct tick_sched *ts)
 {
        ktime_t now;
+       int was_stopped = ts->tick_stopped;
 
        now = tick_nohz_start_idle(smp_processor_id(), ts);
        tick_nohz_stop_sched_tick(ts, now);
+
+       if (!was_stopped && ts->tick_stopped)
+               ts->idle_jiffies = ts->last_jiffies;
 }
 
 /**
@@ -548,15 +551,25 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
 
 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
 {
-#ifndef CONFIG_VIRT_CPU_ACCOUNTING
-       unsigned long ticks;
-#endif
        /* Update jiffies first */
        select_nohz_load_balancer(0);
        tick_do_update_jiffies64(now);
        update_cpu_load_nohz();
 
+       touch_softlockup_watchdog();
+       /*
+        * Cancel the scheduled timer and restore the tick
+        */
+       ts->tick_stopped  = 0;
+       ts->idle_exittime = now;
+
+       tick_nohz_restart(ts, now);
+}
+
+static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
+{
 #ifndef CONFIG_VIRT_CPU_ACCOUNTING
+       unsigned long ticks;
        /*
         * We stopped the tick in idle. Update process times would miss the
         * time we slept as update_process_times does only a 1 tick
@@ -569,15 +582,6 @@ static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
        if (ticks && ticks < LONG_MAX)
                account_idle_ticks(ticks);
 #endif
-
-       touch_softlockup_watchdog();
-       /*
-        * Cancel the scheduled timer and restore the tick
-        */
-       ts->tick_stopped  = 0;
-       ts->idle_exittime = now;
-
-       tick_nohz_restart(ts, now);
 }
 
 /**
@@ -605,8 +609,10 @@ void tick_nohz_idle_exit(void)
        if (ts->idle_active)
                tick_nohz_stop_idle(cpu, now);
 
-       if (ts->tick_stopped)
+       if (ts->tick_stopped) {
                tick_nohz_restart_sched_tick(ts, now);
+               tick_nohz_account_idle_ticks(ts);
+       }
 
        local_irq_enable();
 }
@@ -811,7 +817,8 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
                 */
                if (ts->tick_stopped) {
                        touch_softlockup_watchdog();
-                       ts->idle_jiffies++;
+                       if (idle_cpu(cpu))
+                               ts->idle_jiffies++;
                }
                update_process_times(user_mode(regs));
                profile_tick(CPU_PROFILING);