sched: Cure nr_iowait_cpu() users

Commit 0224cf4c5e (sched: Intoduce get_cpu_iowait_time_us())
broke things by not making sure preemption was indeed disabled
by the callers of nr_iowait_cpu() which took the iowait value of
the current cpu.

This resulted in a heap of preempt warnings. Cure this by making
nr_iowait_cpu() take a cpu number and fix up the callers to pass
in the right number.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Arjan van de Ven <arjan@infradead.org>
Cc: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Cc: Rafael J. Wysocki <rjw@sisk.pl>
Cc: Maxim Levitsky <maximlevitsky@gmail.com>
Cc: Len Brown <len.brown@intel.com>
Cc: Pavel Machek <pavel@ucw.cz>
Cc: Jiri Slaby <jslaby@suse.cz>
Cc: linux-pm@lists.linux-foundation.org
LKML-Reference: <1277968037.1868.120.camel@laptop>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 52ff8aa63f8449b04d5bcd046b028e23dcf6d68b..1b128702d300bc5a07ba7ec09f03ec83f3b11b43 100644 (file)
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -143,7 +143,7 @@ static inline int which_bucket(unsigned int duration)
         * This allows us to calculate
         * E(duration)|iowait
         */
-       if (nr_iowait_cpu())
+       if (nr_iowait_cpu(smp_processor_id()))
                bucket = BUCKETS/2;
 
        if (duration < 10)
@@ -175,7 +175,7 @@ static inline int performance_multiplier(void)
        mult += 2 * get_loadavg();
 
        /* for IO wait tasks (per cpu!) we add 5x each */
-       mult += 10 * nr_iowait_cpu();
+       mult += 10 * nr_iowait_cpu(smp_processor_id());
 
        return mult;
 }

diff --git a/include/linux/sched.h b/include/linux/sched.h
index f118809c953ff137b2a97e0d9e674b2836ee331a..747fcaedddb70d6cb0a49b43c0d2f2a6f23a693f 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -139,7 +139,7 @@ extern int nr_processes(void);
 extern unsigned long nr_running(void);
 extern unsigned long nr_uninterruptible(void);
 extern unsigned long nr_iowait(void);
-extern unsigned long nr_iowait_cpu(void);
+extern unsigned long nr_iowait_cpu(int cpu);
 extern unsigned long this_cpu_load(void);
 
 

diff --git a/kernel/sched.c b/kernel/sched.c
index a24d6d5d83f63a089da658f1eb5ae0c708eaa4bd..f87abe3b0176613d03d722b4ed74fc56f3597dfa 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -2864,9 +2864,9 @@ unsigned long nr_iowait(void)
        return sum;
 }
 
-unsigned long nr_iowait_cpu(void)
+unsigned long nr_iowait_cpu(int cpu)
 {
-       struct rq *this = this_rq();
+       struct rq *this = cpu_rq(cpu);
        return atomic_read(&this->nr_iowait);
 }
 

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 1d7b9bc1c0340e8deccbc5df1418fd71889d837d..1a6f828e57a05cd91168192c01c229a6353e57f2 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -154,14 +154,14 @@ static void tick_nohz_update_jiffies(ktime_t now)
  * Updates the per cpu time idle statistics counters
  */
 static void
-update_ts_time_stats(struct tick_sched *ts, ktime_t now, u64 *last_update_time)
+update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
 {
        ktime_t delta;
 
        if (ts->idle_active) {
                delta = ktime_sub(now, ts->idle_entrytime);
                ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
-               if (nr_iowait_cpu() > 0)
+               if (nr_iowait_cpu(cpu) > 0)
                        ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
                ts->idle_entrytime = now;
        }
@@ -175,19 +175,19 @@ static void tick_nohz_stop_idle(int cpu, ktime_t now)
 {
        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 
-       update_ts_time_stats(ts, now, NULL);
+       update_ts_time_stats(cpu, ts, now, NULL);
        ts->idle_active = 0;
 
        sched_clock_idle_wakeup_event(0);
 }
 
-static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
+static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts)
 {
        ktime_t now;
 
        now = ktime_get();
 
-       update_ts_time_stats(ts, now, NULL);
+       update_ts_time_stats(cpu, ts, now, NULL);
 
        ts->idle_entrytime = now;
        ts->idle_active = 1;
@@ -216,7 +216,7 @@ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
        if (!tick_nohz_enabled)
                return -1;
 
-       update_ts_time_stats(ts, ktime_get(), last_update_time);
+       update_ts_time_stats(cpu, ts, ktime_get(), last_update_time);
 
        return ktime_to_us(ts->idle_sleeptime);
 }
@@ -242,7 +242,7 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
        if (!tick_nohz_enabled)
                return -1;
 
-       update_ts_time_stats(ts, ktime_get(), last_update_time);
+       update_ts_time_stats(cpu, ts, ktime_get(), last_update_time);
 
        return ktime_to_us(ts->iowait_sleeptime);
 }
@@ -284,7 +284,7 @@ void tick_nohz_stop_sched_tick(int inidle)
         */
        ts->inidle = 1;
 
-       now = tick_nohz_start_idle(ts);
+       now = tick_nohz_start_idle(cpu, ts);
 
        /*
         * If this cpu is offline and it is the one which updates