From: Atish Patra Date: Fri, 23 Jun 2017 19:32:57 +0000 (-0600) Subject: sched: Move the loadavg code to a more obvious location X-Git-Tag: v4.1.12-106.0.20170720_1900~18 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=f88d5c9278d6912710c01b94b088faa12bc1248e;p=users%2Fjedix%2Flinux-maple.git sched: Move the loadavg code to a more obvious location A previous commit f33dfff75d968 ("sched/fair: Rewrite runnable load and utilization average tracking") created a regression in global load average in uptime. Active Load average computation function should be invoked periodically to update the delta for each runqueue. Use the following upstream commit 3289bdb42 to fix this in stead of quick-fix. Before the fix procs_ when running load average ======== ======= ================= 13:32:46 1 0.65, 0.22, 0.08 13:33:47 129 0.78, 0.33, 0.12 13:34:47 129 0.74, 0.41, 0.16 13:35:47 129 0.60, 0.42, 0.18 13:36:47 129 0.77, 0.49, 0.22 13:37:47 129 0.78, 0.55, 0.26 After the fix: procs_ when running load average ======== ======= ================= 19:46:35 1 0.58, 0.38, 0.16 19:47:35 129 74.02, 21.09, 7.27 19:48:35 129 103.16, 39.08, 14.31 19:49:35 129 114.25, 53.95, 20.98 19:52:36 257 172.40, 97.26, 42.96 19:53:37 257 221.54, 124.95, 55.87 19:54:37 257 237.13, 147.05, 67.80 Original upstream commit message: I could not find the loadavg code.. turns out it was hidden in a file called proc.c. It further got mingled up with the cruft per rq load indexes (which we really want to get rid of). Move the per rq load indexes into the fair.c load-balance code (that's the only thing that uses them) and rename proc.c to loadavg.c so we can find it again. Orabug: 26266279 Signed-off-by: Peter Zijlstra (Intel) Cc: Borislav Petkov Cc: H. Peter Anvin Cc: Paul Gortmaker Cc: Thomas Gleixner [ Did minor cleanups to the code. ] Signed-off-by: Ingo Molnar (cherry picked from commit 3289bdb429884c0279bf9ab72dff7b934f19dfc6) Conflicts: kernel/sched/fair.c kernel/sched/loadavg.c kernel/sched/sched.h Signed-off-by: Vijay Kumar Signed-off-by: Atish Patra Signed-off-by: Allen Pais Reviewed-by: Dhaval Giani --- diff --git a/include/linux/sched.h b/include/linux/sched.h index 16a61ab87c84..94f9bfdd7271 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -174,7 +174,12 @@ extern unsigned long nr_iowait_cpu(int cpu); extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load); extern void calc_global_load(unsigned long ticks); + +#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) extern void update_cpu_load_nohz(void); +#else +static inline void update_cpu_load_nohz(void) { } +#endif extern unsigned long get_parent_ip(unsigned long addr); diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 46be87024875..67687973ce80 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -11,7 +11,7 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer endif -obj-y += core.o proc.o clock.o cputime.o +obj-y += core.o loadavg.o clock.o cputime.o obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o obj-y += wait.o completion.o idle.o obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 92ca59411c6f..a564e3c90bcf 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2423,9 +2423,9 @@ unsigned long nr_iowait_cpu(int cpu) void get_iowait_load(unsigned long *nr_waiters, unsigned long *load) { - struct rq *this = this_rq(); - *nr_waiters = atomic_read(&this->nr_iowait); - *load = this->cpu_load[0]; + struct rq *rq = this_rq(); + *nr_waiters = atomic_read(&rq->nr_iowait); + *load = rq->load.weight; } #ifdef CONFIG_SMP @@ -2523,6 +2523,7 @@ void scheduler_tick(void) update_rq_clock(rq); curr->sched_class->task_tick(rq, curr, 0); update_cpu_load_active(rq); + calc_global_load_tick(rq); raw_spin_unlock(&rq->lock); perf_event_task_tick(); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index e94d6a6c1c88..3a7f8a4e66de 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4311,7 +4311,7 @@ static unsigned long weighted_cpuload(const int cpu) */ static void update_idle_cpu_load(struct rq *this_rq) { - unsigned long curr_jiffies = READ_ONCE(jiffies); + unsigned long curr_jiffies = ACCESS_ONCE(jiffies); unsigned long load = weighted_cpuload(cpu_of(this_rq)); unsigned long pending_updates; @@ -4333,7 +4333,7 @@ static void update_idle_cpu_load(struct rq *this_rq) void update_cpu_load_nohz(void) { struct rq *this_rq = this_rq(); - unsigned long curr_jiffies = READ_ONCE(jiffies); + unsigned long curr_jiffies = ACCESS_ONCE(jiffies); unsigned long pending_updates; if (curr_jiffies == this_rq->last_load_update_tick) diff --git a/kernel/sched/proc.c b/kernel/sched/loadavg.c similarity index 94% rename from kernel/sched/proc.c rename to kernel/sched/loadavg.c index a5d8787f0056..2b07e90e229e 100644 --- a/kernel/sched/proc.c +++ b/kernel/sched/loadavg.c @@ -1,7 +1,9 @@ /* - * kernel/sched/proc.c + * kernel/sched/loadavg.c * - * Kernel load calculations, forked from sched/core.c + * This file contains the magic bits required to compute the global loadavg + * figure. Its a silly number but people think its important. We go through + * great pains to make it work on big machines and tickless kernels. */ #include @@ -81,7 +83,7 @@ long calc_load_fold_active(struct rq *this_rq) long nr_active, delta = 0; nr_active = this_rq->nr_running; - nr_active += (long) this_rq->nr_uninterruptible; + nr_active += (long)this_rq->nr_uninterruptible; if (nr_active != this_rq->calc_load_active) { delta = nr_active - this_rq->calc_load_active; @@ -186,6 +188,7 @@ void calc_load_enter_idle(void) delta = calc_load_fold_active(this_rq); if (delta) { int idx = calc_load_write_idx(); + atomic_long_add(delta, &calc_load_idle[idx]); } } @@ -241,18 +244,20 @@ fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n) { unsigned long result = 1UL << frac_bits; - if (n) for (;;) { - if (n & 1) { - result *= x; - result += 1UL << (frac_bits - 1); - result >>= frac_bits; + if (n) { + for (;;) { + if (n & 1) { + result *= x; + result += 1UL << (frac_bits - 1); + result >>= frac_bits; + } + n >>= 1; + if (!n) + break; + x *= x; + x += 1UL << (frac_bits - 1); + x >>= frac_bits; } - n >>= 1; - if (!n) - break; - x *= x; - x += 1UL << (frac_bits - 1); - x >>= frac_bits; } return result; @@ -285,7 +290,6 @@ static unsigned long calc_load_n(unsigned long load, unsigned long exp, unsigned long active, unsigned int n) { - return calc_load(load, fixed_power_int(exp, FSHIFT, n), active); } @@ -339,6 +343,8 @@ static inline void calc_global_nohz(void) { } /* * calc_load - update the avenrun load estimates 10 ticks after the * CPUs have updated calc_load_tasks. + * + * Called from the global timer code. */ void calc_global_load(unsigned long ticks) { @@ -370,10 +376,10 @@ void calc_global_load(unsigned long ticks) } /* - * Called from update_cpu_load() to periodically update this CPU's + * Called from scheduler_tick() to periodically update this CPU's * active count. */ -static void calc_load_account_active(struct rq *this_rq) +void calc_global_load_tick(struct rq *this_rq) { long delta; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index b7962f5f5800..b5116cc26655 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -27,8 +27,14 @@ extern __read_mostly int scheduler_running; extern unsigned long calc_load_update; extern atomic_long_t calc_load_tasks; +extern void calc_global_load_tick(struct rq *this_rq); extern long calc_load_fold_active(struct rq *this_rq); + +#ifdef CONFIG_SMP extern void update_cpu_load_active(struct rq *this_rq); +#else +static inline void update_cpu_load_active(struct rq *this_rq) { } +#endif /* * Helpers for converting nanosecond timing to jiffy resolution