sched: Improve latencies and throughput

Make the idle balancer more agressive, to improve a
x264 encoding workload provided by Jason Garrett-Glaser:

 NEXT_BUDDY NO_LB_BIAS
 encoded 600 frames, 252.82 fps, 22096.60 kb/s
 encoded 600 frames, 250.69 fps, 22096.60 kb/s
 encoded 600 frames, 245.76 fps, 22096.60 kb/s

 NO_NEXT_BUDDY LB_BIAS
 encoded 600 frames, 344.44 fps, 22096.60 kb/s
 encoded 600 frames, 346.66 fps, 22096.60 kb/s
 encoded 600 frames, 352.59 fps, 22096.60 kb/s

 NO_NEXT_BUDDY NO_LB_BIAS
 encoded 600 frames, 425.75 fps, 22096.60 kb/s
 encoded 600 frames, 425.45 fps, 22096.60 kb/s
 encoded 600 frames, 422.49 fps, 22096.60 kb/s

Peter pointed out that this is better done via newidle_idx,
not via LB_BIAS, newidle balancing should look for where
there is load _now_, not where there was load 2 ticks ago.

Worst-case latencies are improved as well as no buddies
means less vruntime spread. (as per prior lkml discussions)

This change improves kbuild-peak parallelism as well.

Reported-by: Jason Garrett-Glaser <darkshikari@gmail.com>
Signed-off-by: Mike Galbraith <efault@gmx.de>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <1253011667.9128.16.camel@marge.simson.net>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/ia64/include/asm/topology.h b/arch/ia64/include/asm/topology.h
index 47f3c51d5e27aae240796a5904f662a2403d075b..42f1673ec83f59075174ce6b7c40b4e432c674d5 100644 (file)
--- a/arch/ia64/include/asm/topology.h
+++ b/arch/ia64/include/asm/topology.h
@@ -61,7 +61,7 @@ void build_cpu_to_node_map(void);
        .cache_nice_tries       = 2,                    \
        .busy_idx               = 2,                    \
        .idle_idx               = 1,                    \
-       .newidle_idx            = 2,                    \
+       .newidle_idx            = 0,                    \
        .wake_idx               = 0,                    \
        .forkexec_idx           = 1,                    \
        .flags                  = SD_LOAD_BALANCE       \
@@ -87,10 +87,11 @@ void build_cpu_to_node_map(void);
        .cache_nice_tries       = 2,                    \
        .busy_idx               = 3,                    \
        .idle_idx               = 2,                    \
-       .newidle_idx            = 2,                    \
+       .newidle_idx            = 0,                    \
        .wake_idx               = 0,                    \
        .forkexec_idx           = 1,                    \
        .flags                  = SD_LOAD_BALANCE       \
+                               | SD_BALANCE_NEWIDLE    \
                                | SD_BALANCE_EXEC       \
                                | SD_BALANCE_FORK       \
                                | SD_BALANCE_WAKE       \

diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h
index a6b220ab56db1f97af4924d05d047b28dafeb55b..1a2c9eb42a035c8d05c91916384363a4045a47a2 100644 (file)
--- a/arch/powerpc/include/asm/topology.h
+++ b/arch/powerpc/include/asm/topology.h
@@ -57,7 +57,7 @@ static inline int pcibus_to_node(struct pci_bus *bus)
        .cache_nice_tries       = 1,                    \
        .busy_idx               = 3,                    \
        .idle_idx               = 1,                    \
-       .newidle_idx            = 2,                    \
+       .newidle_idx            = 0,                    \
        .wake_idx               = 0,                    \
        .flags                  = SD_LOAD_BALANCE       \
                                | SD_BALANCE_EXEC       \

diff --git a/arch/sh/include/asm/topology.h b/arch/sh/include/asm/topology.h
index 9054e5c0ad54959f7bcc199f6a965d847bef0ce4..c8436771e31dfc44dea9ba1233b6ecd67c188b57 100644 (file)
--- a/arch/sh/include/asm/topology.h
+++ b/arch/sh/include/asm/topology.h
@@ -15,13 +15,14 @@
        .cache_nice_tries       = 2,                    \
        .busy_idx               = 3,                    \
        .idle_idx               = 2,                    \
-       .newidle_idx            = 2,                    \
+       .newidle_idx            = 0,                    \
        .wake_idx               = 0,                    \
        .forkexec_idx           = 1,                    \
        .flags                  = SD_LOAD_BALANCE       \
                                | SD_BALANCE_FORK       \
                                | SD_BALANCE_EXEC       \
                                | SD_BALANCE_WAKE       \
+                               | SD_BALANCE_NEWIDLE    \
                                | SD_SERIALIZE,         \
        .last_balance           = jiffies,              \
        .balance_interval       = 1,                    \

diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index 4b1b335097b597d622f20f9975c260a1a6673b4e..7fafd1bc41494cfc50893cf6ac479b438fd425b5 100644 (file)
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -116,14 +116,12 @@ extern unsigned long node_remap_size[];
 
 # define SD_CACHE_NICE_TRIES   1
 # define SD_IDLE_IDX           1
-# define SD_NEWIDLE_IDX                2
 # define SD_FORKEXEC_IDX       0
 
 #else
 
 # define SD_CACHE_NICE_TRIES   2
 # define SD_IDLE_IDX           2
-# define SD_NEWIDLE_IDX                2
 # define SD_FORKEXEC_IDX       1
 
 #endif
@@ -137,7 +135,7 @@ extern unsigned long node_remap_size[];
        .cache_nice_tries       = SD_CACHE_NICE_TRIES,                  \
        .busy_idx               = 3,                                    \
        .idle_idx               = SD_IDLE_IDX,                          \
-       .newidle_idx            = SD_NEWIDLE_IDX,                       \
+       .newidle_idx            = 0,                                    \
        .wake_idx               = 0,                                    \
        .forkexec_idx           = SD_FORKEXEC_IDX,                      \
                                                                        \

diff --git a/include/linux/topology.h b/include/linux/topology.h
index c87edcd8796751339b934c254c222827e4dfd7c0..4298745615a5e2b81239e3b025b8a8bdbf387737 100644 (file)
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -151,7 +151,7 @@ int arch_update_cpu_topology(void);
        .cache_nice_tries       = 1,                                    \
        .busy_idx               = 2,                                    \
        .idle_idx               = 1,                                    \
-       .newidle_idx            = 2,                                    \
+       .newidle_idx            = 0,                                    \
        .wake_idx               = 0,                                    \
        .forkexec_idx           = 1,                                    \
                                                                        \

diff --git a/kernel/sched_features.h b/kernel/sched_features.h
index 891ea0f72b461106a721dc22d04531f492263e72..e98c2e8de1d507c0889406cafc81a2ce3e0153d4 100644 (file)
--- a/kernel/sched_features.h
+++ b/kernel/sched_features.h
@@ -67,7 +67,7 @@ SCHED_FEAT(AFFINE_WAKEUPS, 1)
  * wakeup-preemption), since its likely going to consume data we
  * touched, increases cache locality.
  */
-SCHED_FEAT(NEXT_BUDDY, 1)
+SCHED_FEAT(NEXT_BUDDY, 0)
 
 /*
  * Prefer to schedule the task that ran last (when we did