x86/smp: Bring up secondary CPUs in two stages

Now that Thomas fixed the bringup we can do it in parallel for x86

We still can't do the callin part because it gets intertwined with
the TSC sync, so leave that part for later.

This reduces the time taken for bringup on my 28-thread Haswell
system by about a third.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index bd65ed446425fff2915539d14376a204a9b318a5..d8f51e4502301366f923882e3a9b2b1b6f11b6cc 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -57,6 +57,7 @@
 #include <linux/pgtable.h>
 #include <linux/overflow.h>
 #include <linux/syscore_ops.h>
+#include <linux/smpboot.h>
 
 #include <asm/acpi.h>
 #include <asm/desc.h>
@@ -1228,18 +1229,18 @@ unreg_nmi:
        return ret;
 }
 
+/* We aren't ready for this part yet */
+static int i_fixed_parallel_tsc_sync = false;
+
 int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
 {
        int ret;
 
-       ret = do_cpu_up(cpu, tidle);
-       if (ret)
-               return ret;
-
-       ret = do_wait_cpu_initialized(cpu);
-       if (ret)
-               return ret;
-
+       if (!i_fixed_parallel_tsc_sync) {
+               ret = do_wait_cpu_initialized(cpu);
+               if (ret)
+                       return ret;
+       }
        ret = do_wait_cpu_callin(cpu);
        if (ret)
                return ret;
@@ -1256,6 +1257,16 @@ int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
        return ret;
 }
 
+int native_cpu_kick(unsigned int cpu)
+{
+       return do_cpu_up(cpu, idle_thread_get(cpu));
+}
+
+int native_cpu_wait_init(unsigned int cpu)
+{
+       return do_wait_cpu_initialized(cpu);
+}
+
 /**
  * arch_disable_smp_support() - disables SMP support for x86 at runtime
  */
@@ -1427,6 +1438,12 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
        smp_quirk_init_udelay();
 
        speculative_store_bypass_ht_init();
+
+       cpuhp_setup_state_nocalls(CPUHP_BP_PARALLEL_DYN, "x86/cpu:kick",
+                                 native_cpu_kick, NULL);
+       if (i_fixed_parallel_tsc_sync)
+               cpuhp_setup_state_nocalls(CPUHP_BP_PARALLEL_DYN, "x86/cpu:wait-init",
+                                         native_cpu_wait_init, NULL);
 }
 
 void arch_thaw_secondary_cpus_begin(void)