cpu/hotplug: Allow "parallel" bringup up to CPUHP_BP_KICK_AP_STATE
authorThomas Gleixner <tglx@linutronix.de>
Wed, 29 Mar 2023 22:20:07 +0000 (00:20 +0200)
committerDavid Woodhouse <dwmw@amazon.co.uk>
Thu, 30 Mar 2023 12:27:08 +0000 (14:27 +0200)
There is often significant latency in the early stages of CPU bringup, and
time is wasted by waking each CPU (e.g. with SIPI/INIT/INIT on x86) and
then waiting for it to respond before moving on to the next.

Allow a platform to enable parallel setup which brings all to be onlined
CPUs up to the CPUHP_BP_KICK_AP state. While this state advancement on the
control CPU (BP) is single-threaded the important part is the last state
CPUHP_BP_KICK_AP which wakes the to be onlined CPUs up.

This allows the CPUs to run up to the first sychronization point
cpuhp_ap_sync_alive() where they wait for the control CPU to release them
one by one for the full onlining procedure.

This parallelism depends on the CPU hotplug core sync mechanism which
ensures that the parallel brought up CPUs wait for release before touching
any state which would make the CPU visible to anything outside the hotplug
control mechanism.

Co-developed-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Documentation/admin-guide/kernel-parameters.txt
arch/Kconfig
include/linux/cpuhotplug.h
kernel/cpu.c

index 649abbfa71faae57a7571245b7e4b734f36f9192..8355a19690c35781dee73577090e14a8ae3c7156 100644 (file)
                        on every CPU online, such as boot, and resume from suspend.
                        Default: 10000
 
+       cpu_hotplug_parallel_off
+                       [SMP] Disable parallel bringup of secondary CPUs
+
        crash_kexec_post_notifiers
                        Run kdump after running panic-notifiers and dumping
                        kmsg. This only for the users who doubt kdump always
index 4bef497ab3aeee53f0b9bfa1ec33ba163a858df3..1fad7f77b6834399910ec212490e657ce4a517c6 100644 (file)
@@ -41,6 +41,10 @@ config HOTPLUG_SPLIT_STARTUP
        bool
        select HOTPLUG_CORE_SYNC
 
+config HOTPLUG_PARALLEL
+       bool
+       select HOTPLUG_SPLIT_STARTUP
+
 config GENERIC_ENTRY
        bool
 
index d63a16425112dc4b3b80ae0e48a9d3cf86ee3cdd..13408110610583e13871ab09c98fa67e4265b074 100644 (file)
@@ -528,5 +528,6 @@ void cpuhp_ap_report_dead(void);
 int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle);
 void arch_cpuhp_cleanup_kick_cpu(unsigned int cpu);
 void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu);
+bool arch_cpuhp_init_parallel_bringup(void);
 
 #endif
index 39de0d26171f52547bad175a893aced8d191ae43..0570f780f6f209ef3df27c42ae5418b92af864e0 100644 (file)
@@ -608,6 +608,51 @@ static void lockdep_release_cpus_lock(void)
 
 #endif /* CONFIG_HOTPLUG_CPU */
 
+#ifdef CONFIG_HOTPLUG_PARALLEL
+static bool __cpuhp_parallel_bringup __ro_after_init = true;
+
+static int __init no_parallel_bringup(char *str)
+{
+       __cpuhp_parallel_bringup = false;
+       return 0;
+}
+early_param("cpu_hotplug_parallel_off", no_parallel_bringup);
+
+/*
+ * On architectures which have enabled parallel bringup this invokes all BP
+ * prepare states for each of the to be onlined APs. The last state sends
+ * the startup IPI to the APs. The APs proceed through the low level
+ * bringup code in parallel and then wait for the control CPU to release
+ * them one by one for the final onlining procedure.
+ *
+ * This avoids waiting for each AP to respond to the startup IPI in
+ * CPUHP_BRINGUP_CPU.
+ */
+static void cpuhp_bringup_cpus_parallel(void)
+{
+       unsigned int cpu, n = 1;
+
+       if (!__cpuhp_parallel_bringup)
+               return;
+
+       for_each_present_cpu(cpu) {
+               if (n++ >= setup_max_cpus)
+                       break;
+               cpu_up(cpu, CPUHP_BP_KICK_AP);
+       }
+}
+
+static inline void cpuhp_init_parallel_bringup(void)
+{
+       if (__cpuhp_parallel_bringup)
+               __cpuhp_parallel_bringup = arch_cpuhp_init_parallel_bringup();
+}
+
+#else
+static inline void cpuhp_bringup_cpus_parallel(void) { }
+static inline void cpuhp_init_parallel_bringup(void) { }
+#endif
+
 /*
  * Architectures that need SMT-specific errata handling during SMT hotplug
  * should override this.
@@ -1748,11 +1793,27 @@ void bringup_nonboot_cpus(unsigned int setup_max_cpus)
 {
        unsigned int cpu;
 
+       /* If enabled, run the "parallel" bringup states first. */
+       cpuhp_bringup_cpus_parallel(setup_max_cpus);
+
+       /* Do the per CPU serialized bringup to ONLINE state */
        for_each_present_cpu(cpu) {
                if (num_online_cpus() >= setup_max_cpus)
                        break;
-               if (!cpu_online(cpu))
-                       cpu_up(cpu, CPUHP_ONLINE);
+
+               if (!cpu_online(cpu)) {
+                       struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
+                       int ret = cpu_up(cpu, CPUHP_ONLINE);
+
+                       /*
+                        * If parallel bringup is enabled, then the online
+                        * attempt has only rolled back to CPUHP_BP_KICK_AP.
+                        * Do the remaining cleanups. NOOP for the non
+                        * parallel case.
+                        */
+                       if (ret && can_rollback_cpu(st))
+                               WARN_ON(cpuhp_invoke_callback_range(false, cpu, st, CPUHP_OFFLINE));
+               }
        }
 }