#endif /* CONFIG_HOTPLUG_CPU */
+#ifdef CONFIG_HOTPLUG_PARALLEL
+static bool __cpuhp_parallel_bringup __ro_after_init = true;
+
+static int __init no_parallel_bringup(char *str)
+{
+ __cpuhp_parallel_bringup = false;
+ return 0;
+}
+early_param("cpu_hotplug_parallel_off", no_parallel_bringup);
+
+/*
+ * On architectures which have enabled parallel bringup this invokes all BP
+ * prepare states for each of the to be onlined APs. The last state sends
+ * the startup IPI to the APs. The APs proceed through the low level
+ * bringup code in parallel and then wait for the control CPU to release
+ * them one by one for the final onlining procedure.
+ *
+ * This avoids waiting for each AP to respond to the startup IPI in
+ * CPUHP_BRINGUP_CPU.
+ */
+static void cpuhp_bringup_cpus_parallel(void)
+{
+ unsigned int cpu, n = 1;
+
+ if (!__cpuhp_parallel_bringup)
+ return;
+
+ for_each_present_cpu(cpu) {
+ if (n++ >= setup_max_cpus)
+ break;
+ cpu_up(cpu, CPUHP_BP_KICK_AP);
+ }
+}
+
+static inline void cpuhp_init_parallel_bringup(void)
+{
+ if (__cpuhp_parallel_bringup)
+ __cpuhp_parallel_bringup = arch_cpuhp_init_parallel_bringup();
+}
+
+#else
+static inline void cpuhp_bringup_cpus_parallel(void) { }
+static inline void cpuhp_init_parallel_bringup(void) { }
+#endif
+
/*
* Architectures that need SMT-specific errata handling during SMT hotplug
* should override this.
{
unsigned int cpu;
+ /* If enabled, run the "parallel" bringup states first. */
+ cpuhp_bringup_cpus_parallel(setup_max_cpus);
+
+ /* Do the per CPU serialized bringup to ONLINE state */
for_each_present_cpu(cpu) {
if (num_online_cpus() >= setup_max_cpus)
break;
- if (!cpu_online(cpu))
- cpu_up(cpu, CPUHP_ONLINE);
+
+ if (!cpu_online(cpu)) {
+ struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
+ int ret = cpu_up(cpu, CPUHP_ONLINE);
+
+ /*
+ * If parallel bringup is enabled, then the online
+ * attempt has only rolled back to CPUHP_BP_KICK_AP.
+ * Do the remaining cleanups. NOOP for the non
+ * parallel case.
+ */
+ if (ret && can_rollback_cpu(st))
+ WARN_ON(cpuhp_invoke_callback_range(false, cpu, st, CPUHP_OFFLINE));
+ }
}
}