From: David Woodhouse <dwmw@amazon.co.uk>
Date: Tue, 14 Dec 2021 19:58:54 +0000 (+0000)
Subject: parallel part 2
X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=refs%2Fheads%2Fparallel-5.16;p=users%2Fdwmw2%2Flinux.git

parallel part 2
---

diff --git a/arch/x86/include/asm/cpumask.h b/arch/x86/include/asm/cpumask.h
index 3afa990d756b5..4dd9c167e63a9 100644
--- a/arch/x86/include/asm/cpumask.h
+++ b/arch/x86/include/asm/cpumask.h
@@ -7,6 +7,7 @@
 extern cpumask_var_t cpu_callin_mask;
 extern cpumask_var_t cpu_callout_mask;
 extern cpumask_var_t cpu_initialized_mask;
+extern cpumask_var_t cpu_finishup_mask;
 extern cpumask_var_t cpu_sibling_setup_mask;
 
 extern void setup_cpu_local_masks(void);
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 98b4697b2c9b0..0cd6373bc3f2a 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -68,6 +68,7 @@ u32 elf_hwcap2 __read_mostly;
 cpumask_var_t cpu_initialized_mask;
 cpumask_var_t cpu_callout_mask;
 cpumask_var_t cpu_callin_mask;
+cpumask_var_t cpu_finishup_mask;
 
 /* representing cpus for which sibling maps can be computed */
 cpumask_var_t cpu_sibling_setup_mask;
@@ -94,6 +95,7 @@ void __init setup_cpu_local_masks(void)
 	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
 	alloc_bootmem_cpumask_var(&cpu_callin_mask);
 	alloc_bootmem_cpumask_var(&cpu_callout_mask);
+	alloc_bootmem_cpumask_var(&cpu_finishup_mask);
 	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
 }
 
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 060fdc4db8b0d..4a85ef6fa4f4f 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -212,15 +212,20 @@ static void smp_callin(void)
 	wmb();
 
 	/*
-	 * This runs the AP through all the cpuhp states to its target
-	 * state (CPUHP_ONLINE in the case of serial bringup).
+	 * Allow the master to continue.
 	 */
-	notify_cpu_starting(cpuid);
+	cpumask_set_cpu(cpuid, cpu_callin_mask);
+	while (!cpumask_test_cpu(cpuid, cpu_finishup_mask))
+		cpu_relax();
 
 	/*
-	 * Allow the master to continue.
+	 * This runs the AP through all the cpuhp states to its target
+	 * state (CPUHP_ONLINE in the case of serial bringup).
 	 */
-	cpumask_set_cpu(cpuid, cpu_callin_mask);
+	static arch_spinlock_t cpuhp_lock = __ARCH_SPIN_LOCK_UNLOCKED;
+	arch_spin_lock(&cpuhp_lock);
+	notify_cpu_starting(cpuid);
+	arch_spin_unlock(&cpuhp_lock);
 }
 
 static int cpu0_logical_apicid;
@@ -1145,7 +1150,7 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle,
 	init_espfix_ap(cpu);
 
 	/* So we see what's up */
-	announce_cpu(cpu, apicid);
+	//announce_cpu(cpu, apicid);
 
 	/*
 	 * This grunge runs the startup process for
@@ -1238,7 +1243,11 @@ static int do_wait_cpu_callin(unsigned int cpu)
 	/*
 	 * Wait till AP completes initial initialization.
 	 */
-	return do_wait_cpu_cpumask(cpu, cpu_callin_mask);
+	if (do_wait_cpu_cpumask(cpu, cpu_callin_mask))
+		return -1;
+
+	cpumask_set_cpu(cpu, cpu_finishup_mask);
+	return 0;
 }
 
 /*
@@ -1338,17 +1347,20 @@ int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
 {
 	int ret;
 
+	t[3] = get_cycles();
 	/* If parallel AP bringup isn't enabled, perform the first steps now. */
 	if (!do_parallel_bringup) {
 		ret = do_cpu_up(cpu, tidle);
 		if (ret)
 			return ret;
+
+		ret = do_wait_cpu_initialized(cpu);
+		if (ret)
+			return ret;
 	}
-	t[3] = get_cycles();
-	ret = do_wait_cpu_initialized(cpu);
-	if (ret)
-		return ret;
+
 	t[4] = get_cycles();
+
 	ret = do_wait_cpu_callin(cpu);
 	if (ret)
 		return ret;
@@ -1559,9 +1571,12 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 	    cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
 		do_parallel_bringup = false;
 
-	if (do_parallel_bringup)
+	if (do_parallel_bringup) {
 		cpuhp_setup_state_nocalls(CPUHP_BP_PARALLEL_DYN, "x86/cpu:kick",
 					  native_cpu_kick, NULL);
+		cpuhp_setup_state_nocalls(CPUHP_BP_PARALLEL_DYN, "x86/cpu:wait-init",
+					  do_wait_cpu_initialized, NULL);
+	}
 }
 
 void arch_thaw_secondary_cpus_begin(void)
@@ -1583,6 +1598,7 @@ void __init native_smp_prepare_boot_cpu(void)
 	switch_to_new_gdt(me);
 	/* already set me in cpu_online_mask in boot_cpu_init() */
 	cpumask_set_cpu(me, cpu_callout_mask);
+	cpumask_set_cpu(me, cpu_finishup_mask);
 	cpu_set_state_online(me);
 	native_pv_lock_init();
 }
@@ -1763,6 +1779,7 @@ static void remove_cpu_from_maps(int cpu)
 {
 	set_cpu_online(cpu, false);
 	cpumask_clear_cpu(cpu, cpu_callout_mask);
+	cpumask_clear_cpu(cpu, cpu_finishup_mask);
 	cpumask_clear_cpu(cpu, cpu_callin_mask);
 	/* was set by cpu_init() */
 	cpumask_clear_cpu(cpu, cpu_initialized_mask);
diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c
index ff37dff20dc0a..6f4a6bfa2c83d 100644
--- a/arch/x86/xen/smp_pv.c
+++ b/arch/x86/xen/smp_pv.c
@@ -271,6 +271,7 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
 
 	/* used to tell cpu_init() that it can proceed with initialization */
 	cpumask_set_cpu(cpu, cpu_callout_mask);
+	cpumask_set_cpu(cpu, cpu_finishup_mask);
 	if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
 		return 0;