From: David Woodhouse Date: Tue, 7 Mar 2023 19:06:50 +0000 (+0000) Subject: x86/smpboot: Allow parallel bringup for SEV-ES X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=8a53f16b29065ddbde40a73415dae32d2f4c82e5;p=users%2Fdwmw2%2Flinux.git x86/smpboot: Allow parallel bringup for SEV-ES Enable parallel bringup for SEV-ES guests. The APs can't actually execute the CPUID instruction directly during early startup, but they can make the GHCB call directly instead, just as the VC trap handler would do. Factor out a prepare_parallel_bringup() function to help reduce the level of complexity by allowing a simple 'return false' in the bail-out cases/ Thanks to Sabin for talking me through the way this works. Suggested-by: Sabin Rapan Signed-off-by: David Woodhouse --- diff --git a/arch/x86/include/asm/sev-common.h b/arch/x86/include/asm/sev-common.h index b8357d6ecd47e..f25df4bd318ec 100644 --- a/arch/x86/include/asm/sev-common.h +++ b/arch/x86/include/asm/sev-common.h @@ -70,6 +70,7 @@ /* GHCBData[63:12] */ \ (((u64)(v) & GENMASK_ULL(63, 12)) >> 12) +#ifndef __ASSEMBLY__ /* * SNP Page State Change Operation * @@ -160,6 +161,8 @@ struct snp_psc_desc { #define GHCB_RESP_CODE(v) ((v) & GHCB_MSR_INFO_MASK) +#endif /* __ASSEMBLY__ */ + /* * Error codes related to GHCB input that can be communicated back to the guest * by setting the lower 32-bits of the GHCB SW_EXITINFO1 field to 2. diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index defe76ee9e645..b3f67a764bfad 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -204,7 +204,8 @@ extern unsigned int smpboot_control; /* Control bits for startup_64 */ #define STARTUP_APICID_CPUID_0B 0x80000000 #define STARTUP_APICID_CPUID_01 0x40000000 +#define STARTUP_APICID_SEV_ES 0x20000000 -#define STARTUP_PARALLEL_MASK (STARTUP_APICID_CPUID_01 | STARTUP_APICID_CPUID_0B) +#define STARTUP_PARALLEL_MASK (STARTUP_APICID_CPUID_01 | STARTUP_APICID_CPUID_0B | STARTUP_APICID_SEV_ES) #endif /* _ASM_X86_SMP_H */ diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index c35f7c1738321..156a292e94fb4 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -26,6 +26,7 @@ #include #include #include +#include /* * We are not able to switch in one step to the final KERNEL ADDRESS SPACE @@ -242,6 +243,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) * * Bit 31 STARTUP_APICID_CPUID_0B flag (use CPUID 0x0b) * Bit 30 STARTUP_APICID_CPUID_01 flag (use CPUID 0x01) + * Bit 29 STARTUP_APICID_SEV_ES flag (CPUID 0x0b via GHCB MSR) * Bit 0-24 CPU# if STARTUP_APICID_CPUID_xx flags are not set */ movl smpboot_control(%rip), %ecx @@ -249,6 +251,8 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) jnz .Luse_cpuid_0b testl $STARTUP_APICID_CPUID_01, %ecx jnz .Luse_cpuid_01 + testl $STARTUP_APICID_SEV_ES, %ecx + jnz .Luse_sev_cpuid_0b andl $0x0FFFFFFF, %ecx jmp .Lsetup_cpu @@ -259,6 +263,28 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) shr $24, %edx jmp .Lsetup_AP +.Luse_sev_cpuid_0b: + /* Set the GHCB MSR to request CPUID 0xB_EDX */ + movl $MSR_AMD64_SEV_ES_GHCB, %ecx + movl $(GHCB_CPUID_REQ_EDX << 30) | GHCB_MSR_CPUID_REQ, %eax + movl $0x0B, %edx + wrmsr + + /* Perform GHCB MSR protocol */ + vmgexit + + /* + * Get the result. After the RDMSR: + * EAX should be 0xc0000005 + * EDX should have the CPUID register value and since EDX + * is the target register, no need to move the result. + */ + rdmsr + andl $GHCB_MSR_INFO_MASK, %eax + cmpl $GHCB_MSR_CPUID_RESP, %eax + jne 1f + jmp .Lsetup_AP + .Luse_cpuid_0b: mov $0x0B, %eax xorl %ecx, %ecx diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 9d956571ecc1a..d194c4ffeef86 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1510,6 +1510,71 @@ void __init smp_prepare_cpus_common(void) set_cpu_sibling_map(0); } + +/* + * We can do 64-bit AP bringup in parallel if the CPU reports its APIC + * ID in CPUID (either leaf 0x0B if we need the full APIC ID in X2APIC + * mode, or leaf 0x01 if 8 bits are sufficient). Otherwise it's too + * hard. And not for SEV-ES guests because they can't use CPUID that + * early. + */ +static bool __init prepare_parallel_bringup(void) +{ + if (IS_ENABLED(CONFIG_X86_32) || boot_cpu_data.cpuid_level < 1) + return false; + + if (x2apic_mode) { + unsigned int eax, ebx, ecx, edx; + + if (boot_cpu_data.cpuid_level < 0xb) + return false; + + /* + * To support parallel bringup in x2apic mode, the AP will need + * to obtain its APIC ID from CPUID 0x0B, since CPUID 0x01 has + * only 8 bits. Check that it is present and seems correct. + */ + cpuid_count(0xb, 0, &eax, &ebx, &ecx, &edx); + + /* + * AMD says that if executed with an umimplemented level in + * ECX, then it will return all zeroes in EAX. Intel says it + * will return zeroes in both EAX and EBX. Checking only EAX + * should be sufficient. + */ + if (!eax) { + pr_info("Disabling parallel bringup because CPUID 0xb looks untrustworthy\n"); + return false; + } + + if (IS_ENABLED(AMD_MEM_ENCRYPT) && static_branch_unlikely(&sev_es_enable_key)) { + pr_debug("Using SEV-ES CPUID 0xb for parallel CPU startup\n"); + smpboot_control = STARTUP_APICID_SEV_ES; + } else if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) { + /* + * Other forms of memory encryption need to implement a way of + * finding the APs' APIC IDs that early. + */ + return false; + } else { + pr_debug("Using CPUID 0xb for parallel CPU startup\n"); + smpboot_control = STARTUP_APICID_CPUID_0B; + } + } else { + if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) + return false; + + /* Without X2APIC, what's in CPUID 0x01 should suffice. */ + pr_debug("Using CPUID 0x1 for parallel CPU startup\n"); + smpboot_control = STARTUP_APICID_CPUID_01; + } + + cpuhp_setup_state_nocalls(CPUHP_BP_PARALLEL_DYN, "x86/cpu:kick", + native_cpu_kick, NULL); + + return true; +} + /* * Prepare for SMP bootup. * @max_cpus: configured maximum number of CPUs, It is a legacy parameter @@ -1550,51 +1615,8 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus) speculative_store_bypass_ht_init(); - /* - * We can do 64-bit AP bringup in parallel if the CPU reports - * its APIC ID in CPUID (either leaf 0x0B if we need the full - * APIC ID in X2APIC mode, or leaf 0x01 if 8 bits are - * sufficient). Otherwise it's too hard. And not for SEV-ES - * guests because they can't use CPUID that early. - */ - if (IS_ENABLED(CONFIG_X86_32) || boot_cpu_data.cpuid_level < 1 || - (x2apic_mode && boot_cpu_data.cpuid_level < 0xb) || - cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) - do_parallel_bringup = false; - - if (do_parallel_bringup && x2apic_mode) { - unsigned int eax, ebx, ecx, edx; - - /* - * To support parallel bringup in x2apic mode, the AP will need - * to obtain its APIC ID from CPUID 0x0B, since CPUID 0x01 has - * only 8 bits. Check that it is present and seems correct. - */ - cpuid_count(0xb, 0, &eax, &ebx, &ecx, &edx); - - /* - * AMD says that if executed with an umimplemented level in - * ECX, then it will return all zeroes in EAX. Intel says it - * will return zeroes in both EAX and EBX. Checking only EAX - * should be sufficient. - */ - if (eax) { - pr_debug("Using CPUID 0xb for parallel CPU startup\n"); - smpboot_control = STARTUP_APICID_CPUID_0B; - } else { - pr_info("Disabling parallel bringup because CPUID 0xb looks untrustworthy\n"); - do_parallel_bringup = false; - } - } else if (do_parallel_bringup) { - /* Without X2APIC, what's in CPUID 0x01 should suffice. */ - pr_debug("Using CPUID 0x1 for parallel CPU startup\n"); - smpboot_control = STARTUP_APICID_CPUID_01; - } - - if (do_parallel_bringup) { - cpuhp_setup_state_nocalls(CPUHP_BP_PARALLEL_DYN, "x86/cpu:kick", - native_cpu_kick, NULL); - } + if (do_parallel_bringup) + do_parallel_bringup = prepare_parallel_bringup(); snp_set_wakeup_secondary_cpu(); }