x86/smpboot: Allow parallel bringup for SEV-ES

Enable parallel bringup for SEV-ES guests. The APs can't actually
execute the CPUID instruction directly during early startup, but they
can make the GHCB call directly instead, just as the VC trap handler
would do.

Thanks to Sabin for talking me through the way this works.

Suggested-by: Sabin Rapan <sabrapan@amazon.com>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Usama Arif <usama.arif@bytedance.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>
Link: https://lore.kernel.org/r/20230328195758.1049469-9-usama.arif@bytedance.com

diff --git a/arch/x86/include/asm/sev-common.h b/arch/x86/include/asm/sev-common.h
index b63be696b776a7d764359cb990533871d910197d..0abf8a39cee14b615aa8a2b1e0445c5ee1ac7b66 100644 (file)
--- a/arch/x86/include/asm/sev-common.h
+++ b/arch/x86/include/asm/sev-common.h
@@ -70,6 +70,7 @@
        /* GHCBData[63:12] */                           \
        (((u64)(v) & GENMASK_ULL(63, 12)) >> 12)
 
+#ifndef __ASSEMBLY__
 /*
  * SNP Page State Change Operation
  *
@@ -161,6 +162,8 @@ struct snp_psc_desc {
 
 #define GHCB_RESP_CODE(v)              ((v) & GHCB_MSR_INFO_MASK)
 
+#endif /* __ASSEMBLY__ */
+
 /*
  * Error codes related to GHCB input that can be communicated back to the guest
  * by setting the lower 32-bits of the GHCB SW_EXITINFO1 field to 2.

diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 90b1555ed862e22e0246e1f6137b5b18d716832f..ed683b8cccb3df590490e073e8c9de9da78eea61 100644 (file)
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -201,6 +201,7 @@ extern unsigned int smpboot_control;
 #define STARTUP_APICID_CPUID_1F 0x80000000
 #define STARTUP_APICID_CPUID_0B 0x40000000
 #define STARTUP_APICID_CPUID_01 0x20000000
+#define STARTUP_APICID_SEV_ES  0x10000000
 
 /* Top 8 bits are reserved for control */
 #define STARTUP_PARALLEL_MASK  0xFF000000

diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 08ad5ef5acd43e69eb401164750a01995a2aa39a..a462588f0339a36940381613114f4c67761c9c5c 100644 (file)
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -26,6 +26,7 @@
 #include <asm/nospec-branch.h>
 #include <asm/fixmap.h>
 #include <asm/smp.h>
+#include <asm/sev-common.h>
 
 /*
  * We are not able to switch in one step to the final KERNEL ADDRESS SPACE
@@ -243,9 +244,14 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
         * Bit 31       STARTUP_APICID_CPUID_1F flag (use CPUID 0x1f)
         * Bit 30       STARTUP_APICID_CPUID_0B flag (use CPUID 0x0b)
         * Bit 29       STARTUP_APICID_CPUID_01 flag (use CPUID 0x01)
+        * Bit 28       STARTUP_APICID_SEV_ES flag (CPUID 0x0b via GHCB MSR)
         * Bit 0-24     CPU# if STARTUP_APICID_CPUID_xx flags are not set
         */
        movl    smpboot_control(%rip), %ecx
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+       testl   $STARTUP_APICID_SEV_ES, %ecx
+       jnz     .Luse_sev_cpuid
+#endif
        testl   $STARTUP_APICID_CPUID_1F, %ecx
        jnz     .Luse_cpuid_1f
        testl   $STARTUP_APICID_CPUID_0B, %ecx
@@ -262,6 +268,37 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
        shr     $24, %edx
        jmp     .Lsetup_AP
 
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+.Luse_sev_cpuid:
+       /* Set the GHCB MSR to request CPUID 0x[0B|1F]_EDX */
+       movl    $0x1f, %edx
+       testl   $STARTUP_APICID_CPUID_1F, %ecx
+       jnz     .Lsev_ghcb_msr
+       movl    $0x0b, %edx
+       testl   $STARTUP_APICID_CPUID_0B, %ecx
+       jz      1f
+
+.Lsev_ghcb_msr:
+       movl    $MSR_AMD64_SEV_ES_GHCB, %ecx
+       movl    $(GHCB_CPUID_REQ_EDX << 30) | GHCB_MSR_CPUID_REQ, %eax
+       wrmsr
+
+       /* Perform GHCB MSR protocol */
+       rep; vmmcall            /* vmgexit */
+
+       /*
+        * Get the result. After the RDMSR:
+        *   EAX should be 0xc0000005
+        *   EDX should have the CPUID register value and since EDX
+        *   is the target register, no need to move the result.
+        */
+       rdmsr
+       andl    $GHCB_MSR_INFO_MASK, %eax
+       cmpl    $GHCB_MSR_CPUID_RESP, %eax
+       jne     1f
+       jmp     .Lsetup_AP
+#endif
+
 .Luse_cpuid_0b:
        mov     $0x0B, %eax
        xorl    %ecx, %ecx

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 7955b86d4e9c7bca31bdc8732a0d3f0d1ac99bf4..81a2ed243d81483908720d1f34f67799c64f16fd 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -85,6 +85,7 @@
 #include <asm/hw_irq.h>
 #include <asm/stackprotector.h>
 #include <asm/sev.h>
+#include <asm/coco.h>
 
 /* representing HT siblings of each logical CPU */
 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
@@ -1237,15 +1238,21 @@ void __init smp_prepare_cpus_common(void)
  */
 bool __init arch_cpuhp_init_parallel_bringup(void)
 {
-       unsigned int ctrl = 0;
+       unsigned int ctrl = 0, cc_ctrl = 0;
 
        if (boot_cpu_data.cpuid_level < 0x01)
                return false;
 
        /* Encrypted guests require special CPUID handling. */
        if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) {
-               pr_info("Parallel CPU startup disabled due to guest state encryption\n");
-               return false;
+               switch (cc_get_vendor()) {
+               case CC_VENDOR_AMD:
+                       cc_ctrl = STARTUP_APICID_SEV_ES;
+                       break;
+               default:
+                       pr_info("Parallel CPU startup disabled due to guest state encryption\n");
+                       return false;
+               }
        }
 
        switch (topology_extended_leaf) {
@@ -1256,7 +1263,7 @@ bool __init arch_cpuhp_init_parallel_bringup(void)
                ctrl = STARTUP_APICID_CPUID_1F;
                break;
        case 0x00:
-               if (!x2apic_mode) {
+               if (!cc_ctrl && !x2apic_mode) {
                        /* For !x2APIC mode 8 bits from leaf 0x01 are sufficient */
                        ctrl = STARTUP_APICID_CPUID_01;
                        break;
@@ -1268,6 +1275,7 @@ bool __init arch_cpuhp_init_parallel_bringup(void)
                return false;
        }
 
+       ctrl |= cc_ctrl;
        pr_debug("Parallel CPU startup enabled: 0x%08x\n", ctrl);
        smpboot_control = ctrl;
        return true;