x86/smpboot: Allow parallel bringup for SEV-ES

Enable parallel bringup for SEV-ES guests. The APs can't actually
execute the CPUID instruction directly during early startup, but they
can make the GHCB call directly instead, just as the VC trap handler
would do.

Thanks to Sabin for talking me through the way this works.

Suggested-by: Sabin Rapan <sabrapan@amazon.com>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Usama Arif <usama.arif@bytedance.com>
Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>

diff --git a/arch/x86/coco/core.c b/arch/x86/coco/core.c
index 49b44f881484680376a7393dc0f101e4565a4009..0bab38efb15a99f1809a13524a9dbfb0aaaa0e25 100644 (file)
--- a/arch/x86/coco/core.c
+++ b/arch/x86/coco/core.c
@@ -129,6 +129,11 @@ u64 cc_mkdec(u64 val)
 }
 EXPORT_SYMBOL_GPL(cc_mkdec);
 
+enum cc_vendor cc_get_vendor(void)
+{
+       return vendor;
+}
+
 __init void cc_set_vendor(enum cc_vendor v)
 {
        vendor = v;

diff --git a/arch/x86/include/asm/coco.h b/arch/x86/include/asm/coco.h
index 3d98c3a60d34f97fc6403a7248b9c06c03b63f8d..0428d9712c9686eff770fce3efc152ee5f512521 100644 (file)
--- a/arch/x86/include/asm/coco.h
+++ b/arch/x86/include/asm/coco.h
@@ -12,6 +12,7 @@ enum cc_vendor {
 };
 
 void cc_set_vendor(enum cc_vendor v);
+enum cc_vendor cc_get_vendor(void);
 void cc_set_mask(u64 mask);
 
 #ifdef CONFIG_ARCH_HAS_CC_PLATFORM

diff --git a/arch/x86/include/asm/sev-common.h b/arch/x86/include/asm/sev-common.h
index b63be696b776a7d764359cb990533871d910197d..0abf8a39cee14b615aa8a2b1e0445c5ee1ac7b66 100644 (file)
--- a/arch/x86/include/asm/sev-common.h
+++ b/arch/x86/include/asm/sev-common.h
@@ -70,6 +70,7 @@
        /* GHCBData[63:12] */                           \
        (((u64)(v) & GENMASK_ULL(63, 12)) >> 12)
 
+#ifndef __ASSEMBLY__
 /*
  * SNP Page State Change Operation
  *
@@ -161,6 +162,8 @@ struct snp_psc_desc {
 
 #define GHCB_RESP_CODE(v)              ((v) & GHCB_MSR_INFO_MASK)
 
+#endif /* __ASSEMBLY__ */
+
 /*
  * Error codes related to GHCB input that can be communicated back to the guest
  * by setting the lower 32-bits of the GHCB SW_EXITINFO1 field to 2.

diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index defe76ee9e6456a92c5605b85e58d2c1f1140de0..1584f04a7007f6dccf8b230b45478e9b4b9310e2 100644 (file)
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -204,7 +204,10 @@ extern unsigned int smpboot_control;
 /* Control bits for startup_64 */
 #define STARTUP_APICID_CPUID_0B        0x80000000
 #define STARTUP_APICID_CPUID_01        0x40000000
+#define STARTUP_APICID_SEV_ES  0x20000000
 
-#define STARTUP_PARALLEL_MASK (STARTUP_APICID_CPUID_01 | STARTUP_APICID_CPUID_0B)
+#define STARTUP_PARALLEL_MASK (STARTUP_APICID_CPUID_01 | \
+                              STARTUP_APICID_CPUID_0B | \
+                              STARTUP_APICID_SEV_ES)
 
 #endif /* _ASM_X86_SMP_H */

diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index ff3a5f008d8a15fe489bb096b644d6073e0fa0a1..9c38849fcac800e5a8774cdc62182d5235287a12 100644 (file)
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -26,6 +26,7 @@
 #include <asm/nospec-branch.h>
 #include <asm/fixmap.h>
 #include <asm/smp.h>
+#include <asm/sev-common.h>
 
 /*
  * We are not able to switch in one step to the final KERNEL ADDRESS SPACE
@@ -242,6 +243,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
         *
         * Bit 31       STARTUP_APICID_CPUID_0B flag (use CPUID 0x0b)
         * Bit 30       STARTUP_APICID_CPUID_01 flag (use CPUID 0x01)
+        * Bit 29       STARTUP_APICID_SEV_ES flag (CPUID 0x0b via GHCB MSR)
         * Bit 0-24     CPU# if STARTUP_APICID_CPUID_xx flags are not set
         */
        movl    smpboot_control(%rip), %ecx
@@ -249,6 +251,10 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
        jnz     .Luse_cpuid_0b
        testl   $STARTUP_APICID_CPUID_01, %ecx
        jnz     .Luse_cpuid_01
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+       testl   $STARTUP_APICID_SEV_ES, %ecx
+       jnz     .Luse_sev_cpuid_0b
+#endif
        andl    $0x0FFFFFFF, %ecx
        jmp     .Lsetup_cpu
 
@@ -259,6 +265,30 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
        shr     $24, %edx
        jmp     .Lsetup_AP
 
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+.Luse_sev_cpuid_0b:
+       /* Set the GHCB MSR to request CPUID 0xB_EDX */
+       movl    $MSR_AMD64_SEV_ES_GHCB, %ecx
+       movl    $(GHCB_CPUID_REQ_EDX << 30) | GHCB_MSR_CPUID_REQ, %eax
+       movl    $0x0B, %edx
+       wrmsr
+
+       /* Perform GHCB MSR protocol */
+       rep; vmmcall            /* vmgexit */
+
+       /*
+        * Get the result. After the RDMSR:
+        *   EAX should be 0xc0000005
+        *   EDX should have the CPUID register value and since EDX
+        *   is the target register, no need to move the result.
+        */
+       rdmsr
+       andl    $GHCB_MSR_INFO_MASK, %eax
+       cmpl    $GHCB_MSR_CPUID_RESP, %eax
+       jne     1f
+       jmp     .Lsetup_AP
+#endif
+
 .Luse_cpuid_0b:
        mov     $0x0B, %eax
        xorl    %ecx, %ecx

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 05d202b7dcaa3db5841c962a2ca04bb54c2eba25..6ef040fd28a04edcc98a528d452b305bec7c34e1 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -85,6 +85,7 @@
 #include <asm/hw_irq.h>
 #include <asm/stackprotector.h>
 #include <asm/sev.h>
+#include <asm/coco.h>
 
 /* representing HT siblings of each logical CPU */
 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
@@ -1514,15 +1515,36 @@ void __init smp_prepare_cpus_common(void)
  * We can do 64-bit AP bringup in parallel if the CPU reports its APIC
  * ID in CPUID (either leaf 0x0B if we need the full APIC ID in X2APIC
  * mode, or leaf 0x01 if 8 bits are sufficient). Otherwise it's too
- * hard. And not for SEV-ES guests because they can't use CPUID that
- * early.
+ * hard.
  */
 static bool prepare_parallel_bringup(void)
 {
-       if (IS_ENABLED(CONFIG_X86_32) || cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
+       bool has_sev_es = false;
+
+       if (IS_ENABLED(CONFIG_X86_32))
                return false;
 
-       if (x2apic_mode) {
+       /*
+        * Encrypted guests other than SEV-ES (in the future) will need to
+        * implement an early way of finding the APIC ID, since they will
+        * presumably block direct CPUID too. Be kind to our future selves
+        * by warning here instead of just letting them break. Parallel
+        * startup doesn't have to be in the first round of enabling patches
+        * for any such technology.
+        */
+       if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) {
+               switch (cc_get_vendor()) {
+               case CC_VENDOR_AMD:
+                       has_sev_es = true;
+                       break;
+
+               default:
+                       pr_info("Disabling parallel bringup due to guest state encryption\n");
+                       return false;
+               }
+       }
+
+       if (x2apic_mode || has_sev_es) {
                if (boot_cpu_data.cpuid_level < 0x0b)
                        return false;
 
@@ -1531,8 +1553,13 @@ static bool prepare_parallel_bringup(void)
                        return false;
                }
 
-               pr_debug("Using CPUID 0xb for parallel CPU startup\n");
-               smpboot_control = STARTUP_APICID_CPUID_0B;
+               if (has_sev_es) {
+                       pr_debug("Using SEV-ES CPUID 0xb for parallel CPU startup\n");
+                       smpboot_control = STARTUP_APICID_SEV_ES;
+               } else {
+                       pr_debug("Using CPUID 0xb for parallel CPU startup\n");
+                       smpboot_control = STARTUP_APICID_CPUID_0B;
+               }
        } else {
                /* Without X2APIC, what's in CPUID 0x01 should suffice. */
                if (boot_cpu_data.cpuid_level < 0x01)