x86/sev: Move __sev_[get|put]_ghcb() into separate noinstr object

Rename sev-nmi.c to noinstr.c, and move the get/put GHCB routines into it too,
which are also annotated as 'noinstr' and suffer from the same problem as the
NMI code, i.e., that GCC may ignore the __no_sanitize_address__ function
attribute implied by 'noinstr' and insert KASAN instrumentation anyway.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/20250828102202.1849035-37-ardb+git@google.com

diff --git a/arch/x86/boot/startup/sev-startup.c b/arch/x86/boot/startup/sev-startup.c
index 138b26f14ff126c38900bf7a85aa433f30cdec92..9f4b4ca7deaac99b975914cb49c7eef85e5feb91 100644 (file)
--- a/arch/x86/boot/startup/sev-startup.c
+++ b/arch/x86/boot/startup/sev-startup.c
@@ -41,83 +41,9 @@
 #include <asm/cpuid/api.h>
 #include <asm/cmdline.h>
 
-/*
- * Nothing shall interrupt this code path while holding the per-CPU
- * GHCB. The backup GHCB is only for NMIs interrupting this path.
- *
- * Callers must disable local interrupts around it.
- */
-noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
-{
-       struct sev_es_runtime_data *data;
-       struct ghcb *ghcb;
-
-       WARN_ON(!irqs_disabled());
-
-       data = this_cpu_read(runtime_data);
-       ghcb = &data->ghcb_page;
-
-       if (unlikely(data->ghcb_active)) {
-               /* GHCB is already in use - save its contents */
-
-               if (unlikely(data->backup_ghcb_active)) {
-                       /*
-                        * Backup-GHCB is also already in use. There is no way
-                        * to continue here so just kill the machine. To make
-                        * panic() work, mark GHCBs inactive so that messages
-                        * can be printed out.
-                        */
-                       data->ghcb_active        = false;
-                       data->backup_ghcb_active = false;
-
-                       instrumentation_begin();
-                       panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");
-                       instrumentation_end();
-               }
-
-               /* Mark backup_ghcb active before writing to it */
-               data->backup_ghcb_active = true;
-
-               state->ghcb = &data->backup_ghcb;
-
-               /* Backup GHCB content */
-               *state->ghcb = *ghcb;
-       } else {
-               state->ghcb = NULL;
-               data->ghcb_active = true;
-       }
-
-       return ghcb;
-}
-
 /* Include code shared with pre-decompression boot stage */
 #include "sev-shared.c"
 
-noinstr void __sev_put_ghcb(struct ghcb_state *state)
-{
-       struct sev_es_runtime_data *data;
-       struct ghcb *ghcb;
-
-       WARN_ON(!irqs_disabled());
-
-       data = this_cpu_read(runtime_data);
-       ghcb = &data->ghcb_page;
-
-       if (state->ghcb) {
-               /* Restore GHCB from Backup */
-               *ghcb = *state->ghcb;
-               data->backup_ghcb_active = false;
-               state->ghcb = NULL;
-       } else {
-               /*
-                * Invalidate the GHCB so a VMGEXIT instruction issued
-                * from userspace won't appear to be valid.
-                */
-               vc_ghcb_invalidate(ghcb);
-               data->ghcb_active = false;
-       }
-}
-
 void __head
 early_set_pages_state(unsigned long vaddr, unsigned long paddr,
                      unsigned long npages, const struct psc_desc *desc)

diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile
index 342d79f0ab6a8a4df268c04ca1f39ae25c9ecc00..3b8ae214a6a64de6bb208eb3b7c8bf12007ccc2c 100644 (file)
--- a/arch/x86/coco/sev/Makefile
+++ b/arch/x86/coco/sev/Makefile
@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-y += core.o sev-nmi.o vc-handle.o
+obj-y += core.o noinstr.o vc-handle.o
 
 # Clang 14 and older may fail to respect __no_sanitize_undefined when inlining
-UBSAN_SANITIZE_sev-nmi.o       := n
+UBSAN_SANITIZE_noinstr.o       := n
 
 # GCC may fail to respect __no_sanitize_address or __no_kcsan when inlining
-KASAN_SANITIZE_sev-nmi.o       := n
-KCSAN_SANITIZE_sev-nmi.o       := n
+KASAN_SANITIZE_noinstr.o       := n
+KCSAN_SANITIZE_noinstr.o       := n

diff --git a/arch/x86/coco/sev/sev-nmi.c b/arch/x86/coco/sev/noinstr.c
similarity index 61%
rename from arch/x86/coco/sev/sev-nmi.c
rename to arch/x86/coco/sev/noinstr.c
index d8dfaddfb3671eb97a9c75c2ff6f4afd5bba88c0..b527eafb631235d7000e68f7ce0821b15ad767a9 100644 (file)
--- a/arch/x86/coco/sev/sev-nmi.c
+++ b/arch/x86/coco/sev/noinstr.c
@@ -106,3 +106,77 @@ void noinstr __sev_es_nmi_complete(void)
 
        __sev_put_ghcb(&state);
 }
+
+/*
+ * Nothing shall interrupt this code path while holding the per-CPU
+ * GHCB. The backup GHCB is only for NMIs interrupting this path.
+ *
+ * Callers must disable local interrupts around it.
+ */
+noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
+{
+       struct sev_es_runtime_data *data;
+       struct ghcb *ghcb;
+
+       WARN_ON(!irqs_disabled());
+
+       data = this_cpu_read(runtime_data);
+       ghcb = &data->ghcb_page;
+
+       if (unlikely(data->ghcb_active)) {
+               /* GHCB is already in use - save its contents */
+
+               if (unlikely(data->backup_ghcb_active)) {
+                       /*
+                        * Backup-GHCB is also already in use. There is no way
+                        * to continue here so just kill the machine. To make
+                        * panic() work, mark GHCBs inactive so that messages
+                        * can be printed out.
+                        */
+                       data->ghcb_active        = false;
+                       data->backup_ghcb_active = false;
+
+                       instrumentation_begin();
+                       panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");
+                       instrumentation_end();
+               }
+
+               /* Mark backup_ghcb active before writing to it */
+               data->backup_ghcb_active = true;
+
+               state->ghcb = &data->backup_ghcb;
+
+               /* Backup GHCB content */
+               *state->ghcb = *ghcb;
+       } else {
+               state->ghcb = NULL;
+               data->ghcb_active = true;
+       }
+
+       return ghcb;
+}
+
+noinstr void __sev_put_ghcb(struct ghcb_state *state)
+{
+       struct sev_es_runtime_data *data;
+       struct ghcb *ghcb;
+
+       WARN_ON(!irqs_disabled());
+
+       data = this_cpu_read(runtime_data);
+       ghcb = &data->ghcb_page;
+
+       if (state->ghcb) {
+               /* Restore GHCB from Backup */
+               *ghcb = *state->ghcb;
+               data->backup_ghcb_active = false;
+               state->ghcb = NULL;
+       } else {
+               /*
+                * Invalidate the GHCB so a VMGEXIT instruction issued
+                * from userspace won't appear to be valid.
+                */
+               vc_ghcb_invalidate(ghcb);
+               data->ghcb_active = false;
+       }
+}