x86/sev: Move noinstr NMI handling code into separate source file

GCC may ignore the __no_sanitize_address function attribute when
inlining, resulting in KASAN instrumentation in code tagged as
'noinstr'.

Move the SEV NMI handling code, which is noinstr, into a separate source
file so KASAN can be disabled on the whole file without losing coverage
of other SEV core code, once the startup code is split off from it too.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: David Woodhouse <dwmw@amazon.co.uk>
Cc: Dionna Amalie Glaze <dionnaglaze@google.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Kevin Loughlin <kevinloughlin@google.com>
Cc: Len Brown <len.brown@intel.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
Link: https://lore.kernel.org/r/20250418141253.2601348-10-ardb+git@google.com

diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile
index dcb06dc8b5aeaeb6878f33a37da0dc97fdb2a2f8..bc4baa400c55b0522c001bae833b628a55a3b390 100644 (file)
--- a/arch/x86/coco/sev/Makefile
+++ b/arch/x86/coco/sev/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-y += core.o
+obj-y += core.o sev-nmi.o
 
 # jump tables are emitted using absolute references in non-PIC code
 # so they cannot be used in the early SEV startup code
@@ -20,3 +20,6 @@ KCSAN_SANITIZE                := n
 
 # Clang 14 and older may fail to respect __no_sanitize_undefined when inlining
 UBSAN_SANITIZE         := n
+
+# GCC may fail to respect __no_sanitize_address when inlining
+KASAN_SANITIZE_sev-nmi.o       := n

diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index aeb7731862c0dc709e45ba1483d5f70491b8669c..c7a0f3a2f4ac93146b57640cfdbd491a3ff2e3f9 100644 (file)
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -113,77 +113,6 @@ DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
 DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
 DEFINE_PER_CPU(u64, svsm_caa_pa);
 
-static __always_inline bool on_vc_stack(struct pt_regs *regs)
-{
-       unsigned long sp = regs->sp;
-
-       /* User-mode RSP is not trusted */
-       if (user_mode(regs))
-               return false;
-
-       /* SYSCALL gap still has user-mode RSP */
-       if (ip_within_syscall_gap(regs))
-               return false;
-
-       return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC)));
-}
-
-/*
- * This function handles the case when an NMI is raised in the #VC
- * exception handler entry code, before the #VC handler has switched off
- * its IST stack. In this case, the IST entry for #VC must be adjusted,
- * so that any nested #VC exception will not overwrite the stack
- * contents of the interrupted #VC handler.
- *
- * The IST entry is adjusted unconditionally so that it can be also be
- * unconditionally adjusted back in __sev_es_ist_exit(). Otherwise a
- * nested sev_es_ist_exit() call may adjust back the IST entry too
- * early.
- *
- * The __sev_es_ist_enter() and __sev_es_ist_exit() functions always run
- * on the NMI IST stack, as they are only called from NMI handling code
- * right now.
- */
-void noinstr __sev_es_ist_enter(struct pt_regs *regs)
-{
-       unsigned long old_ist, new_ist;
-
-       /* Read old IST entry */
-       new_ist = old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
-
-       /*
-        * If NMI happened while on the #VC IST stack, set the new IST
-        * value below regs->sp, so that the interrupted stack frame is
-        * not overwritten by subsequent #VC exceptions.
-        */
-       if (on_vc_stack(regs))
-               new_ist = regs->sp;
-
-       /*
-        * Reserve additional 8 bytes and store old IST value so this
-        * adjustment can be unrolled in __sev_es_ist_exit().
-        */
-       new_ist -= sizeof(old_ist);
-       *(unsigned long *)new_ist = old_ist;
-
-       /* Set new IST entry */
-       this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist);
-}
-
-void noinstr __sev_es_ist_exit(void)
-{
-       unsigned long ist;
-
-       /* Read IST entry */
-       ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
-
-       if (WARN_ON(ist == __this_cpu_ist_top_va(VC)))
-               return;
-
-       /* Read back old IST entry and write it to the TSS */
-       this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist);
-}
-
 /*
  * Nothing shall interrupt this code path while holding the per-CPU
  * GHCB. The backup GHCB is only for NMIs interrupting this path.
@@ -608,24 +537,6 @@ int svsm_perform_call_protocol(struct svsm_call *call)
        return ret;
 }
 
-void noinstr __sev_es_nmi_complete(void)
-{
-       struct ghcb_state state;
-       struct ghcb *ghcb;
-
-       ghcb = __sev_get_ghcb(&state);
-
-       vc_ghcb_invalidate(ghcb);
-       ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
-       ghcb_set_sw_exit_info_1(ghcb, 0);
-       ghcb_set_sw_exit_info_2(ghcb, 0);
-
-       sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));
-       VMGEXIT();
-
-       __sev_put_ghcb(&state);
-}
-
 static u64 __init get_snp_jump_table_addr(void)
 {
        struct snp_secrets_page *secrets;

diff --git a/arch/x86/coco/sev/sev-nmi.c b/arch/x86/coco/sev/sev-nmi.c
new file mode 100644 (file)
index 0000000..d8dfadd
--- /dev/null
+++ b/arch/x86/coco/sev/sev-nmi.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AMD Memory Encryption Support
+ *
+ * Copyright (C) 2019 SUSE
+ *
+ * Author: Joerg Roedel <jroedel@suse.de>
+ */
+
+#define pr_fmt(fmt)    "SEV: " fmt
+
+#include <linux/bug.h>
+#include <linux/kernel.h>
+
+#include <asm/cpu_entry_area.h>
+#include <asm/msr.h>
+#include <asm/ptrace.h>
+#include <asm/sev.h>
+#include <asm/sev-internal.h>
+
+static __always_inline bool on_vc_stack(struct pt_regs *regs)
+{
+       unsigned long sp = regs->sp;
+
+       /* User-mode RSP is not trusted */
+       if (user_mode(regs))
+               return false;
+
+       /* SYSCALL gap still has user-mode RSP */
+       if (ip_within_syscall_gap(regs))
+               return false;
+
+       return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC)));
+}
+
+/*
+ * This function handles the case when an NMI is raised in the #VC
+ * exception handler entry code, before the #VC handler has switched off
+ * its IST stack. In this case, the IST entry for #VC must be adjusted,
+ * so that any nested #VC exception will not overwrite the stack
+ * contents of the interrupted #VC handler.
+ *
+ * The IST entry is adjusted unconditionally so that it can be also be
+ * unconditionally adjusted back in __sev_es_ist_exit(). Otherwise a
+ * nested sev_es_ist_exit() call may adjust back the IST entry too
+ * early.
+ *
+ * The __sev_es_ist_enter() and __sev_es_ist_exit() functions always run
+ * on the NMI IST stack, as they are only called from NMI handling code
+ * right now.
+ */
+void noinstr __sev_es_ist_enter(struct pt_regs *regs)
+{
+       unsigned long old_ist, new_ist;
+
+       /* Read old IST entry */
+       new_ist = old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
+
+       /*
+        * If NMI happened while on the #VC IST stack, set the new IST
+        * value below regs->sp, so that the interrupted stack frame is
+        * not overwritten by subsequent #VC exceptions.
+        */
+       if (on_vc_stack(regs))
+               new_ist = regs->sp;
+
+       /*
+        * Reserve additional 8 bytes and store old IST value so this
+        * adjustment can be unrolled in __sev_es_ist_exit().
+        */
+       new_ist -= sizeof(old_ist);
+       *(unsigned long *)new_ist = old_ist;
+
+       /* Set new IST entry */
+       this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist);
+}
+
+void noinstr __sev_es_ist_exit(void)
+{
+       unsigned long ist;
+
+       /* Read IST entry */
+       ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
+
+       if (WARN_ON(ist == __this_cpu_ist_top_va(VC)))
+               return;
+
+       /* Read back old IST entry and write it to the TSS */
+       this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist);
+}
+
+void noinstr __sev_es_nmi_complete(void)
+{
+       struct ghcb_state state;
+       struct ghcb *ghcb;
+
+       ghcb = __sev_get_ghcb(&state);
+
+       vc_ghcb_invalidate(ghcb);
+       ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
+       ghcb_set_sw_exit_info_1(ghcb, 0);
+       ghcb_set_sw_exit_info_2(ghcb, 0);
+
+       sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));
+       VMGEXIT();
+
+       __sev_put_ghcb(&state);
+}