x86/speculation/mds: Clear CPU buffers on exit to user

commit 04dcbdb8057827b043b3c71aa397c4c63e67d086 upstream

Add a static key which controls the invocation of the CPU buffer clear
mechanism on exit to user space and add the call into
prepare_exit_to_usermode() and do_nmi() right before actually returning.

Add documentation which kernel to user space transition this covers and
explain why some corner cases are not mitigated.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Reviewed-by: Jon Masters <jcm@redhat.com>
Tested-by: Jon Masters <jcm@redhat.com>
(cherry picked from commit 62ba379c5925f285e3ab9362761c18823e5a049e)

Orabug: 29526900
CVE: CVE-2018-12126
CVE: CVE-2018-12130
CVE: CVE-2018-12127

Signed-off-by: Kanth Ghatraju <kanth.ghatraju@oracle.com>
Reviewed-by: Mihai Carabas <mihai.carabas@oracle.com>
Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Conflicts:
	UEK4 uses bugs_64.c instead of bugs.c arch/x86/entry/common.c doesn't exist.
	Make the corresponding changes to arch/x86/kernel/entry_64.S.

diff --git a/Documentation/x86/mds.rst b/Documentation/x86/mds.rst
index 1096738d50f2a3dc340b567b59949575cbb0ff8c..54d935bf283ba9bae6f279bb1d427f4a366aa16d 100644 (file)
--- a/Documentation/x86/mds.rst
+++ b/Documentation/x86/mds.rst
@@ -97,3 +97,55 @@ According to current knowledge additional mitigations inside the kernel
 itself are not required because the necessary gadgets to expose the leaked
 data cannot be controlled in a way which allows exploitation from malicious
 user space or VM guests.
+
+Mitigation points
+-----------------
+
+1. Return to user space
+^^^^^^^^^^^^^^^^^^^^^^^
+
+   When transitioning from kernel to user space the CPU buffers are flushed
+   on affected CPUs when the mitigation is not disabled on the kernel
+   command line. The migitation is enabled through the static key
+   mds_user_clear.
+
+   The mitigation is invoked in prepare_exit_to_usermode() which covers
+   most of the kernel to user space transitions. There are a few exceptions
+   which are not invoking prepare_exit_to_usermode() on return to user
+   space. These exceptions use the paranoid exit code.
+
+   - Non Maskable Interrupt (NMI):
+
+     Access to sensible data like keys, credentials in the NMI context is
+     mostly theoretical: The CPU can do prefetching or execute a
+     misspeculated code path and thereby fetching data which might end up
+     leaking through a buffer.
+
+     But for mounting other attacks the kernel stack address of the task is
+     already valuable information. So in full mitigation mode, the NMI is
+     mitigated on the return from do_nmi() to provide almost complete
+     coverage.
+
+   - Double fault (#DF):
+
+     A double fault is usually fatal, but the ESPFIX workaround, which can
+     be triggered from user space through modify_ldt(2) is a recoverable
+     double fault. #DF uses the paranoid exit path, so explicit mitigation
+     in the double fault handler is required.
+
+   - Machine Check Exception (#MC):
+
+     Another corner case is a #MC which hits between the CPU buffer clear
+     invocation and the actual return to user. As this still is in kernel
+     space it takes the paranoid exit path which does not clear the CPU
+     buffers. So the #MC handler repopulates the buffers to some
+     extent. Machine checks are not reliably controllable and the window is
+     extremly small so mitigation would just tick a checkbox that this
+     theoretical corner case is covered. To keep the amount of special
+     cases small, ignore #MC.
+
+   - Debug Exception (#DB):
+
+     This takes the paranoid exit path only when the INT1 breakpoint is in
+     kernel space. #DB on a user space address takes the regular exit path,
+     so no extra mitigation required.

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 2ab5ef581ee44b7d4cfd3feefbeb70022df59c65..9e9fe1e463cd1617bec8f31d8b05e6879dba93ca 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -137,11 +137,17 @@
  * combination with microcode which triggers a CPU buffer flush when the
  * instruction is executed.
  */
+.extern mds_user_clear
+
 .macro MDS_CLEAR_CPU_BUFFERS
+       STATIC_JUMP_IF_TRUE     .Lmdsverwcall_\@, mds_user_clear, def=0
+       jmp     .Lmdsverwdone_\@
+.Lmdsverwcall_\@:
        pushw   %cx
        movw    $__KERNEL_DS, %cx
        verw    %cx
        popw    %cx
+.Lmdsverwdone_\@:
 .endm
 #else /* __ASSEMBLY__ */
 
@@ -256,7 +262,9 @@ static inline void vmexit_fill_RSB(void)
                      : "=r" (loops), ASM_CALL_CONSTRAINT
                      : : "memory" );
 #endif
-}
+} 
+
+DECLARE_STATIC_KEY_FALSE(mds_user_clear);
 
 #include <asm/segment.h>
 
@@ -283,5 +291,16 @@ static inline void mds_clear_cpu_buffers(void)
        asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc");
 }
 
+/**
+ * mds_user_clear_cpu_buffers - Mitigation for MDS vulnerability
+ *
+ * Clear CPU buffers if the corresponding static key is enabled
+ */
+static inline void mds_user_clear_cpu_buffers(void)
+{
+       if (static_branch_likely(&mds_user_clear))
+               mds_clear_cpu_buffers();
+}
+
 #endif /* __ASSEMBLY__ */
 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */

diff --git a/arch/x86/kernel/cpu/bugs_64.c b/arch/x86/kernel/cpu/bugs_64.c
index 13579c0714480ea71074fc2ff256c9c9f1cee502..49900fa752f898fc04fcd9a30b814404cb3c3fac 100644 (file)
--- a/arch/x86/kernel/cpu/bugs_64.c
+++ b/arch/x86/kernel/cpu/bugs_64.c
@@ -169,6 +169,9 @@ EXPORT_PER_CPU_SYMBOL(x86_spec_ctrl_restore);
 u64 x86_amd_ls_cfg_base;
 u64 x86_amd_ls_cfg_ssbd_mask;
 
+/* Control MDS CPU buffer clear before returning to user space */
+DEFINE_STATIC_KEY_FALSE(mds_user_clear);
+
 void __init check_bugs(void)
 {
        identify_boot_cpu();

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 92c87adea4dd519a3be59036ac245083d6352c9e..ebd77c177107dd9950729cceb28b74ee600e0023 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -535,6 +535,7 @@ opportunistic_sysret_failed:
         * switch CR3 in NMIs.  Normal interrupts are OK because
         * they are off here.
         */
+       MDS_CLEAR_CPU_BUFFERS
        SWITCH_USER_CR3
        SWAPGS
        jmp     restore_c_regs_and_iret

diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index c1783416b09cc78a38dd9c3e02d7f9f891a4a346..c3e5fd5288dc0cfb9926be3433b263bcbc5062b4 100644 (file)
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -30,6 +30,7 @@
 #include <asm/nmi.h>
 #include <asm/x86_init.h>
 #include <asm/reboot.h>
+#include <asm/nospec-branch.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/nmi.h>
@@ -540,6 +541,10 @@ nmi_restart:
                write_cr2(this_cpu_read(nmi_cr2));
        if (this_cpu_dec_return(nmi_state))
                goto nmi_restart;
+
+       if (user_mode(regs))
+               mds_user_clear_cpu_buffers();
+
        return 0;
 }
 NOKPROBE_SYMBOL(do_nmi);

diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 4ee9eb65276c1b411450a63526cb0ab7d3747171..3c034deb1601059b211db752c31e8ed367d4b54f 100644 (file)
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -61,6 +61,7 @@
 #include <asm/mach_traps.h>
 #include <asm/alternative.h>
 #include <asm/mpx.h>
+#include <asm/nospec-branch.h>
 
 #ifdef CONFIG_X86_64
 #include <asm/x86_init.h>
@@ -349,6 +350,13 @@ dotraplinkage int do_double_fault(struct pt_regs *regs, long error_code)
                regs->ip = (unsigned long)general_protection;
                regs->sp = (unsigned long)&normal_regs->orig_ax;
 
+               /*
+                * This situation can be triggered by userspace via
+                * modify_ldt(2) and the return does not take the regular
+                * user space exit, so a CPU buffer clear is required when
+                * MDS mitigation is enabled.
+                */
+               mds_user_clear_cpu_buffers();
                return 0;
        }
 #endif