x86/speculation/mds: Improve coverage for MDS vulnerability

We seem to be missing a bunch of cases when we don't clear fill/store
buffers for MDS vulnerability during return to userspace.

Since we always call DISABLE_IBRS in those cases let's define a new
macro SPEC_RETURN_TO_USER than will both disable IBRS and flush the
buffers.

Orabug: 29526900
CVE: CVE-2018-12126
CVE: CVE-2018-12130
CVE: CVE-2018-12127

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Reviewed-by: Mihai Carabas <mihai.carabas@oracle.com>

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index bf4cd7336c6db685cad0d3f8a70d4c09fc647380..ac0f7f5ff91f60625d4cce2af3c9350dc1d11bba 100644 (file)
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -18,6 +18,7 @@
 #include <asm/spec_ctrl.h>
 #include <asm/pgtable_types.h>
 #include <asm/kaiser.h>
+#include <asm/nospec-branch.h>
 #include <linux/linkage.h>
 #include <linux/err.h>
 
@@ -220,7 +221,7 @@ sysexit_from_sys_call:
        /*CFI_RESTORE rflags*/
        TRACE_IRQS_ON
 
-       DISABLE_IBRS
+       SPEC_RETURN_TO_USER
 
        SWITCH_USER_CR3
 
@@ -446,7 +447,7 @@ sysretl_from_sys_call:
        xorq    %r9,%r9
        xorq    %r8,%r8
        TRACE_IRQS_ON
-       DISABLE_IBRS
+       SPEC_RETURN_TO_USER
        SWITCH_USER_CR3
        movl RSP(%rsp),%esp
        CFI_RESTORE rsp

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 6e1b4325243e557eca6cdfbf7a003007622582b9..6e07a1b255da127e7d7de15111fe31751a44dca9 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -149,6 +149,12 @@
        popw    %cx
 .Lmdsverwdone_\@:
 .endm
+
+.macro SPEC_RETURN_TO_USER
+       DISABLE_IBRS
+       MDS_CLEAR_CPU_BUFFERS
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #ifdef CONFIG_RETPOLINE

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index ebd77c177107dd9950729cceb28b74ee600e0023..65c8270dd9120038d992e3d60f8795cf91c11a4a 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -312,7 +312,7 @@ system_call_fastpath:
        CFI_REGISTER    rip,rcx
        movq    EFLAGS(%rsp),%r11
        /*CFI_REGISTER  rflags,r11*/
-       DISABLE_IBRS
+       SPEC_RETURN_TO_USER
        RESTORE_C_REGS_EXCEPT_RCX_R11
        /*
         * This opens a window where we have a user CR3, but are
@@ -510,7 +510,7 @@ syscall_return:
         * perf profiles.  Nothing jumps here.
         */
 syscall_return_via_sysret:
-       DISABLE_IBRS
+       SPEC_RETURN_TO_USER
        CFI_REMEMBER_STATE
        /* r11 is already restored (see code above) */
        RESTORE_C_REGS_EXCEPT_R11
@@ -527,7 +527,7 @@ syscall_return_via_sysret:
        CFI_RESTORE_STATE
 
 opportunistic_sysret_failed:
-       DISABLE_IBRS
+       SPEC_RETURN_TO_USER
        /*
         * This opens a window where we have a user CR3, but are
         * running in the kernel.  This makes using the CS
@@ -535,7 +535,6 @@ opportunistic_sysret_failed:
         * switch CR3 in NMIs.  Normal interrupts are OK because
         * they are off here.
         */
-       MDS_CLEAR_CPU_BUFFERS
        SWITCH_USER_CR3
        SWAPGS
        jmp     restore_c_regs_and_iret
@@ -819,7 +818,7 @@ retint_swapgs:              /* return to user-space */
        DISABLE_INTERRUPTS(CLBR_ANY)
        TRACE_IRQS_IRETQ
 
-       DISABLE_IBRS
+       SPEC_RETURN_TO_USER
        SWITCH_USER_CR3
        SWAPGS
        jmp     restore_c_regs_and_iret
@@ -1764,7 +1763,7 @@ ENTRY(nmi)
        STUFF_RSB
        ENABLE_IBRS
        call    do_nmi
-       DISABLE_IBRS
+       SPEC_RETURN_TO_USER
 #ifdef CONFIG_PAGE_TABLE_ISOLATION
        /*
         * Unconditionally restore CR3.  I know we return to

diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index c3e5fd5288dc0cfb9926be3433b263bcbc5062b4..e0885befd3ca2a3c02bdc9b864c8187cafe64271 100644 (file)
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -542,9 +542,6 @@ nmi_restart:
        if (this_cpu_dec_return(nmi_state))
                goto nmi_restart;
 
-       if (user_mode(regs))
-               mds_user_clear_cpu_buffers();
-
        return 0;
 }
 NOKPROBE_SYMBOL(do_nmi);