x86/spec: STUFF_RSB _before_ ENABLE_IBRS

And also we need to STUFF_RSB _before_ calls.

In our case we have a bunch of ENABLE_INTERRUPTS
which are (in objdump):
       callq  *0x40b379(%rip)         <pv_cpu_ops+0x128>

During bootup they do change to 'cld' (on baremetal).

On Xen PV they end up being those calls and STUFF_RSB is still
in effect which means it should be done before those calls are made.

Also the semantics of the IBRS MSR is "If IBRS is set, .. indirect
calls will not allow their predicated target address to be controlled ...
so long as as all RSB entries from previous less privileged prediction
mode are overwritten."

In other words - STUFF_RSB, then ENABLE_IBRS.

Xen hypervisor code follows that religiously and so shall we.

OraBug: 27448169
Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Reviewed-by: Ankur Arora <ankur.a.arora@oracle.com>
Reviewed-by: Kris Van Hees <kris.van.hees@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 5aead52c7d4ecd5feebc5553b68fca7363522761..39f4debdf5bbe42be1610bf14a40c74f5b0f4be0 100644 (file)
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -125,8 +125,8 @@ ENTRY(ia32_sysenter_target)
        SWITCH_KERNEL_CR3_NO_STACK
        movq    PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 
-       ENABLE_IBRS
        STUFF_RSB
+       ENABLE_IBRS
 
        ENABLE_INTERRUPTS(CLBR_NONE)
 
@@ -373,8 +373,8 @@ ENTRY(ia32_cstar_target)
        CFI_REGISTER    rsp,r8
        movq    PER_CPU_VAR(cpu_current_top_of_stack),%rsp
 
-       ENABLE_IBRS
        STUFF_RSB
+       ENABLE_IBRS
 
        ENABLE_INTERRUPTS(CLBR_NONE)
 
@@ -537,6 +537,10 @@ ENTRY(ia32_syscall)
        ASM_CLAC                        /* Do this early to minimize exposure */
        SWAPGS
        SWITCH_KERNEL_CR3_NO_STACK
+
+       STUFF_RSB
+       ENABLE_IBRS
+
        ENABLE_INTERRUPTS(CLBR_NONE)
 
        /* Zero-extending 32-bit regs, do not remove */
@@ -555,8 +559,6 @@ ENTRY(ia32_syscall)
 
        SAVE_EXTRA_REGS
        CLEAR_R8_TO_R15
-       ENABLE_IBRS
-       STUFF_RSB
 
        orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
        testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 133e44326138e8cf6a61e45845e009405ad330d1..993cba9ca08b3c9fd3f32136296dbc67503f5443 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -222,6 +222,7 @@ GLOBAL(system_call_after_swapgs)
        movq    %rsp,PER_CPU_VAR(rsp_scratch)
        movq    PER_CPU_VAR(cpu_current_top_of_stack),%rsp
 
+       STUFF_RSB
        ENABLE_IBRS
 
        TRACE_IRQS_OFF
@@ -261,8 +262,6 @@ GLOBAL(system_call_after_swapgs)
        SAVE_EXTRA_REGS
        ZERO_EXTRA_REGS
 
-       STUFF_RSB
-
        testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
        jnz tracesys
 system_call_fastpath:
@@ -1753,6 +1752,7 @@ ENTRY(nmi)
        movq    %rax, %cr3
 2:
 #endif
+       STUFF_RSB
        ENABLE_IBRS
        call    do_nmi
        DISABLE_IBRS