x86/enter: Use IBRS on syscall and interrupts

Set IBRS upon kernel entrance via syscall and interrupts. Clear it
upon exit.

Orabug: 27344012
CVE: CVE-2017-5715

Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
[Backport: I had to add 'asm/spec_ctrl.h' in the assembler files]
 Also we should not put ENABLE_IBRS on irq_entries_start]

Reviewed-by: John Haxby <john.haxby@oracle.com>
Signed-off-by: Kirtikar Kashyap <kirtikar.kashyap@oracle.com>

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 27e54946ef35613222bca60ee4d65196dbae31ea..961ae5a055996f97f9b3783cf48d28ebffd121d0 100644 (file)
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -15,6 +15,7 @@
 #include <asm/irqflags.h>
 #include <asm/asm.h>
 #include <asm/smap.h>
+#include <asm/spec_ctrl.h>
 #include <linux/linkage.h>
 #include <linux/err.h>
 
@@ -147,6 +148,8 @@ ENTRY(ia32_sysenter_target)
        sub     $(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
        CFI_ADJUST_CFA_OFFSET 10*8
 
+       ENABLE_IBRS
+
        /*
         * no need to do an access_ok check here because rbp has been
         * 32bit zero extended
@@ -529,6 +532,8 @@ ENTRY(ia32_syscall)
        sub     $(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
        CFI_ADJUST_CFA_OFFSET 10*8
 
+       ENABLE_IBRS
+
        orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
        testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
        jnz ia32_tracesys
@@ -546,6 +551,7 @@ ia32_sysret:
        movq %rax,RAX(%rsp)
 ia32_ret_from_sys_call:
        CLEAR_RREGS
+       DISABLE_IBRS
        jmp int_ret_from_sys_call
 
 ia32_tracesys:

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 4b7000e770427eab5b5a796098481c432bafc361..82148ae50eb1c2d5f397a9bd5fce661beed17d36 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -46,6 +46,7 @@
 #include <asm/smap.h>
 #include <asm/pgtable_types.h>
 #include <asm/dtrace_util.h>
+#include <asm/spec_ctrl.h>
 #include <linux/err.h>
 
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
@@ -219,6 +220,8 @@ GLOBAL(system_call_after_swapgs)
        movq    %rsp,PER_CPU_VAR(rsp_scratch)
        movq    PER_CPU_VAR(kernel_stack),%rsp
 
+       ENABLE_IBRS
+
        /* Construct struct pt_regs on stack */
        pushq_cfi $__USER_DS                    /* pt_regs->ss */
        pushq_cfi PER_CPU_VAR(rsp_scratch)      /* pt_regs->sp */
@@ -475,6 +478,7 @@ syscall_return:
         * perf profiles.  Nothing jumps here.
         */
 syscall_return_via_sysret:
+       DISABLE_IBRS
        CFI_REMEMBER_STATE
        /* r11 is already restored (see code above) */
        RESTORE_C_REGS_EXCEPT_R11
@@ -792,6 +796,7 @@ retint_kernel:
  * which come from interrupts/exception and from syscalls, merge.
  */
 restore_c_regs_and_iret:
+       DISABLE_IBRS
        RESTORE_C_REGS
        REMOVE_PT_GPREGS_FROM_STACK 8
        INTERRUPT_RETURN
@@ -1432,7 +1437,8 @@ ENTRY(paranoid_entry)
        js 1f   /* negative -> in kernel */
        SWAPGS
        xorl %ebx,%ebx
-1:     ret
+1:     ENABLE_IBRS_CLOBBER
+       ret
        CFI_ENDPROC
 END(paranoid_entry)
 
@@ -1480,6 +1486,7 @@ ENTRY(error_entry)
        je error_kernelspace
 error_swapgs:
        SWAPGS
+       ENABLE_IBRS_CLOBBER
 error_sti:
        TRACE_IRQS_OFF
        ret
@@ -1510,6 +1517,7 @@ bstep_iret:
 
 error_bad_iret:
        SWAPGS
+       ENABLE_IBRS
        mov %rsp,%rdi
        call fixup_bad_iret
        mov %rax,%rsp
@@ -1613,6 +1621,7 @@ ENTRY(nmi)
        cld
        movq    %rsp, %rdx
        movq    PER_CPU_VAR(kernel_stack), %rsp
+       ENABLE_IBRS
        pushq   5*8(%rdx)       /* pt_regs->ss */
        pushq   4*8(%rdx)       /* pt_regs->rsp */
        pushq   3*8(%rdx)       /* pt_regs->flags */