x86/speculation: Use IBRS if available before calling into firmware

Retpoline means the kernel is safe because it has no indirect branches.
But firmware isn't, so use IBRS for firmware calls if it's available.

Block preemption while IBRS is set, although in practice the call sites
already had to be doing that.

Ignore hpwdt.c for now. It's taking spinlocks and calling into firmware
code, from an NMI handler. I don't want to touch that with a bargepole.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: arjan.van.de.ven@intel.com
Cc: bp@alien8.de
Cc: dave.hansen@intel.com
Cc: jmattson@google.com
Cc: karahmed@amazon.de
Cc: kvm@vger.kernel.org
Cc: pbonzini@redhat.com
Cc: rkrcmar@redhat.com
Link: http://lkml.kernel.org/r/1519037457-7643-2-git-send-email-dwmw@amazon.co.uk
Signed-off-by: Ingo Molnar <mingo@kernel.org>
(cherry-pick from dd84441a7971)
[Backport:

We need to be more dynamic. We may have retpoline disabled for some time and
then when somebody loads an proprietary module (say nvidia.ko) we can stop making
these calls (as we would be doing IBRS calls now).

As such we we use a new bit on the ibrs global value - which on bootup is set
to be enabled  (if IBRS firmware is detected), and then if retpoline is selected
it is still used. But if 'spectre_v2=off' is off, then it is disabled.

The original feature uses a CPU feature, but we are much more dynamic
thanks to the SysFS and retpoline-module-check.]

Orabug: 27516477
Signed-off-by: Krish Sadhukhan <krish.sadhukhan@oracle.com>
Reviewed-by: Darren Kenny <darren.kenny@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>

diff --git a/arch/x86/include/asm/apm.h b/arch/x86/include/asm/apm.h
index 20370c6db74bb6dd09bbb78d4ded0b7adfd27d3e..9e2fbc19d6ab93d3d9f01d36da1540937665847f 100644 (file)
--- a/arch/x86/include/asm/apm.h
+++ b/arch/x86/include/asm/apm.h
@@ -6,6 +6,8 @@
 #ifndef _ASM_X86_MACH_DEFAULT_APM_H
 #define _ASM_X86_MACH_DEFAULT_APM_H
 
+#include <asm/spec_ctrl.h>
+
 #ifdef APM_ZERO_SEGS
 #      define APM_DO_ZERO_SEGS \
                "pushl %%ds\n\t" \
@@ -27,6 +29,8 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in,
                                        u32 *eax, u32 *ebx, u32 *ecx,
                                        u32 *edx, u32 *esi)
 {
+       unprotected_firmware_begin();
+
        /*
         * N.B. We do NOT need a cld after the BIOS call
         * because we always save and restore the flags.
@@ -43,6 +47,8 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in,
                  "=S" (*esi)
                : "a" (func), "b" (ebx_in), "c" (ecx_in)
                : "memory", "cc");
+
+       unprotected_firmware_end();
 }
 
 static inline u8 apm_bios_call_simple_asm(u32 func, u32 ebx_in,
@@ -51,6 +57,8 @@ static inline u8 apm_bios_call_simple_asm(u32 func, u32 ebx_in,
        int     cx, dx, si;
        u8      error;
 
+       unprotected_firmware_begin();
+
        /*
         * N.B. We do NOT need a cld after the BIOS call
         * because we always save and restore the flags.
@@ -67,6 +75,8 @@ static inline u8 apm_bios_call_simple_asm(u32 func, u32 ebx_in,
                  "=S" (si)
                : "a" (func), "b" (ebx_in), "c" (ecx_in)
                : "memory", "cc");
+
+       unprotected_firmware_end();
        return error;
 }
 

diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index a7c9206b5beafcdc919cd5c6cdfef1fbdefefe63..ed41dda1933abeee9fea0fa3e2de69fe027ddbf6 100644 (file)
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -3,6 +3,7 @@
 
 #include <asm/i387.h>
 #include <asm/pgtable.h>
+#include <asm/spec_ctrl.h>
 
 /*
  * We map the EFI regions needed for runtime services non-contiguously,
@@ -39,8 +40,10 @@ extern unsigned long asmlinkage efi_call_phys(void *, ...);
 ({                                                                     \
        efi_status_t __s;                                               \
        kernel_fpu_begin();                                             \
+       unprotected_firmware_begin();                                   \
        __s = ((efi_##f##_t __attribute__((regparm(0)))*)               \
                efi.systab->runtime->f)(args);                          \
+       unprotected_firmware_end();                                     \
        kernel_fpu_end();                                               \
        __s;                                                            \
 })
@@ -49,8 +52,10 @@ extern unsigned long asmlinkage efi_call_phys(void *, ...);
 #define __efi_call_virt(f, args...) \
 ({                                                                     \
        kernel_fpu_begin();                                             \
+       unprotected_firmware_begin();                                   \
        ((efi_##f##_t __attribute__((regparm(0)))*)                     \
                efi.systab->runtime->f)(args);                          \
+       unprotected_firmware_end();                                     \
        kernel_fpu_end();                                               \
 })
 
@@ -71,7 +76,9 @@ extern u64 asmlinkage efi_call(void *fp, ...);
        efi_sync_low_kernel_mappings();                                 \
        preempt_disable();                                              \
        __kernel_fpu_begin();                                           \
+       unprotected_firmware_begin();                                   \
        __s = efi_call((void *)efi.systab->runtime->f, __VA_ARGS__);    \
+       unprotected_firmware_end();                                     \
        __kernel_fpu_end();                                             \
        preempt_enable();                                               \
        __s;                                                            \

diff --git a/arch/x86/include/asm/spec_ctrl.h b/arch/x86/include/asm/spec_ctrl.h
index 66424e6b4c373f8ace2142143bdca8e89992a256..d32e6bb9fc4290944ede2e57cc4f4f99b811e635 100644 (file)
--- a/arch/x86/include/asm/spec_ctrl.h
+++ b/arch/x86/include/asm/spec_ctrl.h
@@ -9,6 +9,7 @@
 #define SPEC_CTRL_IBRS_INUSE           (1<<0)  /* OS enables IBRS usage */
 #define SPEC_CTRL_IBRS_SUPPORTED       (1<<1)  /* System supports IBRS */
 #define SPEC_CTRL_IBRS_ADMIN_DISABLED  (1<<2)  /* Admin disables IBRS */
+#define SPEC_CTRL_IBRS_FIRMWARE        (1<<3)  /* IBRS to be used on firmware paths */
 
 #ifdef __ASSEMBLY__
 
@@ -208,6 +209,10 @@ extern int use_ibrs;
 extern u32 sysctl_ibrs_enabled;
 extern struct mutex spec_ctrl_mutex;
 
+extern void unprotected_firmware_begin(void);
+extern void unprotected_firmware_end(void);
+
+#define ibrs_firmware          (use_ibrs & SPEC_CTRL_IBRS_FIRMWARE)
 #define ibrs_supported         (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED)
 #define ibrs_disabled          (use_ibrs & SPEC_CTRL_IBRS_ADMIN_DISABLED)
 
@@ -250,6 +255,17 @@ static inline void set_ibrs_disabled(void)
        sysctl_ibrs_enabled = ibrs_inuse ? 1 : 0;
 }
 
+static inline void set_ibrs_firmware(void)
+{
+       if (ibrs_supported)
+               use_ibrs |= SPEC_CTRL_IBRS_FIRMWARE;
+}
+
+static inline void disable_ibrs_firmware(void)
+{
+       use_ibrs &= ~SPEC_CTRL_IBRS_FIRMWARE;
+}
+
 static inline void clear_ibrs_disabled(void)
 {
        use_ibrs &= ~SPEC_CTRL_IBRS_ADMIN_DISABLED;

diff --git a/arch/x86/kernel/cpu/bugs_64.c b/arch/x86/kernel/cpu/bugs_64.c
index c38997a0d2273758249543d21c44a54435a3652a..5ae5dcdb54642c72e3f5b019094bf9c18dc852d7 100644 (file)
--- a/arch/x86/kernel/cpu/bugs_64.c
+++ b/arch/x86/kernel/cpu/bugs_64.c
@@ -285,9 +285,15 @@ static void __init disable_ibrs_and_friends(bool disable_ibpb)
 
                put_online_cpus();
        }
-       /* We need to use IBPB with retpoline if it is available. */
-       if (disable_ibpb)
+       /*
+        * We need to use IBPB with retpoline if it is available.
+        * And also IBRS for firmware paths.
+        */
+       if (disable_ibpb) {
                set_ibpb_disabled();
+               disable_ibrs_firmware();
+       } else
+               set_ibrs_firmware();
 }
 
 static bool __init retpoline_selected(enum spectre_v2_mitigation_cmd cmd)
@@ -426,6 +432,9 @@ out:
        /* Initialize Indirect Branch Prediction Barrier if supported */
        if (boot_cpu_has(X86_FEATURE_IBPB) && ibpb_inuse)
                pr_info("Enabling Indirect Branch Prediction Barrier\n");
+
+       if (ibrs_firmware)
+               pr_info("Enabling Restricted Speculation for firmware calls\n");
 }
 
 #undef pr_fmt
@@ -456,7 +465,8 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
        if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
                return sprintf(buf, "Not affected\n");
 
-       return sprintf(buf, "%s%s\n", spectre_v2_strings[spectre_v2_enabled],
+       return sprintf(buf, "%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
+                                       ibrs_firmware ? ", IBRS_FW" : "",
                                        ibpb_inuse ? ", IBPB" : "");
 }
 #endif

diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c
index 0bcaed164c54ce076f42656d32409e5fcee1d97b..04ca665116818d7922736ad2cfa396b0f10d8935 100644 (file)
--- a/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@ -173,6 +173,12 @@ void init_scattered_cpuid_features(struct cpuinfo_x86 *c)
 
        if (cpu_has(c, X86_FEATURE_IBRS)) {
                set_ibrs_supported();
+               /*
+                * Don't do this after disable_ibrs_and_friends as we would
+                * re-enable it (say if spectre_v2=off is used).
+                */
+               if (&boot_cpu_data == c)
+                       set_ibrs_firmware();
                sysctl_ibrs_enabled = ibrs_inuse ? 1 : 0;
        }
 

diff --git a/arch/x86/kernel/cpu/spec_ctrl.c b/arch/x86/kernel/cpu/spec_ctrl.c
index ec3e307f0289ae51af48fd42c403f9aecb108927..a568124222d346429557972434b1d9c7f215d224 100644 (file)
--- a/arch/x86/kernel/cpu/spec_ctrl.c
+++ b/arch/x86/kernel/cpu/spec_ctrl.c
@@ -6,6 +6,7 @@
 #include <linux/cpu.h>
 #include <asm/spec_ctrl.h>
 #include <asm/cpufeature.h>
+#include <asm/microcode.h>
 
 /*
  * use_ibrs
@@ -78,10 +79,12 @@ static ssize_t ibrs_enabled_write(struct file *file,
 
        if (!enable) {
                set_ibrs_disabled();
+               disable_ibrs_firmware();
                if (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED)
                        spec_ctrl_flush_all_cpus(MSR_IA32_SPEC_CTRL, SPEC_CTRL_FEATURE_DISABLE_IBRS);
        } else {
                clear_ibrs_disabled();
+               set_ibrs_firmware();
        }
        refresh_set_spectre_v2_enabled();
 
@@ -155,3 +158,42 @@ static int __init debugfs_spec_ctrl(void)
         return 0;
 }
 late_initcall(debugfs_spec_ctrl);
+
+#ifdef RETPOLINE
+/*
+ * RETPOLINE does not protect against indirect speculation
+ * in firmware code.  Enable IBRS to protect firmware execution.
+ */
+void unprotected_firmware_begin(void)
+{
+        if (retpoline_enabled() && ibrs_firmware) {
+                native_wrmsrl(MSR_IA32_SPEC_CTRL, SPEC_CTRL_FEATURE_ENABLE_IBRS);
+        } else {
+                /*
+                 * rmb prevents unwanted speculation when we
+                 * are setting IBRS
+                 */
+                rmb();
+        }
+}
+EXPORT_SYMBOL_GPL(unprotected_firmware_begin);
+
+void unprotected_firmware_end(void)
+{
+        if (retpoline_enabled() && ibrs_firmware) {
+                native_wrmsrl(MSR_IA32_SPEC_CTRL, SPEC_CTRL_FEATURE_DISABLE_IBRS);
+        }
+}
+EXPORT_SYMBOL_GPL(unprotected_firmware_end);
+
+#else
+void unprotected_firmware_begin(void)
+{
+}
+EXPORT_SYMBOL_GPL(unprotected_firmware_begin);
+
+void unprotected_firmware_end(void)
+{
+}
+EXPORT_SYMBOL_GPL(unprotected_firmware_end);
+#endif