x86/speculation: functions for supporting enhanced IBRS

Indirect Branch Restricted Speculation (IBRS) is available either
with a basic support (basic IBRS) or with an enhanced support
(enhanced IBRS). Currently only basic IBRS is implemented, and it
requires IBRS to be set after every transition to a more privileged
predictor mode.

Enhanced IBRS supports an 'always on' model in which IBRS is enabled
once and never disabled. We enhance the existing functions, introduce
new identifiers, and rename existing ones in order to be able to
differentiate between basic and enhanced IBRS.

Signed-off-by: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>
Co-developed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
(cherry picked from commit d762c3e419e1df1e7671c346ab4247a495fbc3dd from UEK5)

Orabug: 28474851

Signed-off-by: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Signed-off-by: Brian Maly <brian.maly@oracle.com>
Conflicts:
	arch/x86/include/asm/spec_ctrl.h
(Differences in the IBRS related macro definitions. Not adding
spec_ctrl_flush_all_cpus() that is already in bugs_64.c. Change
set_ibrs_inuse() return value from boolean to void.)

	arch/x86/kernel/cpu/bugs.c
(UEK4 uses bugs_64.c. Slight differences in
disable_ibrs_and_friends() and cpu_show_common().)

	arch/x86/kernel/cpu/spec_ctrl.c
(No need to remove the spec_ctrl_flush_all_cpus(), it is in bugs_64.c)

Signed-off-by: Brian Maly <brian.maly@oracle.com>

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 6fecb6e76f8b43288ab383a3a003ad56c0b6d6e1..ccfd20a9ef190de6fc17332c41fdcf9c67782e90 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -192,6 +192,7 @@ enum spectre_v2_mitigation {
        SPECTRE_V2_RETPOLINE_GENERIC,
        SPECTRE_V2_RETPOLINE_AMD,
        SPECTRE_V2_IBRS,
+       SPECTRE_V2_IBRS_ENHANCED,
 };
 
 enum spec_ctrl_set_context {

diff --git a/arch/x86/include/asm/spec_ctrl.h b/arch/x86/include/asm/spec_ctrl.h
index f3d58700b132d4cef0170b93ac72c996830fc989..b55f9db1e3bd2cc9c6fc94efa43a3f5c38e4c5ad 100644 (file)
--- a/arch/x86/include/asm/spec_ctrl.h
+++ b/arch/x86/include/asm/spec_ctrl.h
@@ -6,10 +6,22 @@
 #include <asm/cpufeature.h>
 #include <asm/alternative-asm.h>
 
-#define SPEC_CTRL_IBRS_INUSE           (1<<0)  /* OS enables IBRS usage */
-#define SPEC_CTRL_IBRS_SUPPORTED       (1<<1)  /* System supports IBRS */
-#define SPEC_CTRL_IBRS_ADMIN_DISABLED  (1<<2)  /* Admin disables IBRS */
+/*
+ * IBRS Flags.
+ *
+ * Note that we use dedicated bits to specify if basic IBRS is
+ * in use (SPEC_CTRL_BASIC_IBRS_INUSE) or enhanced IBRS is in use
+ * (SPEC_CTRL_ENHCD_IBRS_INUSE), instead of combining multiple
+ * bits (e.g. SPEC_CTRL_BASIC_IBRS_INUSE | SPEC_CTRL_ENHCD_IBRS_SUPPORTED).
+ * This is to optimize testing when checking if basic or enhanced
+ * IBRS is in use, in particular for assembly code.
+ */
+#define SPEC_CTRL_BASIC_IBRS_INUSE     (1<<0)  /* OS enables basic IBRS usage */
+#define SPEC_CTRL_IBRS_SUPPORTED       (1<<1)  /* System supports IBRS (basic or enhanced) */
+#define SPEC_CTRL_IBRS_ADMIN_DISABLED  (1<<2)  /* Admin disables IBRS (basic and enhanced) */
 #define SPEC_CTRL_IBRS_FIRMWARE        (1<<3)  /* IBRS to be used on firmware paths */
+#define SPEC_CTRL_ENHCD_IBRS_SUPPORTED (1<<4)  /* System supports enhanced IBRS */
+#define SPEC_CTRL_ENHCD_IBRS_INUSE     (1<<5)  /* OS enables enhanced IBRS usage */
 
 #ifdef __ASSEMBLY__
 
@@ -125,7 +137,7 @@
        add $(32*8), %rsp;
 
 .macro ENABLE_IBRS
-       testl   $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
+       testl   $SPEC_CTRL_BASIC_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
        jz      7f
        __ASM_ENABLE_IBRS
        jmp     20f
@@ -135,7 +147,7 @@
 .endm
 
 .macro ENABLE_IBRS_CLOBBER
-       testl   $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
+       testl   $SPEC_CTRL_BASIC_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
        jz      11f
        __ASM_ENABLE_IBRS_CLOBBER
        jmp     21f
@@ -145,7 +157,7 @@
 .endm
 
 .macro ENABLE_IBRS_SAVE_AND_CLOBBER save_reg:req
-       testl   $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
+       testl   $SPEC_CTRL_BASIC_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
        jz      12f
 
        movl    $MSR_IA32_SPEC_CTRL, %ecx
@@ -163,7 +175,7 @@
 .endm
 
 .macro RESTORE_IBRS_CLOBBER save_reg:req
-       testl   $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
+       testl   $SPEC_CTRL_BASIC_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
        jz      13f
 
        cmp     \save_reg, PER_CPU_VAR(x86_spec_ctrl_priv_cpu)
@@ -180,7 +192,7 @@
 .endm
 
 .macro DISABLE_IBRS
-       testl   $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
+       testl   $SPEC_CTRL_BASIC_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
        jz      9f
        __ASM_DISABLE_IBRS
 9:
@@ -194,7 +206,7 @@ ALTERNATIVE "", __stringify(__ASM_SET_IBPB), X86_FEATURE_IBRS
 ALTERNATIVE __stringify(__ASM_STUFF_RSB), "", X86_FEATURE_STUFF_RSB
 .endm
 
-#else
+#else /* __ASSEMBLY__ */
 
 /* Defined in bugs_64.c */
 extern u64 x86_spec_ctrl_priv;
@@ -226,8 +238,7 @@ DECLARE_STATIC_KEY_FALSE(retpoline_enabled_key);
 #define ibrs_firmware          (use_ibrs & SPEC_CTRL_IBRS_FIRMWARE)
 #define ibrs_supported         (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED)
 #define ibrs_disabled          (use_ibrs & SPEC_CTRL_IBRS_ADMIN_DISABLED)
-
-#define ibrs_inuse             (cpu_ibrs_inuse())
+#define eibrs_supported                (use_ibrs & SPEC_CTRL_ENHCD_IBRS_SUPPORTED)
 
 static inline void update_cpu_spec_ctrl(int cpu)
 {
@@ -264,26 +275,33 @@ static inline void update_cpu_ibrs_all(void)
                update_cpu_ibrs(&cpu_data(cpu_index));
 }
 
-static inline bool set_ibrs_inuse(void)
+static inline void set_ibrs_inuse(void)
 {
-       if (ibrs_supported && !ibrs_disabled) {
-               use_ibrs |= SPEC_CTRL_IBRS_INUSE;
-               update_cpu_ibrs_all();
-               /* Update what sysfs shows. */
-               sysctl_ibrs_enabled = true;
-               /* When entering kernel */
-               x86_spec_ctrl_priv |= SPEC_CTRL_FEATURE_ENABLE_IBRS;
-               /* Update per-cpu spec_ctrl */
-               update_cpu_spec_ctrl_all();
-               return true;
-       } else {
-               return false;
-       }
+       if (!ibrs_supported || ibrs_disabled)
+               return;
+
+       use_ibrs &= ~(SPEC_CTRL_BASIC_IBRS_INUSE | SPEC_CTRL_ENHCD_IBRS_INUSE);
+
+       if (eibrs_supported)
+               /* Enhanced IBRS is available */
+               use_ibrs |= SPEC_CTRL_ENHCD_IBRS_INUSE;
+       else
+               /* Basic IBRS is available */
+               use_ibrs |= SPEC_CTRL_BASIC_IBRS_INUSE;
+
+       /* Propagate the change to each cpu */
+       update_cpu_ibrs_all();
+       /* Update what sysfs shows */
+       sysctl_ibrs_enabled = true;
+       /* When entering kernel */
+       x86_spec_ctrl_priv |= SPEC_CTRL_FEATURE_ENABLE_IBRS;
+       /* Update per-cpu spec_ctrl */
+       update_cpu_spec_ctrl_all();
 }
 
 static inline void clear_ibrs_inuse(void)
 {
-       use_ibrs &= ~SPEC_CTRL_IBRS_INUSE;
+       use_ibrs &= ~(SPEC_CTRL_BASIC_IBRS_INUSE | SPEC_CTRL_ENHCD_IBRS_INUSE);
        update_cpu_ibrs_all();
        /* Update what sysfs shows. */
        sysctl_ibrs_enabled = false;
@@ -295,20 +313,31 @@ static inline void clear_ibrs_inuse(void)
        update_cpu_spec_ctrl_all();
 }
 
+static inline int check_basic_ibrs_inuse(void)
+{
+       if (use_ibrs & SPEC_CTRL_BASIC_IBRS_INUSE)
+               return 1;
+
+       /* rmb to prevent wrong speculation for security */
+       rmb();
+       return 0;
+}
+
 static inline int check_ibrs_inuse(void)
 {
-       if (use_ibrs & SPEC_CTRL_IBRS_INUSE) {
+       if (use_ibrs & (SPEC_CTRL_BASIC_IBRS_INUSE |
+               SPEC_CTRL_ENHCD_IBRS_INUSE))
                return 1;
-       } else {
-               /* rmb to prevent wrong speculation for security */
-               rmb();
-       }
+
+       /* rmb to prevent wrong speculation for security */
+       rmb();
        return 0;
 }
 
-static inline int cpu_ibrs_inuse(void)
+static inline int cpu_ibrs_inuse_any(void)
 {
-       return (this_cpu_read(cpu_ibrs) & SPEC_CTRL_IBRS_INUSE) ? 1 : 0;
+       return (this_cpu_read(cpu_ibrs) &
+           (SPEC_CTRL_BASIC_IBRS_INUSE | SPEC_CTRL_ENHCD_IBRS_INUSE)) ? 1 : 0;
 }
 
 static inline void set_ibrs_supported(void)
@@ -326,7 +355,12 @@ static inline void set_ibrs_disabled(void)
 static inline void clear_ibrs_disabled(void)
 {
        use_ibrs &= ~SPEC_CTRL_IBRS_ADMIN_DISABLED;
-       (void)set_ibrs_inuse();
+       set_ibrs_inuse();
+}
+
+static inline void set_ibrs_enhanced(void)
+{
+       use_ibrs |= SPEC_CTRL_ENHCD_IBRS_SUPPORTED;
 }
 
 static inline void set_ibrs_firmware(void)

diff --git a/arch/x86/kernel/cpu/bugs_64.c b/arch/x86/kernel/cpu/bugs_64.c
index 7f0252f8009b248af1c3bcb8d5a7bc129870eaa3..08efcb18dc5a9f1a45fb43084908399dcc9259ce 100644 (file)
--- a/arch/x86/kernel/cpu/bugs_64.c
+++ b/arch/x86/kernel/cpu/bugs_64.c
@@ -27,9 +27,12 @@
 
 /*
  * use_ibrs flags:
- * SPEC_CTRL_IBRS_INUSE                        indicate if ibrs is currently in use
- * SPEC_CTRL_IBRS_SUPPORTED            indicate if system supports ibrs
- * SPEC_CTRL_IBRS_ADMIN_DISABLED       indicate if admin disables ibrs
+ * SPEC_CTRL_BASIC_IBRS_INUSE          basic ibrs is currently in use
+ * SPEC_CTRL_IBRS_SUPPORTED            system supports basic ibrs
+ * SPEC_CTRL_IBRS_ADMIN_DISABLED       admin disables ibrs (basic and enhanced)
+ * SPEC_CTRL_IBRS_FIRMWARE             ibrs to be used on firmware paths
+ * SPEC_CTRL_ENHCD_IBRS_SUPPORTED      system supports enhanced ibrs
+ * SPEC_CTRL_ENHCD_IBRS_INUSE          Enhanced ibrs is currently in use
  */
 int use_ibrs;
 EXPORT_SYMBOL(use_ibrs);
@@ -258,8 +261,8 @@ static const char *spectre_v2_strings[] = {
        [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
        [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
        [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
-       [SPECTRE_V2_IBRS]                       = "Mitigation: IBRS",
-
+       [SPECTRE_V2_IBRS]                       = "Mitigation: Basic IBRS",
+       [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
 };
 
 #undef pr_fmt
@@ -324,7 +327,7 @@ x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
                 * modifiable bits in the host base value and or the
                 * modifiable bits from the guest value.
                 */
-               if (ibrs_inuse)
+               if (cpu_ibrs_inuse_any())
                        /*
                         * Except on IBRS we don't want to use host base value
                         * but rather the privilege value which has IBRS set.
@@ -334,7 +337,7 @@ x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
                guestval = hostval & ~x86_spec_ctrl_mask;
                guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 
-               if (ibrs_inuse) {
+               if (cpu_ibrs_inuse_any()) {
                        /* You may wonder why we don't just jump to the
                         * 'if (hostval ! guestval)' conditional to save an MSR.
                         * (by say the guest MSR value is IBRS and hostval being
@@ -409,7 +412,7 @@ void find_retpoline_alternative(void)
                return;
 
        if (allow_retpoline_fallback) {
-               if (!ibrs_inuse) {
+               if (!cpu_ibrs_inuse_any()) {
                        /* try to enable ibrs */
                        if (ibrs_supported) {
                                change_spectre_v2_mitigation(SPECTRE_V2_ENABLE_IBRS);
@@ -595,7 +598,8 @@ void refresh_set_spectre_v2_enabled(void)
                spectre_v2_enabled = test_taint(TAINT_NO_RETPOLINE) ?
                        SPECTRE_V2_NONE : retpoline_mode;
        } else if (check_ibrs_inuse()) {
-               spectre_v2_enabled = SPECTRE_V2_IBRS;
+               spectre_v2_enabled = (check_basic_ibrs_inuse() ?
+                       SPECTRE_V2_IBRS : SPECTRE_V2_IBRS_ENHANCED);
        } else {
                spectre_v2_enabled = SPECTRE_V2_NONE;
        }
@@ -671,26 +675,30 @@ disable:
 static void __init ibrs_select(enum spectre_v2_mitigation *mode)
 {
        /* Turn it on (if possible) */
-       if (set_ibrs_inuse()) {
-               *mode = SPECTRE_V2_IBRS;
-               if (!boot_cpu_has(X86_FEATURE_SMEP))
-                       setup_force_cpu_cap(X86_FEATURE_STUFF_RSB);
-       } else
+       set_ibrs_inuse();
+       if (!check_ibrs_inuse()) {
                pr_info("IBRS could not be enabled.\n");
+               return;
+       }
+       /* Determine the specific IBRS variant in use */
+       *mode = (check_basic_ibrs_inuse() ?
+               SPECTRE_V2_IBRS : SPECTRE_V2_IBRS_ENHANCED);
+
+       if (boot_cpu_has(X86_FEATURE_SMEP))
+               return;
+
+       setup_force_cpu_cap(X86_FEATURE_STUFF_RSB);
+
+       if (*mode == SPECTRE_V2_IBRS_ENHANCED)
+               pr_warn("Enhanced IBRS might not provide full mitigation against Spectre v2 if SMEP is not available.\n");
 }
 
 static void __init disable_ibrs_and_friends(bool disable)
 {
-       if (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED) {
-               unsigned int cpu;
-
-               get_online_cpus();
-               for_each_online_cpu(cpu)
-                       wrmsrl_on_cpu(cpu, MSR_IA32_SPEC_CTRL,
-                                     x86_spec_ctrl_base & ~SPEC_CTRL_FEATURE_ENABLE_IBRS);
-
-               put_online_cpus();
-       }
+       if (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED)
+               /* Disable IBRS an all cpus */
+               spec_ctrl_flush_all_cpus(MSR_IA32_SPEC_CTRL,
+                       x86_spec_ctrl_base & ~SPEC_CTRL_FEATURE_ENABLE_IBRS);
 
        if (disable) {
                set_ibrs_disabled();
@@ -760,7 +768,7 @@ static void __init spectre_v2_select_mitigation(void)
            mode != SPECTRE_V2_RETPOLINE_MINIMAL_AMD) {
 
                pr_info("Options: %s%s%s\n",
-                       ibrs_supported ? "IBRS " : "",
+                       ibrs_supported ? (eibrs_supported ? "IBRS(enhanced) " : "IBRS(basic) ") : "",
                        check_ibpb_inuse() ? "IBPB " : "",
                        retp_compiler() ? "retpoline" : "");
 
@@ -776,7 +784,8 @@ static void __init spectre_v2_select_mitigation(void)
 
                                /* Start the engine! */
                                ibrs_select(&mode);
-                               if (mode == SPECTRE_V2_IBRS)
+                               if (mode == SPECTRE_V2_IBRS ||
+                                       mode == SPECTRE_V2_IBRS_ENHANCED)
                                        goto display;
                                /* But if we can't, then just use retpoline */
                        }