x86/speculation: Enable enhanced IBRS usage

Enhanced IBRS supports an 'always on' model (aka IBRS_ALL) in
which IBRS is enabled once and never disabled, while basic IBRS
requires IBRS to be set after every transition to a more
privileged predictor mode.

IBRS is enabled at boot if selected as the spectre v2
mitigation, or by using the debugfs interface at
/sys/kernel/debug/x86/ibrs_enabled
to dynamically toggle between IBRS and retpoline during
regular system operation. In both cases, if enhanced IBRS is
available it will be preferred over basic IBRS.

Signed-off-by: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>
Co-developed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
(cherry picked from commit d19b574a5e5dabca5158b3331aa1a31070da753c from UEK5)

Orabug: 28474851

Signed-off-by: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Signed-off-by: Brian Maly <brian.maly@oracle.com>
Conflicts:
	arch/x86/include/asm/cpufeatures.h
(File named cpufeature.h in UEK4)

	arch/x86/kernel/cpu/bugs.c
(File named bugs_64.c in UEK4)

	arch/x86/kernel/cpu/common.c
	arch/x86/kernel/cpu/spec_ctrl.c
(Corresponding code located in bug_64.c in UEK4)

Signed-off-by: Brian Maly <brian.maly@oracle.com>

diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index c6a4944caefaf2bd349fb40b146845e3f05ab4a7..403322a3ce0b781f7063475c2f0487062d2541b4 100644 (file)
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -206,7 +206,7 @@
 #define X86_FEATURE_IBRS       ( 7*32+20) /* Control Speculation Control */
 #define X86_FEATURE_STIBP      ( 7*32+21) /* Single Thread Indirect Branch Predictors */
 #define X86_FEATURE_IA32_ARCH_CAPS     ( 7*32+22) /* Control Speculation Control */
-#define X86_FEATURE_IBRS_ALL   ( 7*32+23) /* IBRS all the time */
+#define X86_FEATURE_IBRS_ENHANCED      ( 7*32+23) /* IBRS all the time */
 
 /* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13.
  * But thanks to kABI we have to jam it somewhere else. */

diff --git a/arch/x86/kernel/cpu/bugs_64.c b/arch/x86/kernel/cpu/bugs_64.c
index 08efcb18dc5a9f1a45fb43084908399dcc9259ce..faf92188b47498f13161db8fd5666f625ce7891e 100644 (file)
--- a/arch/x86/kernel/cpu/bugs_64.c
+++ b/arch/x86/kernel/cpu/bugs_64.c
@@ -501,14 +501,14 @@ void change_spectre_v2_mitigation(enum spectre_v2_mitigation_action action)
        /*
         * Define the current state.
         *
-        * IBRS firmware is enabled if either IBRS or retpoline is enabled.
-        * If both IBRS and retpoline are disabled, then IBRS firmware is
-        * disabled too.
+        * IBRS firmware is enabled if either basic IBRS or retpoline is
+        * enabled. If both IBRS and retpoline are disabled, then IBRS firmware
+        * is disabled too.
         */
 
        ibrs_used = !ibrs_disabled;
        retpoline_used = !!retpoline_enabled();
-       ibrs_fw_used = (ibrs_used || retpoline_used);
+       ibrs_fw_used = ((ibrs_used && !eibrs_supported) || retpoline_used);
 
        /*
         * Define the requested state.
@@ -527,7 +527,7 @@ void change_spectre_v2_mitigation(enum spectre_v2_mitigation_action action)
 
        case SPECTRE_V2_ENABLE_IBRS:
                ibrs_requested = true;
-               ibrs_fw_requested = true;
+               ibrs_fw_requested = !eibrs_supported;
                retpoline_requested = false;
                break;
 
@@ -545,7 +545,7 @@ void change_spectre_v2_mitigation(enum spectre_v2_mitigation_action action)
 
        case SPECTRE_V2_DISABLE_RETPOLINE:
                ibrs_requested = ibrs_used;
-               ibrs_fw_requested = ibrs_used;
+               ibrs_fw_requested = ibrs_used && !eibrs_supported;
                retpoline_requested = false;
                break;
        }
@@ -555,6 +555,10 @@ void change_spectre_v2_mitigation(enum spectre_v2_mitigation_action action)
        if (ibrs_requested != ibrs_used) {
                if (ibrs_requested) {
                        clear_ibrs_disabled();
+                       /* If enhanced IBRS is available, turn it on now */
+                       if (eibrs_supported)
+                               spec_ctrl_flush_all_cpus(MSR_IA32_SPEC_CTRL,
+                                       x86_spec_ctrl_priv);
                } else {
                        set_ibrs_disabled();
                        if (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED) {
@@ -753,8 +757,10 @@ static void __init spectre_v2_select_mitigation(void)
 
        if (cmd == SPECTRE_V2_CMD_IBRS || !IS_ENABLED(CONFIG_RETPOLINE)) {
                ibrs_select(&mode);
-               if (mode != SPECTRE_V2_NONE)
+               if (mode != SPECTRE_V2_NONE) {
+                       /* One of the IBRS modes was successfully selected */
                        goto display;
+               }
                if (!IS_ENABLED(CONFIG_RETPOLINE)) {
                        pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
                        goto out;
@@ -810,11 +816,16 @@ out:
        setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
        pr_info("Filling RSB on context switch\n");
 
-       /* IBRS is unnecessary with retpoline mitigation. */
-       if (mode == SPECTRE_V2_RETPOLINE_GENERIC ||
-           mode == SPECTRE_V2_RETPOLINE_AMD) {
+       if (mode == SPECTRE_V2_IBRS_ENHANCED) {
+               /* If enhanced IBRS is available, enable it on all cpus now */
+               spec_ctrl_flush_all_cpus(MSR_IA32_SPEC_CTRL,
+                       x86_spec_ctrl_base | SPEC_CTRL_FEATURE_ENABLE_IBRS);
+
+       } else if (mode == SPECTRE_V2_RETPOLINE_GENERIC ||
+               mode == SPECTRE_V2_RETPOLINE_AMD) {
                disable_ibrs_and_friends(false /* set not-in-use */);
        }
+
        /* Future CPUs with IBRS_ALL might be able to avoid this. */
        setup_force_cpu_cap(X86_FEATURE_VMEXIT_RSB_FULL);
 

diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c
index b2b3eb247b8a8ced52dd1a1f12203fcc6aa8b26e..0477bfa53a2ec33de199049ed66242f668989a92 100644 (file)
--- a/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@ -149,7 +149,7 @@ void init_scattered_cpuid_features(struct cpuinfo_x86 *c,
                u64 cap;
                rdmsrl(MSR_IA32_ARCH_CAPABILITIES, cap);
                if (cap & 2) /* IBRS all the time */
-                       set_cpu_cap(c, X86_FEATURE_IBRS_ALL);
+                       set_cpu_cap(c, X86_FEATURE_IBRS_ENHANCED);
        }
 
        if (cpu_has(c, X86_FEATURE_IBRS))
@@ -194,11 +194,17 @@ void init_scattered_cpuid_features(struct cpuinfo_x86 *c,
                mutex_lock(&spec_ctrl_mutex);
                if (cpu_has(c, X86_FEATURE_IBRS)) {
                        set_ibrs_supported();
-                       /*
-                        * Don't do this after disable_ibrs_and_friends as we
-                        * would re-enable it (say if spectre_v2=off is used).
-                        */
-                       set_ibrs_firmware();
+                       /* Enable enhanced IBRS usage if available */
+                       if (cpu_has(c, X86_FEATURE_IBRS_ENHANCED)) {
+                               set_ibrs_enhanced();
+                       } else {
+                               /*
+                                * Don't do this after disable_ibrs_and_friends
+                                * as we would re-enable it (say if
+                                * spectre_v2=off is used).
+                                */
+                               set_ibrs_firmware();
+                       }
                }
                if (cpu_has(c, X86_FEATURE_IBPB))
                        set_ibpb_supported();