x86/speculation: Use static key to enable/disable retpoline

Change the way retpoline is enabled.

Up to now, the retpoline feature was enabled using alternatives
depending whether the X86_FEATURE_RETPOLINE cpu capability was set
or not. The usage of alternatives prevented enabling or disabling
after the boot sequence.

Now, retpoline is enabled using jump labels controlled by the
retpoline_enabled_key static key. This enables retpoline to be
selectively enabled or disabled even after the boot sequence.

Signed-off-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Reviewed-by: Zhenzhong Duan <zhenzhong.duan@oracle.com>
(cherry picked from UEK5 commit 3d01785866e2a73119ed4e35d5cb6b02edf5957a)

Orabug: 28607548

Signed-off-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Reviewed-by: Darren Kenny <darren.kenny@oracle.com>
Reviewed-by: Mihai Carabas <mihai.carabas@oracle.com>
Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Signed-off-by: Brian Maly <brian.maly@oracle.com>

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 117bd5b11bb0e4e1d445a8a0070af179e23c2cd7..7c9c2f337d4e18c30f994e38368a41232958bd57 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -3,6 +3,8 @@
 #ifndef _ASM_X86_NOSPEC_BRANCH_H_
 #define _ASM_X86_NOSPEC_BRANCH_H_
 
+#include <linux/jump_label.h>
+
 #include <asm/alternative.h>
 #include <asm/alternative-asm.h>
 #include <asm/cpufeature.h>
@@ -53,6 +55,8 @@
 
 #ifdef __ASSEMBLY__
 
+.extern retpoline_enabled_key
+
 /*
  * These are the bare retpoline primitives for indirect jmp and call.
  * Do not use these directly; they only exist to make the ALTERNATIVE
@@ -88,6 +92,9 @@
  */
 .macro JMP_NOSPEC reg:req
 #ifdef CONFIG_RETPOLINE
+       STATIC_JUMP_IF_TRUE .Lretpoline_jmp_\@, retpoline_enabled_key, def=0
+       jmp     *\reg
+.Lretpoline_jmp_\@:
        ALTERNATIVE_2 __stringify(jmp *\reg),                           \
                __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
                __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
@@ -98,9 +105,14 @@
 
 .macro CALL_NOSPEC reg:req
 #ifdef CONFIG_RETPOLINE
+       STATIC_JUMP_IF_TRUE .Lretpoline_call_\@, retpoline_enabled_key, def=0
+       call    *\reg
+       jmp     .Ldone_call_\@
+.Lretpoline_call_\@:
        ALTERNATIVE_2 __stringify(call *\reg),                          \
                __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
                __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
+.Ldone_call_\@:
 #else
        call    *\reg
 #endif
@@ -140,8 +152,14 @@
  * otherwise we'll run out of registers. We don't care about CET
  * here, anyway.
  */
-# define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n",    \
-       "       jmp    904f;\n"                                 \
+# define CALL_NOSPEC                                           \
+       "910: .byte " __stringify(STATIC_KEY_INIT_NOP) "\n"     \
+       ".pushsection __jump_table, \"aw\"\n"                   \
+       _ASM_ALIGN "\n"                                         \
+       _ASM_PTR "910b, 904f, retpoline_enabled_key\n"          \
+       ".popsection\n"                                         \
+       "       call *%[thunk_target];\n"                       \
+       "       jmp   905f;\n"                                  \
        "       .align 16\n"                                    \
        "901:   call   903f;\n"                                 \
        "902:   pause;\n"                                       \
@@ -152,8 +170,8 @@
        "       pushl  %[thunk_target];\n"                      \
        "       ret;\n"                                         \
        "       .align 16\n"                                    \
-       "904:   call   901b;\n",                                \
-       X86_FEATURE_RETPOLINE)
+       "904:   call   901b;\n"                                 \
+       "905:"
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #else /* No retpoline for C / inline asm */

diff --git a/arch/x86/include/asm/spec_ctrl.h b/arch/x86/include/asm/spec_ctrl.h
index 1abaf122a6e8218bd65a606f8d0de9ab9c3007d2..3c1f6df899cdc9362e807e85f4aefe8f797e7ee2 100644 (file)
--- a/arch/x86/include/asm/spec_ctrl.h
+++ b/arch/x86/include/asm/spec_ctrl.h
@@ -230,6 +230,8 @@ extern struct mutex spec_ctrl_mutex;
 extern void unprotected_firmware_begin(void);
 extern void unprotected_firmware_end(void);
 
+DECLARE_STATIC_KEY_FALSE(retpoline_enabled_key);
+
 #define ibrs_firmware          (use_ibrs & SPEC_CTRL_IBRS_FIRMWARE)
 #define ibrs_supported         (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED)
 #define ibrs_disabled          (use_ibrs & SPEC_CTRL_IBRS_ADMIN_DISABLED)

diff --git a/arch/x86/kernel/cpu/bugs_64.c b/arch/x86/kernel/cpu/bugs_64.c
index 1355b0249a983860294e5e1a6c57efb6484897bf..2eb01142b2d2d10db4d5afa9bf8532e4e4594647 100644 (file)
--- a/arch/x86/kernel/cpu/bugs_64.c
+++ b/arch/x86/kernel/cpu/bugs_64.c
@@ -63,6 +63,9 @@ int retpoline_fallback = SPEC_CTRL_USE_RETPOLINE_FALLBACK;
 EXPORT_SYMBOL(retpoline_fallback);
 
 
+DEFINE_STATIC_KEY_FALSE(retpoline_enabled_key);
+EXPORT_SYMBOL(retpoline_enabled_key);
+
 int __init spectre_v2_heuristics_setup(char *p)
 {
        ssize_t len;
@@ -583,6 +586,21 @@ static void __init spectre_v2_select_mitigation(void)
                return;
        }
 
+       /*
+        * Set the retpoline capability to advertise that that retpoline
+        * is available, however the retpoline feature is enabled via
+        * the retpoline_enabled_key static key.
+        */
+       if (IS_ENABLED(CONFIG_RETPOLINE)) {
+               setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+               if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
+                       if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
+                               pr_err("Spectre mitigation: LFENCE not serializing, setting up generic retpoline\n");
+                       else
+                               setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
+               }
+       }
+
        switch (cmd) {
        case SPECTRE_V2_CMD_NONE:
                disable_ibrs_and_friends(true);
@@ -617,17 +635,12 @@ static void __init spectre_v2_select_mitigation(void)
        goto out;
 
 retpoline_auto:
-       if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
+       if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
+           boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
        retpoline_amd:
-               if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
-                       pr_err("LFENCE not serializing. Switching to generic retpoline\n");
-                       goto retpoline_generic;
-               }
                mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
                                         SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
                /* On AMD we do not need IBRS, so lets use the ASM mitigation. */
-               setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
-               setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
        } else {
        retpoline_generic:
                mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
@@ -656,8 +669,9 @@ retpoline_auto:
                                /* But if we can't, then just use retpoline */
                        }
                }
-               setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
        }
+       /* Enable retpoline */
+       static_branch_enable(&retpoline_enabled_key);
 display:
        spectre_v2_enabled = mode;
        pr_info("%s\n", spectre_v2_strings[mode]);