From: Alexandre Chartre Date: Thu, 30 Aug 2018 13:51:47 +0000 (+0200) Subject: x86/speculation: Use static key to enable/disable retpoline X-Git-Tag: v4.1.12-124.31.3~464 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=92296f5f9b6ed55d3961727731fafda0f68cb344;p=users%2Fjedix%2Flinux-maple.git x86/speculation: Use static key to enable/disable retpoline Change the way retpoline is enabled. Up to now, the retpoline feature was enabled using alternatives depending whether the X86_FEATURE_RETPOLINE cpu capability was set or not. The usage of alternatives prevented enabling or disabling after the boot sequence. Now, retpoline is enabled using jump labels controlled by the retpoline_enabled_key static key. This enables retpoline to be selectively enabled or disabled even after the boot sequence. Signed-off-by: Alexandre Chartre Reviewed-by: Zhenzhong Duan (cherry picked from UEK5 commit 3d01785866e2a73119ed4e35d5cb6b02edf5957a) Orabug: 28607548 Signed-off-by: Alexandre Chartre Reviewed-by: Darren Kenny Reviewed-by: Mihai Carabas Reviewed-by: Boris Ostrovsky Signed-off-by: Brian Maly --- diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 117bd5b11bb0..7c9c2f337d4e 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -3,6 +3,8 @@ #ifndef _ASM_X86_NOSPEC_BRANCH_H_ #define _ASM_X86_NOSPEC_BRANCH_H_ +#include + #include #include #include @@ -53,6 +55,8 @@ #ifdef __ASSEMBLY__ +.extern retpoline_enabled_key + /* * These are the bare retpoline primitives for indirect jmp and call. * Do not use these directly; they only exist to make the ALTERNATIVE @@ -88,6 +92,9 @@ */ .macro JMP_NOSPEC reg:req #ifdef CONFIG_RETPOLINE + STATIC_JUMP_IF_TRUE .Lretpoline_jmp_\@, retpoline_enabled_key, def=0 + jmp *\reg +.Lretpoline_jmp_\@: ALTERNATIVE_2 __stringify(jmp *\reg), \ __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD @@ -98,9 +105,14 @@ .macro CALL_NOSPEC reg:req #ifdef CONFIG_RETPOLINE + STATIC_JUMP_IF_TRUE .Lretpoline_call_\@, retpoline_enabled_key, def=0 + call *\reg + jmp .Ldone_call_\@ +.Lretpoline_call_\@: ALTERNATIVE_2 __stringify(call *\reg), \ __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD +.Ldone_call_\@: #else call *\reg #endif @@ -140,8 +152,14 @@ * otherwise we'll run out of registers. We don't care about CET * here, anyway. */ -# define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \ - " jmp 904f;\n" \ +# define CALL_NOSPEC \ + "910: .byte " __stringify(STATIC_KEY_INIT_NOP) "\n" \ + ".pushsection __jump_table, \"aw\"\n" \ + _ASM_ALIGN "\n" \ + _ASM_PTR "910b, 904f, retpoline_enabled_key\n" \ + ".popsection\n" \ + " call *%[thunk_target];\n" \ + " jmp 905f;\n" \ " .align 16\n" \ "901: call 903f;\n" \ "902: pause;\n" \ @@ -152,8 +170,8 @@ " pushl %[thunk_target];\n" \ " ret;\n" \ " .align 16\n" \ - "904: call 901b;\n", \ - X86_FEATURE_RETPOLINE) + "904: call 901b;\n" \ + "905:" # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) #else /* No retpoline for C / inline asm */ diff --git a/arch/x86/include/asm/spec_ctrl.h b/arch/x86/include/asm/spec_ctrl.h index 1abaf122a6e8..3c1f6df899cd 100644 --- a/arch/x86/include/asm/spec_ctrl.h +++ b/arch/x86/include/asm/spec_ctrl.h @@ -230,6 +230,8 @@ extern struct mutex spec_ctrl_mutex; extern void unprotected_firmware_begin(void); extern void unprotected_firmware_end(void); +DECLARE_STATIC_KEY_FALSE(retpoline_enabled_key); + #define ibrs_firmware (use_ibrs & SPEC_CTRL_IBRS_FIRMWARE) #define ibrs_supported (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED) #define ibrs_disabled (use_ibrs & SPEC_CTRL_IBRS_ADMIN_DISABLED) diff --git a/arch/x86/kernel/cpu/bugs_64.c b/arch/x86/kernel/cpu/bugs_64.c index 1355b0249a98..2eb01142b2d2 100644 --- a/arch/x86/kernel/cpu/bugs_64.c +++ b/arch/x86/kernel/cpu/bugs_64.c @@ -63,6 +63,9 @@ int retpoline_fallback = SPEC_CTRL_USE_RETPOLINE_FALLBACK; EXPORT_SYMBOL(retpoline_fallback); +DEFINE_STATIC_KEY_FALSE(retpoline_enabled_key); +EXPORT_SYMBOL(retpoline_enabled_key); + int __init spectre_v2_heuristics_setup(char *p) { ssize_t len; @@ -583,6 +586,21 @@ static void __init spectre_v2_select_mitigation(void) return; } + /* + * Set the retpoline capability to advertise that that retpoline + * is available, however the retpoline feature is enabled via + * the retpoline_enabled_key static key. + */ + if (IS_ENABLED(CONFIG_RETPOLINE)) { + setup_force_cpu_cap(X86_FEATURE_RETPOLINE); + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) + pr_err("Spectre mitigation: LFENCE not serializing, setting up generic retpoline\n"); + else + setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); + } + } + switch (cmd) { case SPECTRE_V2_CMD_NONE: disable_ibrs_and_friends(true); @@ -617,17 +635,12 @@ static void __init spectre_v2_select_mitigation(void) goto out; retpoline_auto: - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && + boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { retpoline_amd: - if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { - pr_err("LFENCE not serializing. Switching to generic retpoline\n"); - goto retpoline_generic; - } mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD : SPECTRE_V2_RETPOLINE_MINIMAL_AMD; /* On AMD we do not need IBRS, so lets use the ASM mitigation. */ - setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); - setup_force_cpu_cap(X86_FEATURE_RETPOLINE); } else { retpoline_generic: mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC : @@ -656,8 +669,9 @@ retpoline_auto: /* But if we can't, then just use retpoline */ } } - setup_force_cpu_cap(X86_FEATURE_RETPOLINE); } + /* Enable retpoline */ + static_branch_enable(&retpoline_enabled_key); display: spectre_v2_enabled = mode; pr_info("%s\n", spectre_v2_strings[mode]);