From: Alejandro Jimenez Date: Fri, 26 Oct 2018 16:11:32 +0000 (-0400) Subject: x86/speculation: functions for supporting enhanced IBRS X-Git-Tag: v4.1.12-124.31.3~357 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=9fa5b5f6e8f239d0b668029358b0453c9b5955af;p=users%2Fjedix%2Flinux-maple.git x86/speculation: functions for supporting enhanced IBRS Indirect Branch Restricted Speculation (IBRS) is available either with a basic support (basic IBRS) or with an enhanced support (enhanced IBRS). Currently only basic IBRS is implemented, and it requires IBRS to be set after every transition to a more privileged predictor mode. Enhanced IBRS supports an 'always on' model in which IBRS is enabled once and never disabled. We enhance the existing functions, introduce new identifiers, and rename existing ones in order to be able to differentiate between basic and enhanced IBRS. Signed-off-by: Alejandro Jimenez Co-developed-by: Alexandre Chartre Reviewed-by: Alexandre Chartre (cherry picked from commit d762c3e419e1df1e7671c346ab4247a495fbc3dd from UEK5) Orabug: 28474851 Signed-off-by: Alejandro Jimenez Reviewed-by: Alexandre Chartre Signed-off-by: Brian Maly Conflicts: arch/x86/include/asm/spec_ctrl.h (Differences in the IBRS related macro definitions. Not adding spec_ctrl_flush_all_cpus() that is already in bugs_64.c. Change set_ibrs_inuse() return value from boolean to void.) arch/x86/kernel/cpu/bugs.c (UEK4 uses bugs_64.c. Slight differences in disable_ibrs_and_friends() and cpu_show_common().) arch/x86/kernel/cpu/spec_ctrl.c (No need to remove the spec_ctrl_flush_all_cpus(), it is in bugs_64.c) Signed-off-by: Brian Maly --- diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 6fecb6e76f8b4..ccfd20a9ef190 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -192,6 +192,7 @@ enum spectre_v2_mitigation { SPECTRE_V2_RETPOLINE_GENERIC, SPECTRE_V2_RETPOLINE_AMD, SPECTRE_V2_IBRS, + SPECTRE_V2_IBRS_ENHANCED, }; enum spec_ctrl_set_context { diff --git a/arch/x86/include/asm/spec_ctrl.h b/arch/x86/include/asm/spec_ctrl.h index f3d58700b132d..b55f9db1e3bd2 100644 --- a/arch/x86/include/asm/spec_ctrl.h +++ b/arch/x86/include/asm/spec_ctrl.h @@ -6,10 +6,22 @@ #include #include -#define SPEC_CTRL_IBRS_INUSE (1<<0) /* OS enables IBRS usage */ -#define SPEC_CTRL_IBRS_SUPPORTED (1<<1) /* System supports IBRS */ -#define SPEC_CTRL_IBRS_ADMIN_DISABLED (1<<2) /* Admin disables IBRS */ +/* + * IBRS Flags. + * + * Note that we use dedicated bits to specify if basic IBRS is + * in use (SPEC_CTRL_BASIC_IBRS_INUSE) or enhanced IBRS is in use + * (SPEC_CTRL_ENHCD_IBRS_INUSE), instead of combining multiple + * bits (e.g. SPEC_CTRL_BASIC_IBRS_INUSE | SPEC_CTRL_ENHCD_IBRS_SUPPORTED). + * This is to optimize testing when checking if basic or enhanced + * IBRS is in use, in particular for assembly code. + */ +#define SPEC_CTRL_BASIC_IBRS_INUSE (1<<0) /* OS enables basic IBRS usage */ +#define SPEC_CTRL_IBRS_SUPPORTED (1<<1) /* System supports IBRS (basic or enhanced) */ +#define SPEC_CTRL_IBRS_ADMIN_DISABLED (1<<2) /* Admin disables IBRS (basic and enhanced) */ #define SPEC_CTRL_IBRS_FIRMWARE (1<<3) /* IBRS to be used on firmware paths */ +#define SPEC_CTRL_ENHCD_IBRS_SUPPORTED (1<<4) /* System supports enhanced IBRS */ +#define SPEC_CTRL_ENHCD_IBRS_INUSE (1<<5) /* OS enables enhanced IBRS usage */ #ifdef __ASSEMBLY__ @@ -125,7 +137,7 @@ add $(32*8), %rsp; .macro ENABLE_IBRS - testl $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs) + testl $SPEC_CTRL_BASIC_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs) jz 7f __ASM_ENABLE_IBRS jmp 20f @@ -135,7 +147,7 @@ .endm .macro ENABLE_IBRS_CLOBBER - testl $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs) + testl $SPEC_CTRL_BASIC_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs) jz 11f __ASM_ENABLE_IBRS_CLOBBER jmp 21f @@ -145,7 +157,7 @@ .endm .macro ENABLE_IBRS_SAVE_AND_CLOBBER save_reg:req - testl $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs) + testl $SPEC_CTRL_BASIC_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs) jz 12f movl $MSR_IA32_SPEC_CTRL, %ecx @@ -163,7 +175,7 @@ .endm .macro RESTORE_IBRS_CLOBBER save_reg:req - testl $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs) + testl $SPEC_CTRL_BASIC_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs) jz 13f cmp \save_reg, PER_CPU_VAR(x86_spec_ctrl_priv_cpu) @@ -180,7 +192,7 @@ .endm .macro DISABLE_IBRS - testl $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs) + testl $SPEC_CTRL_BASIC_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs) jz 9f __ASM_DISABLE_IBRS 9: @@ -194,7 +206,7 @@ ALTERNATIVE "", __stringify(__ASM_SET_IBPB), X86_FEATURE_IBRS ALTERNATIVE __stringify(__ASM_STUFF_RSB), "", X86_FEATURE_STUFF_RSB .endm -#else +#else /* __ASSEMBLY__ */ /* Defined in bugs_64.c */ extern u64 x86_spec_ctrl_priv; @@ -226,8 +238,7 @@ DECLARE_STATIC_KEY_FALSE(retpoline_enabled_key); #define ibrs_firmware (use_ibrs & SPEC_CTRL_IBRS_FIRMWARE) #define ibrs_supported (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED) #define ibrs_disabled (use_ibrs & SPEC_CTRL_IBRS_ADMIN_DISABLED) - -#define ibrs_inuse (cpu_ibrs_inuse()) +#define eibrs_supported (use_ibrs & SPEC_CTRL_ENHCD_IBRS_SUPPORTED) static inline void update_cpu_spec_ctrl(int cpu) { @@ -264,26 +275,33 @@ static inline void update_cpu_ibrs_all(void) update_cpu_ibrs(&cpu_data(cpu_index)); } -static inline bool set_ibrs_inuse(void) +static inline void set_ibrs_inuse(void) { - if (ibrs_supported && !ibrs_disabled) { - use_ibrs |= SPEC_CTRL_IBRS_INUSE; - update_cpu_ibrs_all(); - /* Update what sysfs shows. */ - sysctl_ibrs_enabled = true; - /* When entering kernel */ - x86_spec_ctrl_priv |= SPEC_CTRL_FEATURE_ENABLE_IBRS; - /* Update per-cpu spec_ctrl */ - update_cpu_spec_ctrl_all(); - return true; - } else { - return false; - } + if (!ibrs_supported || ibrs_disabled) + return; + + use_ibrs &= ~(SPEC_CTRL_BASIC_IBRS_INUSE | SPEC_CTRL_ENHCD_IBRS_INUSE); + + if (eibrs_supported) + /* Enhanced IBRS is available */ + use_ibrs |= SPEC_CTRL_ENHCD_IBRS_INUSE; + else + /* Basic IBRS is available */ + use_ibrs |= SPEC_CTRL_BASIC_IBRS_INUSE; + + /* Propagate the change to each cpu */ + update_cpu_ibrs_all(); + /* Update what sysfs shows */ + sysctl_ibrs_enabled = true; + /* When entering kernel */ + x86_spec_ctrl_priv |= SPEC_CTRL_FEATURE_ENABLE_IBRS; + /* Update per-cpu spec_ctrl */ + update_cpu_spec_ctrl_all(); } static inline void clear_ibrs_inuse(void) { - use_ibrs &= ~SPEC_CTRL_IBRS_INUSE; + use_ibrs &= ~(SPEC_CTRL_BASIC_IBRS_INUSE | SPEC_CTRL_ENHCD_IBRS_INUSE); update_cpu_ibrs_all(); /* Update what sysfs shows. */ sysctl_ibrs_enabled = false; @@ -295,20 +313,31 @@ static inline void clear_ibrs_inuse(void) update_cpu_spec_ctrl_all(); } +static inline int check_basic_ibrs_inuse(void) +{ + if (use_ibrs & SPEC_CTRL_BASIC_IBRS_INUSE) + return 1; + + /* rmb to prevent wrong speculation for security */ + rmb(); + return 0; +} + static inline int check_ibrs_inuse(void) { - if (use_ibrs & SPEC_CTRL_IBRS_INUSE) { + if (use_ibrs & (SPEC_CTRL_BASIC_IBRS_INUSE | + SPEC_CTRL_ENHCD_IBRS_INUSE)) return 1; - } else { - /* rmb to prevent wrong speculation for security */ - rmb(); - } + + /* rmb to prevent wrong speculation for security */ + rmb(); return 0; } -static inline int cpu_ibrs_inuse(void) +static inline int cpu_ibrs_inuse_any(void) { - return (this_cpu_read(cpu_ibrs) & SPEC_CTRL_IBRS_INUSE) ? 1 : 0; + return (this_cpu_read(cpu_ibrs) & + (SPEC_CTRL_BASIC_IBRS_INUSE | SPEC_CTRL_ENHCD_IBRS_INUSE)) ? 1 : 0; } static inline void set_ibrs_supported(void) @@ -326,7 +355,12 @@ static inline void set_ibrs_disabled(void) static inline void clear_ibrs_disabled(void) { use_ibrs &= ~SPEC_CTRL_IBRS_ADMIN_DISABLED; - (void)set_ibrs_inuse(); + set_ibrs_inuse(); +} + +static inline void set_ibrs_enhanced(void) +{ + use_ibrs |= SPEC_CTRL_ENHCD_IBRS_SUPPORTED; } static inline void set_ibrs_firmware(void) diff --git a/arch/x86/kernel/cpu/bugs_64.c b/arch/x86/kernel/cpu/bugs_64.c index 7f0252f8009b2..08efcb18dc5a9 100644 --- a/arch/x86/kernel/cpu/bugs_64.c +++ b/arch/x86/kernel/cpu/bugs_64.c @@ -27,9 +27,12 @@ /* * use_ibrs flags: - * SPEC_CTRL_IBRS_INUSE indicate if ibrs is currently in use - * SPEC_CTRL_IBRS_SUPPORTED indicate if system supports ibrs - * SPEC_CTRL_IBRS_ADMIN_DISABLED indicate if admin disables ibrs + * SPEC_CTRL_BASIC_IBRS_INUSE basic ibrs is currently in use + * SPEC_CTRL_IBRS_SUPPORTED system supports basic ibrs + * SPEC_CTRL_IBRS_ADMIN_DISABLED admin disables ibrs (basic and enhanced) + * SPEC_CTRL_IBRS_FIRMWARE ibrs to be used on firmware paths + * SPEC_CTRL_ENHCD_IBRS_SUPPORTED system supports enhanced ibrs + * SPEC_CTRL_ENHCD_IBRS_INUSE Enhanced ibrs is currently in use */ int use_ibrs; EXPORT_SYMBOL(use_ibrs); @@ -258,8 +261,8 @@ static const char *spectre_v2_strings[] = { [SPECTRE_V2_RETPOLINE_MINIMAL_AMD] = "Vulnerable: Minimal AMD ASM retpoline", [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline", - [SPECTRE_V2_IBRS] = "Mitigation: IBRS", - + [SPECTRE_V2_IBRS] = "Mitigation: Basic IBRS", + [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS", }; #undef pr_fmt @@ -324,7 +327,7 @@ x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest) * modifiable bits in the host base value and or the * modifiable bits from the guest value. */ - if (ibrs_inuse) + if (cpu_ibrs_inuse_any()) /* * Except on IBRS we don't want to use host base value * but rather the privilege value which has IBRS set. @@ -334,7 +337,7 @@ x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest) guestval = hostval & ~x86_spec_ctrl_mask; guestval |= guest_spec_ctrl & x86_spec_ctrl_mask; - if (ibrs_inuse) { + if (cpu_ibrs_inuse_any()) { /* You may wonder why we don't just jump to the * 'if (hostval ! guestval)' conditional to save an MSR. * (by say the guest MSR value is IBRS and hostval being @@ -409,7 +412,7 @@ void find_retpoline_alternative(void) return; if (allow_retpoline_fallback) { - if (!ibrs_inuse) { + if (!cpu_ibrs_inuse_any()) { /* try to enable ibrs */ if (ibrs_supported) { change_spectre_v2_mitigation(SPECTRE_V2_ENABLE_IBRS); @@ -595,7 +598,8 @@ void refresh_set_spectre_v2_enabled(void) spectre_v2_enabled = test_taint(TAINT_NO_RETPOLINE) ? SPECTRE_V2_NONE : retpoline_mode; } else if (check_ibrs_inuse()) { - spectre_v2_enabled = SPECTRE_V2_IBRS; + spectre_v2_enabled = (check_basic_ibrs_inuse() ? + SPECTRE_V2_IBRS : SPECTRE_V2_IBRS_ENHANCED); } else { spectre_v2_enabled = SPECTRE_V2_NONE; } @@ -671,26 +675,30 @@ disable: static void __init ibrs_select(enum spectre_v2_mitigation *mode) { /* Turn it on (if possible) */ - if (set_ibrs_inuse()) { - *mode = SPECTRE_V2_IBRS; - if (!boot_cpu_has(X86_FEATURE_SMEP)) - setup_force_cpu_cap(X86_FEATURE_STUFF_RSB); - } else + set_ibrs_inuse(); + if (!check_ibrs_inuse()) { pr_info("IBRS could not be enabled.\n"); + return; + } + /* Determine the specific IBRS variant in use */ + *mode = (check_basic_ibrs_inuse() ? + SPECTRE_V2_IBRS : SPECTRE_V2_IBRS_ENHANCED); + + if (boot_cpu_has(X86_FEATURE_SMEP)) + return; + + setup_force_cpu_cap(X86_FEATURE_STUFF_RSB); + + if (*mode == SPECTRE_V2_IBRS_ENHANCED) + pr_warn("Enhanced IBRS might not provide full mitigation against Spectre v2 if SMEP is not available.\n"); } static void __init disable_ibrs_and_friends(bool disable) { - if (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED) { - unsigned int cpu; - - get_online_cpus(); - for_each_online_cpu(cpu) - wrmsrl_on_cpu(cpu, MSR_IA32_SPEC_CTRL, - x86_spec_ctrl_base & ~SPEC_CTRL_FEATURE_ENABLE_IBRS); - - put_online_cpus(); - } + if (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED) + /* Disable IBRS an all cpus */ + spec_ctrl_flush_all_cpus(MSR_IA32_SPEC_CTRL, + x86_spec_ctrl_base & ~SPEC_CTRL_FEATURE_ENABLE_IBRS); if (disable) { set_ibrs_disabled(); @@ -760,7 +768,7 @@ static void __init spectre_v2_select_mitigation(void) mode != SPECTRE_V2_RETPOLINE_MINIMAL_AMD) { pr_info("Options: %s%s%s\n", - ibrs_supported ? "IBRS " : "", + ibrs_supported ? (eibrs_supported ? "IBRS(enhanced) " : "IBRS(basic) ") : "", check_ibpb_inuse() ? "IBPB " : "", retp_compiler() ? "retpoline" : ""); @@ -776,7 +784,8 @@ static void __init spectre_v2_select_mitigation(void) /* Start the engine! */ ibrs_select(&mode); - if (mode == SPECTRE_V2_IBRS) + if (mode == SPECTRE_V2_IBRS || + mode == SPECTRE_V2_IBRS_ENHANCED) goto display; /* But if we can't, then just use retpoline */ }