x86/speculation: Implement per-cpu IBRS control

If a system is booted with non-IBRS microcode, starts a microcode update
which enables IBRS, flags IBRS as supported, then an NMI is taken on a CPU
that hasn't actually received the update, we get a #GP since the SPEC_CTRL
MSR will get frobbed on the unsupported CPU anyway.

IBRS usage is now defined globally as well as per-cpu. The boot cpu defines
the initial IBRS usage, then each cpu defines its own per-cpu IBRS usage
based on the global IBRS uage and its cpu capabilities.

Signed-off-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Reviewed-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
(cherry picked from UEK5 commit fabdd62357acce2531c9e16c5510e04862eab52d
 and part of UEK5 commit 5055b50f67b59ec07c64c833cb351f088486cb02)

[Backport: backport includes part of UEK5 commit 5055b50f67b59e
 ("x86/topology: Avoid wasting 128k for package id array") to track when
 cpu data are initialized.]

Orabug: 28064081

Signed-off-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Reviewed-by: Darren Kenny <darren.kenny@oracle.com>
Signed-off-by: Brian Maly <brian.maly@oracle.com>

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index ace5387b67781e15ba2b182598c27a41bc98a8b3..70fe9446390a5299284e0fc955c9f86358887b7c 100644 (file)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -29,6 +29,7 @@ struct mm_struct;
 #include <linux/math64.h>
 #include <linux/err.h>
 #include <linux/irqflags.h>
+#include <linux/uek_kabi.h>
 
 /*
  * We handle most unaligned accesses in hardware.  On the other hand
@@ -130,6 +131,7 @@ struct cpuinfo_x86 {
        /* Index into per_cpu list: */
        u16                     cpu_index;
        u32                     microcode;
+       UEK_KABI_EXTEND(unsigned initialized : 1)
 };
 
 #define X86_VENDOR_INTEL       0

diff --git a/arch/x86/include/asm/spec_ctrl.h b/arch/x86/include/asm/spec_ctrl.h
index 76cc5155c10d1d22dd918bbc0fac4c7b8ae71e19..656cca673a547e7a984f5d51a41d0ec0e7ec4772 100644 (file)
--- a/arch/x86/include/asm/spec_ctrl.h
+++ b/arch/x86/include/asm/spec_ctrl.h
@@ -131,7 +131,7 @@
        add $(32*8), %rsp;
 
 .macro ENABLE_IBRS
-       testl   $SPEC_CTRL_IBRS_INUSE, use_ibrs
+       testl   $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
        jz      7f
        __ASM_ENABLE_IBRS
        jmp     20f
@@ -141,7 +141,7 @@
 .endm
 
 .macro ENABLE_IBRS_CLOBBER
-       testl   $SPEC_CTRL_IBRS_INUSE, use_ibrs
+       testl   $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
        jz      11f
        __ASM_ENABLE_IBRS_CLOBBER
        jmp     21f
@@ -151,7 +151,7 @@
 .endm
 
 .macro ENABLE_IBRS_SAVE_AND_CLOBBER save_reg:req
-       testl   $SPEC_CTRL_IBRS_INUSE, use_ibrs
+       testl   $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
        jz      12f
 
        movl    $MSR_IA32_SPEC_CTRL, %ecx
@@ -169,7 +169,7 @@
 .endm
 
 .macro RESTORE_IBRS_CLOBBER save_reg:req
-       testl   $SPEC_CTRL_IBRS_INUSE, use_ibrs
+       testl   $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
        jz      13f
 
        testl   $SPEC_CTRL_FEATURE_ENABLE_IBRS, \save_reg
@@ -186,7 +186,7 @@
 .endm
 
 .macro DISABLE_IBRS
-       testl   $SPEC_CTRL_IBRS_INUSE, use_ibrs
+       testl   $SPEC_CTRL_IBRS_INUSE, PER_CPU_VAR(cpu_ibrs)
        jz      9f
        __ASM_DISABLE_IBRS
 9:
@@ -210,8 +210,19 @@ ALTERNATIVE __stringify(__ASM_STUFF_RSB), "", X86_FEATURE_STUFF_RSB
 extern u64 x86_spec_ctrl_priv;
 extern u64 x86_spec_ctrl_base;
 
-/* indicate usage of IBRS to control execution speculation */
+/*
+ * Indicate usage of IBRS to control execution speculation.
+ *
+ * IBRS usage is defined globally with the use_ibrs variable, and
+ * per-cpu with the per-cpu variable cpu_ibrs. During the boot,
+ * the boot cpu will set the initial value of use_ibrs and the
+ * per-cpu value of all online cpus which support IBRS. If, after
+ * that, a cpu comes online or has its microcode updated, it will
+ * set its own per-cpu value based on the value of use_ibrs and
+ * the IBRS capability of the cpu.
+ */
 extern int use_ibrs;
+DECLARE_PER_CPU(unsigned int, cpu_ibrs);
 extern u32 sysctl_ibrs_enabled;
 extern struct mutex spec_ctrl_mutex;
 
@@ -222,12 +233,34 @@ extern void unprotected_firmware_end(void);
 #define ibrs_supported         (use_ibrs & SPEC_CTRL_IBRS_SUPPORTED)
 #define ibrs_disabled          (use_ibrs & SPEC_CTRL_IBRS_ADMIN_DISABLED)
 
-#define ibrs_inuse             (check_ibrs_inuse())
+#define ibrs_inuse             (cpu_ibrs_inuse())
+
+static inline void update_cpu_ibrs(struct cpuinfo_x86 *cpu)
+{
+       struct cpuinfo_x86 *cpu_info;
+
+       /*
+        * IBRS can be set at boot time while cpu capabilities
+        * haven't been copied from boot_cpu_data yet.
+        */
+       cpu_info = (cpu->initialized) ? cpu : &boot_cpu_data;
+       per_cpu(cpu_ibrs, cpu->cpu_index) =
+           cpu_has(cpu_info, X86_FEATURE_IBRS) ? use_ibrs : 0;
+}
+
+static inline void update_cpu_ibrs_all(void)
+{
+       int cpu_index;
+
+       for_each_online_cpu(cpu_index)
+               update_cpu_ibrs(&cpu_data(cpu_index));
+}
 
 static inline bool set_ibrs_inuse(void)
 {
        if (ibrs_supported && !ibrs_disabled) {
                use_ibrs |= SPEC_CTRL_IBRS_INUSE;
+               update_cpu_ibrs_all();
                /* Update what sysfs shows. */
                sysctl_ibrs_enabled = true;
                /* When entering kernel */
@@ -241,6 +274,7 @@ static inline bool set_ibrs_inuse(void)
 static inline void clear_ibrs_inuse(void)
 {
        use_ibrs &= ~SPEC_CTRL_IBRS_INUSE;
+       update_cpu_ibrs_all();
        /* Update what sysfs shows. */
        sysctl_ibrs_enabled = false;
        /*
@@ -262,6 +296,11 @@ static inline int check_ibrs_inuse(void)
        return 0;
 }
 
+static inline int cpu_ibrs_inuse(void)
+{
+       return (this_cpu_read(cpu_ibrs) & SPEC_CTRL_IBRS_INUSE) ? 1 : 0;
+}
+
 static inline void set_ibrs_supported(void)
 {
        use_ibrs |= SPEC_CTRL_IBRS_SUPPORTED;

diff --git a/arch/x86/kernel/cpu/bugs_64.c b/arch/x86/kernel/cpu/bugs_64.c
index a5c9d000e22035beb0edde68ddcca2fed47a1f93..632fd6d5808b5d0f452e2a0149323e684443dca4 100644 (file)
--- a/arch/x86/kernel/cpu/bugs_64.c
+++ b/arch/x86/kernel/cpu/bugs_64.c
@@ -31,6 +31,9 @@
 int use_ibrs;
 EXPORT_SYMBOL(use_ibrs);
 
+DEFINE_PER_CPU(unsigned int, cpu_ibrs) = 0;
+EXPORT_PER_CPU_SYMBOL(cpu_ibrs);
+
 /*
  * use_ibpb flags:
  * SPEC_CTRL_IBPB_INUSE                        indicate if ibpb is currently in use
@@ -983,7 +986,7 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr
 
        case X86_BUG_SPECTRE_V2:
                return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
-                              ibrs_inuse && retpoline_enabled() ? ", IBRS" : "",
+                              (check_ibrs_inuse() && retpoline_enabled()) ? ", IBRS" : "",
                               ibrs_firmware ? ", IBRS_FW" : "",
                               ibpb_inuse ? ", IBPB" : "");
 

diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c
index 996d1c2e73a98dbb1054ded4dbbede8a65dce958..e524417b72607c36e304aa475bd0fc242d270dcd 100644 (file)
--- a/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@ -201,4 +201,10 @@ void init_scattered_cpuid_features(struct cpuinfo_x86 *c,
                        set_ibpb_supported();
                mutex_unlock(&spec_ctrl_mutex);
        }
+
+       if (!xen_pv_domain()) {
+               mutex_lock(&spec_ctrl_mutex);
+               update_cpu_ibrs(c);
+               mutex_unlock(&spec_ctrl_mutex);
+       }
 }

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index fd3e1b0537738ff62e62ea229e2d66ae948cbafe..05520fb819f6e2a06446864c31e3250732209181 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -277,6 +277,7 @@ void __init smp_store_boot_cpu_info(void)
 
        *c = boot_cpu_data;
        c->cpu_index = id;
+       c->initialized = true;
 }
 
 /*
@@ -287,13 +288,16 @@ void smp_store_cpu_info(int id)
 {
        struct cpuinfo_x86 *c = &cpu_data(id);
 
-       *c = boot_cpu_data;
+       /* Copy boot_cpu_data only on the first bringup */
+       if (!c->initialized)
+               *c = boot_cpu_data;
        c->cpu_index = id;
        /*
         * During boot time, CPU0 has this setup already. Save the info when
         * bringing up AP or offlined CPU0.
         */
        identify_secondary_cpu(c);
+       c->initialized = true;
 }
 
 static bool