x86/mce: Use arch atomic and bit helpers

The arch helpers do not have explicit KASAN instrumentation. Use them in
noinstr code.

Inline a couple more functions with single call sites, while at it:

mce_severity_amd_smca() has a single call-site which is noinstr so force
the inlining and fix:

  vmlinux.o: warning: objtool: mce_severity_amd.constprop.0()+0xca: call to \
	  mce_severity_amd_smca() leaves .noinstr.text section

Always inline mca_msr_reg():

     text    data     bss     dec     hex filename
  16065240        128031326       36405368        180501934       ac23dae vmlinux.before
  16065240        128031294       36405368        180501902       ac23d8e vmlinux.after

and mce_no_way_out() as the latter one is used only once, to fix:

  vmlinux.o: warning: objtool: mce_read_aux()+0x53: call to mca_msr_reg() leaves .noinstr.text section
  vmlinux.o: warning: objtool: do_machine_check()+0xc9: call to mce_no_way_out() leaves .noinstr.text section

Signed-off-by: Borislav Petkov <bp@suse.de>
Acked-by: Marco Elver <elver@google.com>
Link: https://lore.kernel.org/r/20220204083015.17317-4-bp@alien8.de

diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index 5818b837fd4d4cc1512f04aec19f8942c3b71040..0e7147430ec0df842c2504cc9a934a07efcd86e7 100644 (file)
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -173,27 +173,6 @@ void mce_unregister_decode_chain(struct notifier_block *nb)
 }
 EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);
 
-u32 mca_msr_reg(int bank, enum mca_msr reg)
-{
-       if (mce_flags.smca) {
-               switch (reg) {
-               case MCA_CTL:    return MSR_AMD64_SMCA_MCx_CTL(bank);
-               case MCA_ADDR:   return MSR_AMD64_SMCA_MCx_ADDR(bank);
-               case MCA_MISC:   return MSR_AMD64_SMCA_MCx_MISC(bank);
-               case MCA_STATUS: return MSR_AMD64_SMCA_MCx_STATUS(bank);
-               }
-       }
-
-       switch (reg) {
-       case MCA_CTL:    return MSR_IA32_MCx_CTL(bank);
-       case MCA_ADDR:   return MSR_IA32_MCx_ADDR(bank);
-       case MCA_MISC:   return MSR_IA32_MCx_MISC(bank);
-       case MCA_STATUS: return MSR_IA32_MCx_STATUS(bank);
-       }
-
-       return 0;
-}
-
 static void __print_mce(struct mce *m)
 {
        pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n",
@@ -814,7 +793,8 @@ EXPORT_SYMBOL_GPL(machine_check_poll);
  * the severity assessment code. Pretend that EIPV was set, and take the
  * ip/cs values from the pt_regs that mce_gather_info() ignored earlier.
  */
-static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
+static __always_inline void
+quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
 {
        if (bank != 0)
                return;
@@ -838,8 +818,8 @@ static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
  * Do a quick check if any of the events requires a panic.
  * This decides if we keep the events around or clear them.
  */
-static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
-                         struct pt_regs *regs)
+static __always_inline int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
+                                         struct pt_regs *regs)
 {
        char *tmp = *msg;
        int i;
@@ -849,7 +829,7 @@ static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
                if (!(m->status & MCI_STATUS_VAL))
                        continue;
 
-               __set_bit(i, validp);
+               arch___set_bit(i, validp);
                if (mce_flags.snb_ifu_quirk)
                        quirk_sandybridge_ifu(i, m, regs);
 
@@ -1015,13 +995,13 @@ static noinstr int mce_start(int *no_way_out)
        if (!timeout)
                return ret;
 
-       atomic_add(*no_way_out, &global_nwo);
+       arch_atomic_add(*no_way_out, &global_nwo);
        /*
         * Rely on the implied barrier below, such that global_nwo
         * is updated before mce_callin.
         */
-       order = atomic_inc_return(&mce_callin);
-       cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus);
+       order = arch_atomic_inc_return(&mce_callin);
+       arch_cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus);
 
        /* Enable instrumentation around calls to external facilities */
        instrumentation_begin();
@@ -1029,10 +1009,10 @@ static noinstr int mce_start(int *no_way_out)
        /*
         * Wait for everyone.
         */
-       while (atomic_read(&mce_callin) != num_online_cpus()) {
+       while (arch_atomic_read(&mce_callin) != num_online_cpus()) {
                if (mce_timed_out(&timeout,
                                  "Timeout: Not all CPUs entered broadcast exception handler")) {
-                       atomic_set(&global_nwo, 0);
+                       arch_atomic_set(&global_nwo, 0);
                        goto out;
                }
                ndelay(SPINUNIT);
@@ -1047,7 +1027,7 @@ static noinstr int mce_start(int *no_way_out)
                /*
                 * Monarch: Starts executing now, the others wait.
                 */
-               atomic_set(&mce_executing, 1);
+               arch_atomic_set(&mce_executing, 1);
        } else {
                /*
                 * Subject: Now start the scanning loop one by one in
@@ -1055,10 +1035,10 @@ static noinstr int mce_start(int *no_way_out)
                 * This way when there are any shared banks it will be
                 * only seen by one CPU before cleared, avoiding duplicates.
                 */
-               while (atomic_read(&mce_executing) < order) {
+               while (arch_atomic_read(&mce_executing) < order) {
                        if (mce_timed_out(&timeout,
                                          "Timeout: Subject CPUs unable to finish machine check processing")) {
-                               atomic_set(&global_nwo, 0);
+                               arch_atomic_set(&global_nwo, 0);
                                goto out;
                        }
                        ndelay(SPINUNIT);
@@ -1068,7 +1048,7 @@ static noinstr int mce_start(int *no_way_out)
        /*
         * Cache the global no_way_out state.
         */
-       *no_way_out = atomic_read(&global_nwo);
+       *no_way_out = arch_atomic_read(&global_nwo);
 
        ret = order;
 
@@ -1153,12 +1133,12 @@ out:
        return ret;
 }
 
-static void mce_clear_state(unsigned long *toclear)
+static __always_inline void mce_clear_state(unsigned long *toclear)
 {
        int i;
 
        for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
-               if (test_bit(i, toclear))
+               if (arch_test_bit(i, toclear))
                        mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0);
        }
 }
@@ -1208,8 +1188,8 @@ __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final,
        int severity, i, taint = 0;
 
        for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
-               __clear_bit(i, toclear);
-               if (!test_bit(i, valid_banks))
+               arch___clear_bit(i, toclear);
+               if (!arch_test_bit(i, valid_banks))
                        continue;
 
                if (!mce_banks[i].ctl)
@@ -1244,7 +1224,7 @@ __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final,
                     severity == MCE_UCNA_SEVERITY) && !no_way_out)
                        continue;
 
-               __set_bit(i, toclear);
+               arch___set_bit(i, toclear);
 
                /* Machine check event was not enabled. Clear, but ignore. */
                if (severity == MCE_NO_SEVERITY)

diff --git a/arch/x86/kernel/cpu/mce/internal.h b/arch/x86/kernel/cpu/mce/internal.h
index 52c633950b38dbd8d82cc041fcc5d46a865a719f..a04b61e278278ad646d1f43438dd04a5562dff1c 100644 (file)
--- a/arch/x86/kernel/cpu/mce/internal.h
+++ b/arch/x86/kernel/cpu/mce/internal.h
@@ -182,8 +182,6 @@ enum mca_msr {
        MCA_MISC,
 };
 
-u32 mca_msr_reg(int bank, enum mca_msr reg);
-
 /* Decide whether to add MCE record to MCE event pool or filter it out. */
 extern bool filter_mce(struct mce *m);
 
@@ -209,4 +207,25 @@ static inline void winchip_machine_check(struct pt_regs *regs) {}
 
 noinstr u64 mce_rdmsrl(u32 msr);
 
+static __always_inline u32 mca_msr_reg(int bank, enum mca_msr reg)
+{
+       if (mce_flags.smca) {
+               switch (reg) {
+               case MCA_CTL:    return MSR_AMD64_SMCA_MCx_CTL(bank);
+               case MCA_ADDR:   return MSR_AMD64_SMCA_MCx_ADDR(bank);
+               case MCA_MISC:   return MSR_AMD64_SMCA_MCx_MISC(bank);
+               case MCA_STATUS: return MSR_AMD64_SMCA_MCx_STATUS(bank);
+               }
+       }
+
+       switch (reg) {
+       case MCA_CTL:    return MSR_IA32_MCx_CTL(bank);
+       case MCA_ADDR:   return MSR_IA32_MCx_ADDR(bank);
+       case MCA_MISC:   return MSR_IA32_MCx_MISC(bank);
+       case MCA_STATUS: return MSR_IA32_MCx_STATUS(bank);
+       }
+
+       return 0;
+}
+
 #endif /* __X86_MCE_INTERNAL_H__ */

diff --git a/arch/x86/kernel/cpu/mce/severity.c b/arch/x86/kernel/cpu/mce/severity.c
index 7aa2bda93cbb6c97bd95eecaeed9cf2e0fad8ac0..ca0d775346fbfbede9fa1ffc76c65e9632a36e6e 100644 (file)
--- a/arch/x86/kernel/cpu/mce/severity.c
+++ b/arch/x86/kernel/cpu/mce/severity.c
@@ -301,7 +301,7 @@ static noinstr int error_context(struct mce *m, struct pt_regs *regs)
        }
 }
 
-static int mce_severity_amd_smca(struct mce *m, enum context err_ctx)
+static __always_inline int mce_severity_amd_smca(struct mce *m, enum context err_ctx)
 {
        u64 mcx_cfg;