x86/process: Move the buffer clearing before MONITOR

Move the VERW clearing before the MONITOR so that VERW doesn't disarm it
and the machine never enters C1.

Original idea by Kim Phillips <kim.phillips@amd.com>.

Suggested-by: Andrew Cooper <andrew.cooper3@citrix.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>

diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
index 82bd9eb73b3ca644f47f4609d235814908a7b174..6ca6516c7492e8cbbe2c5e8d7401fc4c5159dca9 100644 (file)
--- a/arch/x86/include/asm/mwait.h
+++ b/arch/x86/include/asm/mwait.h
@@ -43,8 +43,6 @@ static __always_inline void __monitorx(const void *eax, u32 ecx, u32 edx)
 
 static __always_inline void __mwait(u32 eax, u32 ecx)
 {
-       x86_idle_clear_cpu_buffers();
-
        /*
         * Use the instruction mnemonic with implicit operands, as the LLVM
         * assembler fails to assemble the mnemonic with explicit operands:
@@ -98,7 +96,6 @@ static __always_inline void __mwaitx(u32 eax, u32 ebx, u32 ecx)
  */
 static __always_inline void __sti_mwait(u32 eax, u32 ecx)
 {
-       x86_idle_clear_cpu_buffers();
 
        asm volatile("sti; mwait" :: "a" (eax), "c" (ecx));
 }
@@ -115,21 +112,29 @@ static __always_inline void __sti_mwait(u32 eax, u32 ecx)
  */
 static __always_inline void mwait_idle_with_hints(u32 eax, u32 ecx)
 {
+       if (need_resched())
+               return;
+
+       x86_idle_clear_cpu_buffers();
+
        if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) {
                const void *addr = &current_thread_info()->flags;
 
                alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr));
                __monitor(addr, 0, 0);
 
-               if (!need_resched()) {
-                       if (ecx & 1) {
-                               __mwait(eax, ecx);
-                       } else {
-                               __sti_mwait(eax, ecx);
-                               raw_local_irq_disable();
-                       }
+               if (need_resched())
+                       goto out;
+
+               if (ecx & 1) {
+                       __mwait(eax, ecx);
+               } else {
+                       __sti_mwait(eax, ecx);
+                       raw_local_irq_disable();
                }
        }
+
+out:
        current_clr_polling();
 }
 

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 704883c21f3a178dbc5ec24f9a8f4da2077f558e..a838be04f87495a3cd6395dd61fadba3d891a071 100644 (file)
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -907,16 +907,24 @@ static __init bool prefer_mwait_c1_over_halt(void)
  */
 static __cpuidle void mwait_idle(void)
 {
+       if (need_resched())
+               return;
+
+       x86_idle_clear_cpu_buffers();
+
        if (!current_set_polling_and_test()) {
                const void *addr = &current_thread_info()->flags;
 
                alternative_input("", "clflush (%[addr])", X86_BUG_CLFLUSH_MONITOR, [addr] "a" (addr));
                __monitor(addr, 0, 0);
-               if (!need_resched()) {
-                       __sti_mwait(0, 0);
-                       raw_local_irq_disable();
-               }
+               if (need_resched())
+                       goto out;
+
+               __sti_mwait(0, 0);
+               raw_local_irq_disable();
        }
+
+out:
        __current_clr_polling();
 }