x86/asm: Use register variable to get stack pointer value

commit 196bd485ee4f03ce4c690bfcf38138abfcd0a4bc upstream.

Currently we use current_stack_pointer() function to get the value
of the stack pointer register. Since commit:

  f5caf621ee35 ("x86/asm: Fix inline asm call constraints for Clang")

... we have a stack register variable declared. It can be used instead of
current_stack_pointer() function which allows to optimize away some
excessive "mov %rsp, %<dst>" instructions:

 -mov    %rsp,%rdx
 -sub    %rdx,%rax
 -cmp    $0x3fff,%rax
 -ja     ffffffff810722fd <ist_begin_non_atomic+0x2d>

 +sub    %rsp,%rax
 +cmp    $0x3fff,%rax
 +ja     ffffffff810722fa <ist_begin_non_atomic+0x2a>

Remove current_stack_pointer(), rename __asm_call_sp to current_stack_pointer
and use it instead of the removed function.

Signed-off-by: Andrey Ryabinin <aryabinin@virtuozzo.com>
Reviewed-by: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20170929141537.29167-1-aryabinin@virtuozzo.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
[dwmw2: We want ASM_CALL_CONSTRAINT for retpoline]
Signed-off-by: David Woodhouse <dwmw@amazon.co.ku>
Signed-off-by: Razvan Ghitulete <rga@amazon.de>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
(cherry picked from commit cfc8c1d61e46fd3c60a34a5b1962eeeb03222a3d)
Orabug: 27477743
CVE: CVE-2017-5715
Signed-off-by: Daniel Jordan <daniel.m.jordan@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Reviewed-by: Mike Kravetz <mike.kravetz@oracle.com>
Reviewed-by: Pavel Tatashin <pasha.tatashin@oracle.com>

diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
index 7730c1c5c83aa7aaf6859170f812ad1f410c1a0b..5a5852de05f9e43d25dafef32487ad8e23cc0bf5 100644 (file)
--- a/arch/x86/include/asm/asm.h
+++ b/arch/x86/include/asm/asm.h
@@ -80,4 +80,15 @@
 /* For C file, we already have NOKPROBE_SYMBOL macro */
 #endif
 
+#ifndef __ASSEMBLY__
+/*
+ * This output constraint should be used for any inline asm which has a "call"
+ * instruction.  Otherwise the asm may be inserted before the frame pointer
+ * gets set up by the containing function.  If you forget to do this, objtool
+ * may print a "call without frame pointer save/setup" warning.
+ */
+register unsigned long current_stack_pointer asm(_ASM_SP);
+#define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer)
+#endif
+
 #endif /* _ASM_X86_ASM_H */

diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 28a9562bbddf2140d14d0e91418c7e01321968a9..9dce035b3ee0b6b06eb564fdc6abf5efe23e84dd 100644 (file)
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -183,17 +183,6 @@ static inline struct thread_info *current_thread_info(void)
        return (struct thread_info *)(current_top_of_stack() - THREAD_SIZE);
 }
 
-static inline unsigned long current_stack_pointer(void)
-{
-       unsigned long sp;
-#ifdef CONFIG_X86_64
-       asm("mov %%rsp,%0" : "=g" (sp));
-#else
-       asm("mov %%esp,%0" : "=g" (sp));
-#endif
-       return sp;
-}
-
 #else /* !__ASSEMBLY__ */
 
 #ifdef CONFIG_X86_64

diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index f9fd86a7fcc7d1bc8c2cc920fc5b5037f5859336..9f4ffc122d9ec557a3a0071403cb45aabf61830d 100644 (file)
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -71,7 +71,7 @@ static void call_on_stack(void *func, void *stack)
 
 static inline void *current_stack(void)
 {
-       return (void *)(current_stack_pointer() & ~(THREAD_SIZE - 1));
+       return (void *)(current_stack_pointer & ~(THREAD_SIZE - 1));
 }
 
 static inline int
@@ -96,7 +96,7 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
 
        /* Save the next esp at the bottom of the stack */
        prev_esp = (u32 *)irqstk;
-       *prev_esp = current_stack_pointer();
+       *prev_esp = current_stack_pointer;
 
        if (unlikely(overflow))
                call_on_stack(print_stack_overflow, isp);
@@ -149,7 +149,7 @@ void do_softirq_own_stack(void)
 
        /* Push the previous esp onto the stack */
        prev_esp = (u32 *)irqstk;
-       *prev_esp = current_stack_pointer();
+       *prev_esp = current_stack_pointer;
 
        call_on_stack(__do_softirq, isp);
 }

diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index b6d11b7d24d2807f9b7fed427c87135218ffbdde..e2089319d0c58e89322c8289b9c94f42327fdd28 100644 (file)
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -175,7 +175,7 @@ void ist_begin_non_atomic(struct pt_regs *regs)
         * from double_fault.
         */
        BUG_ON((unsigned long)(current_top_of_stack() -
-                              current_stack_pointer()) >= THREAD_SIZE);
+                              current_stack_pointer) >= THREAD_SIZE);
 
        preempt_count_sub(HARDIRQ_OFFSET);
 }