x86/entry: Split and inline prepare_exit_to_usermode()

GCC is unable to properly optimize functions that have a very
short likely case and a longer and register-heavier cold part --
it fails to sink all of the register saving and stack frame
setup code into the unlikely part.

Help it out with prepare_exit_to_usermode() by splitting it into
two parts and inline the hot part.

Saves 6-8 cycles for compat syscalls.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Link: http://lkml.kernel.org/r/9fc53eda4a5b924070952f12fa4ae3e477640a07.1444091585.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index d0874210d5b5ef306f7c251cb8abb3c48b99f147..66ccbd664d4c2d44c8b2459164ec17fe362d872e 100644 (file)
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -218,14 +218,12 @@ long syscall_trace_enter(struct pt_regs *regs)
                return syscall_trace_enter_phase2(regs, arch, phase1_result);
 }
 
-/* Called with IRQs disabled. */
-__visible void prepare_exit_to_usermode(struct pt_regs *regs)
-{
-       if (IS_ENABLED(CONFIG_PROVE_LOCKING) && WARN_ON(!irqs_disabled()))
-               local_irq_disable();
-
-       lockdep_sys_exit();
+#define EXIT_TO_USERMODE_LOOP_FLAGS                            \
+       (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE |   \
+        _TIF_NEED_RESCHED | _TIF_USER_RETURN_NOTIFY)
 
+static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
+{
        /*
         * In order to return to user mode, we need to have IRQs off with
         * none of _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_USER_RETURN_NOTIFY,
@@ -235,14 +233,6 @@ __visible void prepare_exit_to_usermode(struct pt_regs *regs)
         * work to clear some of the flags can sleep.
         */
        while (true) {
-               u32 cached_flags =
-                       READ_ONCE(pt_regs_to_thread_info(regs)->flags);
-
-               if (!(cached_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME |
-                                     _TIF_UPROBE | _TIF_NEED_RESCHED |
-                                     _TIF_USER_RETURN_NOTIFY)))
-                       break;
-
                /* We have work to do. */
                local_irq_enable();
 
@@ -266,7 +256,30 @@ __visible void prepare_exit_to_usermode(struct pt_regs *regs)
 
                /* Disable IRQs and retry */
                local_irq_disable();
+
+               cached_flags = READ_ONCE(pt_regs_to_thread_info(regs)->flags);
+
+               if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
+                       break;
+
        }
+}
+
+/* Called with IRQs disabled. */
+__visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
+{
+       u32 cached_flags;
+
+       if (IS_ENABLED(CONFIG_PROVE_LOCKING) && WARN_ON(!irqs_disabled()))
+               local_irq_disable();
+
+       lockdep_sys_exit();
+
+       cached_flags =
+               READ_ONCE(pt_regs_to_thread_info(regs)->flags);
+
+       if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
+               exit_to_usermode_loop(regs, cached_flags);
 
        user_enter();
 }