x86/entry: Switch the stack after error_entry() returns

error_entry() calls fixup_bad_iret() before sync_regs() if it is a fault
from a bad IRET, to copy pt_regs to the kernel stack. It switches to the
kernel stack directly after sync_regs().

But error_entry() itself is also a function call, so it has to stash
the address it is going to return to, in %r12 which is unnecessarily
complicated.

Move the stack switching after error_entry() and get rid of the need to
handle the return address.

  [ bp: Massage commit message. ]

Signed-off-by: Lai Jiangshan <jiangshan.ljs@antgroup.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Link: https://lore.kernel.org/r/20220503032107.680190-3-jiangshanlai@gmail.com

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ecbfca3cc18ceeda65d6d9e116f6ea8e2acf7278..ca3e99e08a444fad57fd73cc7e1c33a74b3bfbfa 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -326,6 +326,8 @@ SYM_CODE_END(ret_from_fork)
 .macro idtentry_body cfunc has_error_code:req
 
        call    error_entry
+       movq    %rax, %rsp                      /* switch to the task stack if from userspace */
+       ENCODE_FRAME_POINTER
        UNWIND_HINT_REGS
 
        movq    %rsp, %rdi                      /* pt_regs pointer into 1st argument*/
@@ -1002,14 +1004,10 @@ SYM_CODE_START_LOCAL(error_entry)
        /* We have user CR3.  Change to kernel CR3. */
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 
+       leaq    8(%rsp), %rdi                   /* arg0 = pt_regs pointer */
 .Lerror_entry_from_usermode_after_swapgs:
        /* Put us onto the real thread stack. */
-       popq    %r12                            /* save return addr in %12 */
-       movq    %rsp, %rdi                      /* arg0 = pt_regs pointer */
        call    sync_regs
-       movq    %rax, %rsp                      /* switch stack */
-       ENCODE_FRAME_POINTER
-       pushq   %r12
        RET
 
        /*
@@ -1041,6 +1039,7 @@ SYM_CODE_START_LOCAL(error_entry)
         */
 .Lerror_entry_done_lfence:
        FENCE_SWAPGS_KERNEL_ENTRY
+       leaq    8(%rsp), %rax                   /* return pt_regs pointer */
        RET
 
 .Lbstep_iret:
@@ -1061,12 +1060,9 @@ SYM_CODE_START_LOCAL(error_entry)
         * Pretend that the exception came from user mode: set up pt_regs
         * as if we faulted immediately after IRET.
         */
-       popq    %r12                            /* save return addr in %12 */
-       movq    %rsp, %rdi                      /* arg0 = pt_regs pointer */
+       leaq    8(%rsp), %rdi                   /* arg0 = pt_regs pointer */
        call    fixup_bad_iret
-       mov     %rax, %rsp
-       ENCODE_FRAME_POINTER
-       pushq   %r12
+       mov     %rax, %rdi
        jmp     .Lerror_entry_from_usermode_after_swapgs
 SYM_CODE_END(error_entry)