x86/asm/entry/64: Save user RSP in pt_regs->sp on SYSCALL64 fastpath

Prepare for the removal of 'usersp', by simplifying PER_CPU(old_rsp) usage:

  - use it only as temp storage

  - store the userspace stack pointer immediately in pt_regs->sp
    on syscall entry, instead of using it later, on syscall exit.

  - change C code to use pt_regs->sp only, instead of PER_CPU(old_rsp)
    and task->thread.usersp.

FIXUP/RESTORE_TOP_OF_STACK are simplified as well.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1425926364-9526-4-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h
index 59c6c401f79f16d9b98533275d66d437cbaeff0c..acdee09228b30e020332f0beb7e55e58e65b718e 100644 (file)
--- a/arch/x86/include/asm/compat.h
+++ b/arch/x86/include/asm/compat.h
@@ -301,7 +301,7 @@ static inline void __user *arch_compat_alloc_user_space(long len)
                sp = task_pt_regs(current)->sp;
        } else {
                /* -128 for the x32 ABI redzone */
-               sp = this_cpu_read(old_rsp) - 128;
+               sp = task_pt_regs(current)->sp - 128;
        }
 
        return (void __user *)round_down(sp - len, 16);

diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h
index 4077d963a1a0a561171652c75c69f55f2ce98c01..74bb2e0f3030d1aabc7bed7aee073c15e786b411 100644 (file)
--- a/arch/x86/include/asm/ptrace.h
+++ b/arch/x86/include/asm/ptrace.h
@@ -145,12 +145,8 @@ static inline bool user_64bit_mode(struct pt_regs *regs)
 #endif
 }
 
-#define current_user_stack_pointer()   this_cpu_read(old_rsp)
-/* ia32 vs. x32 difference */
-#define compat_user_stack_pointer()    \
-       (test_thread_flag(TIF_IA32)     \
-        ? current_pt_regs()->sp        \
-        : this_cpu_read(old_rsp))
+#define current_user_stack_pointer()   current_pt_regs()->sp
+#define compat_user_stack_pointer()    current_pt_regs()->sp
 #endif
 
 #ifdef CONFIG_X86_32

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 703ced057199f910267594ad77f3c4ed1ce75993..d86788c3257b84fcd8c792213b7e784cbf1ad2cc 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -128,8 +128,6 @@ ENDPROC(native_usergs_sysret64)
  * manipulation.
  */
        .macro FIXUP_TOP_OF_STACK tmp offset=0
-       movq PER_CPU_VAR(old_rsp),\tmp
-       movq \tmp,RSP+\offset(%rsp)
        movq $__USER_DS,SS+\offset(%rsp)
        movq $__USER_CS,CS+\offset(%rsp)
        movq RIP+\offset(%rsp),\tmp  /* get rip */
@@ -139,8 +137,7 @@ ENDPROC(native_usergs_sysret64)
        .endm
 
        .macro RESTORE_TOP_OF_STACK tmp offset=0
-       movq RSP+\offset(%rsp),\tmp
-       movq \tmp,PER_CPU_VAR(old_rsp)
+       /* nothing to do */
        .endm
 
 /*
@@ -222,9 +219,6 @@ ENDPROC(native_usergs_sysret64)
  * Interrupts are off on entry.
  * Only called from user space.
  *
- * XXX if we had a free scratch register we could save the RSP into the stack frame
- *      and report it properly in ps. Unfortunately we haven't.
- *
  * When user can change the frames always force IRET. That is because
  * it deals with uncanonical addresses better. SYSRET has trouble
  * with them due to bugs in both AMD and Intel CPUs.
@@ -253,11 +247,13 @@ GLOBAL(system_call_after_swapgs)
         */
        ENABLE_INTERRUPTS(CLBR_NONE)
        ALLOC_PT_GPREGS_ON_STACK 8              /* +8: space for orig_ax */
+       movq    %rcx,RIP(%rsp)
+       movq    PER_CPU_VAR(old_rsp),%rcx
+       movq    %r11,EFLAGS(%rsp)
+       movq    %rcx,RSP(%rsp)
+       movq_cfi rax,ORIG_RAX
        SAVE_C_REGS_EXCEPT_RAX_RCX_R11
        movq    $-ENOSYS,RAX(%rsp)
-       movq_cfi rax,ORIG_RAX
-       movq    %r11,EFLAGS(%rsp)
-       movq    %rcx,RIP(%rsp)
        CFI_REL_OFFSET rip,RIP
        testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
        jnz tracesys
@@ -293,7 +289,7 @@ ret_from_sys_call:
        CFI_REGISTER    rip,rcx
        movq    EFLAGS(%rsp),%r11
        /*CFI_REGISTER  rflags,r11*/
-       movq    PER_CPU_VAR(old_rsp), %rsp
+       movq    RSP(%rsp),%rsp
        /*
         * 64bit SYSRET restores rip from rcx,
         * rflags from r11 (but RF and VM bits are forced to 0),

diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c
index 781861cc5ee8d7b9bbd27e9b13c380da59bb06c0..02a8720414c00d00deb1897cb38451b7f55cef19 100644 (file)
--- a/arch/x86/kernel/perf_regs.c
+++ b/arch/x86/kernel/perf_regs.c
@@ -177,7 +177,7 @@ void perf_get_regs_user(struct perf_regs *regs_user,
                 * than just blindly copying user_regs.
                 */
                regs_user->abi = PERF_SAMPLE_REGS_ABI_64;
-               regs_user_copy->sp = this_cpu_read(old_rsp);
+               regs_user_copy->sp = user_regs->sp;
                regs_user_copy->cs = __USER_CS;
                regs_user_copy->ss = __USER_DS;
                regs_user_copy->cx = -1;  /* usually contains garbage */

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 1e393d27d7015f5b67d8d9dce44d09265a27edfe..e8c124a1f8851c359edf5ccfe67a22b5f86c3626 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -602,6 +602,5 @@ long sys_arch_prctl(int code, unsigned long addr)
 
 unsigned long KSTK_ESP(struct task_struct *task)
 {
-       return (test_tsk_thread_flag(task, TIF_IA32)) ?
-                       (task_pt_regs(task)->sp) : ((task)->thread.usersp);
+       return task_pt_regs(task)->sp;
 }