perf/x86: Optimize stack walk user accesses

Change the perf user stack walking to use the new
__copy_from_user_nmi(), and split each access into word sized transfer
sizes. This allows to inline the complete access and optimize it all
into a single load.

The main advantage is that this avoids the overhead of double page
faults.  When normal copy_from_user() fails it reexecutes the copy to
compute an accurate number of non copied bytes. This leads to
executing the expensive page fault twice.

While walking stacks having a fault at some point is relatively common
(typically when some part of the program isn't compiled with frame
pointers), so this is a large overhead.

With the optimized copies we avoid this problem because they only do
all accesses once. And of course they're much faster too when the
access does not fault because they're just single instructions instead
of complex function calls.

While profiling a kernel build with -g, the patch brings down the
average time of the PMI handler from 966ns to 552ns (-43%).

Signed-off-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Jiri Olsa <jolsa@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Stephane Eranian <eranian@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: http://lkml.kernel.org/r/1445551641-13379-2-git-send-email-andi@firstfloor.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 2bf79d7c97dfb8848b1e7b060a66ddedb4605a1a..9dfbba5ce6e84c20239bae24786288de3c3aa665 100644 (file)
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -2250,12 +2250,19 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
        ss_base = get_segment_base(regs->ss);
 
        fp = compat_ptr(ss_base + regs->bp);
+       pagefault_disable();
        while (entry->nr < PERF_MAX_STACK_DEPTH) {
                unsigned long bytes;
                frame.next_frame     = 0;
                frame.return_address = 0;
 
-               bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
+               if (!access_ok(VERIFY_READ, fp, 8))
+                       break;
+
+               bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4);
+               if (bytes != 0)
+                       break;
+               bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4);
                if (bytes != 0)
                        break;
 
@@ -2265,6 +2272,7 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
                perf_callchain_store(entry, cs_base + frame.return_address);
                fp = compat_ptr(ss_base + frame.next_frame);
        }
+       pagefault_enable();
        return 1;
 }
 #else
@@ -2302,12 +2310,19 @@ perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
        if (perf_callchain_user32(regs, entry))
                return;
 
+       pagefault_disable();
        while (entry->nr < PERF_MAX_STACK_DEPTH) {
                unsigned long bytes;
                frame.next_frame             = NULL;
                frame.return_address = 0;
 
-               bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
+               if (!access_ok(VERIFY_READ, fp, 16))
+                       break;
+
+               bytes = __copy_from_user_nmi(&frame.next_frame, fp, 8);
+               if (bytes != 0)
+                       break;
+               bytes = __copy_from_user_nmi(&frame.return_address, fp+8, 8);
                if (bytes != 0)
                        break;
 
@@ -2315,8 +2330,9 @@ perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
                        break;
 
                perf_callchain_store(entry, frame.return_address);
-               fp = frame.next_frame;
+               fp = (void __user *)frame.next_frame;
        }
+       pagefault_enable();
 }
 
 /*