perf/trace: Add ability to set a target task for events

A few events are interesting not only for a current task.
For example, sched_stat_* events are interesting for a task
which wakes up. For this reason, it will be good if such
events will be delivered to a target task too.

Now a target task can be set by using __perf_task().

The original idea and a draft patch belongs to Peter Zijlstra.

I need these events for profiling sleep times. sched_switch is used for
getting callchains and sched_stat_* is used for getting time periods.
These events are combined in user space, then it can be analyzed by
perf tools.

Inspired-by: Peter Zijlstra <peterz@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Arun Sharma <asharma@fb.com>
Signed-off-by: Andrew Vagin <avagin@openvz.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/1342016098-213063-1-git-send-email-avagin@openvz.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/include/linux/ftrace_event.h b/include/linux/ftrace_event.h
index af961d6f7ab14edc6b3a0c8ec813124fba5415da..642928cf57b4b47672c6fd18552b3fd3e114153e 100644 (file)
--- a/include/linux/ftrace_event.h
+++ b/include/linux/ftrace_event.h
@@ -306,9 +306,10 @@ extern void *perf_trace_buf_prepare(int size, unsigned short type,
 
 static inline void
 perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
-                      u64 count, struct pt_regs *regs, void *head)
+                      u64 count, struct pt_regs *regs, void *head,
+                      struct task_struct *task)
 {
-       perf_tp_event(addr, count, raw_data, size, regs, head, rctx);
+       perf_tp_event(addr, count, raw_data, size, regs, head, rctx, task);
 }
 #endif
 

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 76c5c8b724a77253e3ca635c90e53ed6230b3c5f..7602ccb3f40ec672001be2eae9be3395604493f2 100644 (file)
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1272,7 +1272,8 @@ static inline bool perf_paranoid_kernel(void)
 extern void perf_event_init(void);
 extern void perf_tp_event(u64 addr, u64 count, void *record,
                          int entry_size, struct pt_regs *regs,
-                         struct hlist_head *head, int rctx);
+                         struct hlist_head *head, int rctx,
+                         struct task_struct *task);
 extern void perf_bp_event(struct perf_event *event, void *data);
 
 #ifndef perf_misc_flags

diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h
index ea7a2035456d4f913e15b9997696ab5ca4934ad1..5a8671e8a67ff8fa8f715311300901181555d78e 100644 (file)
--- a/include/trace/events/sched.h
+++ b/include/trace/events/sched.h
@@ -73,6 +73,9 @@ DECLARE_EVENT_CLASS(sched_wakeup_template,
                __entry->prio           = p->prio;
                __entry->success        = success;
                __entry->target_cpu     = task_cpu(p);
+       )
+       TP_perf_assign(
+               __perf_task(p);
        ),
 
        TP_printk("comm=%s pid=%d prio=%d success=%d target_cpu=%03d",
@@ -325,6 +328,7 @@ DECLARE_EVENT_CLASS(sched_stat_template,
        )
        TP_perf_assign(
                __perf_count(delay);
+               __perf_task(tsk);
        ),
 
        TP_printk("comm=%s pid=%d delay=%Lu [ns]",

diff --git a/include/trace/ftrace.h b/include/trace/ftrace.h
index c6bc2faaf2611533b1b2a72e6d3e191fa6033616..a763888a36f961339d5a437190c1c944077f875f 100644 (file)
--- a/include/trace/ftrace.h
+++ b/include/trace/ftrace.h
@@ -712,6 +712,9 @@ __attribute__((section("_ftrace_events"))) *__event_##call = &event_##call
 #undef __perf_count
 #define __perf_count(c) __count = (c)
 
+#undef __perf_task
+#define __perf_task(t) __task = (t)
+
 #undef TP_perf_assign
 #define TP_perf_assign(args...) args
 
@@ -725,6 +728,7 @@ perf_trace_##call(void *__data, proto)                                      \
        struct ftrace_raw_##call *entry;                                \
        struct pt_regs __regs;                                          \
        u64 __addr = 0, __count = 1;                                    \
+       struct task_struct *__task = NULL;                              \
        struct hlist_head *head;                                        \
        int __entry_size;                                               \
        int __data_size;                                                \
@@ -752,7 +756,7 @@ perf_trace_##call(void *__data, proto)                                      \
                                                                        \
        head = this_cpu_ptr(event_call->perf_events);                   \
        perf_trace_buf_submit(entry, __entry_size, rctx, __addr,        \
-               __count, &__regs, head);                                \
+               __count, &__regs, head, __task);                        \
 }
 
 /*

diff --git a/kernel/events/callchain.c b/kernel/events/callchain.c
index 6581a040f39926dd46878640413bfd180922415a..98d4597f43d69e35a76535fac7d0bb6821ebdf7f 100644 (file)
--- a/kernel/events/callchain.c
+++ b/kernel/events/callchain.c
@@ -153,7 +153,8 @@ put_callchain_entry(int rctx)
        put_recursion_context(__get_cpu_var(callchain_recursion), rctx);
 }
 
-struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
+struct perf_callchain_entry *
+perf_callchain(struct perf_event *event, struct pt_regs *regs)
 {
        int rctx;
        struct perf_callchain_entry *entry;
@@ -178,6 +179,12 @@ struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
        }
 
        if (regs) {
+               /*
+                * Disallow cross-task user callchains.
+                */
+               if (event->ctx->task && event->ctx->task != current)
+                       goto exit_put;
+
                perf_callchain_store(entry, PERF_CONTEXT_USER);
                perf_callchain_user(entry, regs);
        }

diff --git a/kernel/events/core.c b/kernel/events/core.c
index f1cf0edeb39afa1a3f6543a2f383ac8c4828127b..b7935fcec7d923b0b0b89fe0fe7dfdf61967b447 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -4039,7 +4039,7 @@ void perf_prepare_sample(struct perf_event_header *header,
        if (sample_type & PERF_SAMPLE_CALLCHAIN) {
                int size = 1;
 
-               data->callchain = perf_callchain(regs);
+               data->callchain = perf_callchain(event, regs);
 
                if (data->callchain)
                        size += data->callchain->nr;
@@ -5209,7 +5209,8 @@ static int perf_tp_event_match(struct perf_event *event,
 }
 
 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
-                  struct pt_regs *regs, struct hlist_head *head, int rctx)
+                  struct pt_regs *regs, struct hlist_head *head, int rctx,
+                  struct task_struct *task)
 {
        struct perf_sample_data data;
        struct perf_event *event;
@@ -5228,6 +5229,31 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
                        perf_swevent_event(event, count, &data, regs);
        }
 
+       /*
+        * If we got specified a target task, also iterate its context and
+        * deliver this event there too.
+        */
+       if (task && task != current) {
+               struct perf_event_context *ctx;
+               struct trace_entry *entry = record;
+
+               rcu_read_lock();
+               ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
+               if (!ctx)
+                       goto unlock;
+
+               list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
+                       if (event->attr.type != PERF_TYPE_TRACEPOINT)
+                               continue;
+                       if (event->attr.config != entry->type)
+                               continue;
+                       if (perf_tp_event_match(event, &data, regs))
+                               perf_swevent_event(event, count, &data, regs);
+               }
+unlock:
+               rcu_read_unlock();
+       }
+
        perf_swevent_put_recursion_context(rctx);
 }
 EXPORT_SYMBOL_GPL(perf_tp_event);

diff --git a/kernel/events/internal.h b/kernel/events/internal.h
index b0b107f90afc983a0a7abd4b2f0f9519670f2275..a096c19f2c2a1aba50b62df4b91618cdcdb276d9 100644 (file)
--- a/kernel/events/internal.h
+++ b/kernel/events/internal.h
@@ -101,7 +101,8 @@ __output_copy(struct perf_output_handle *handle,
 }
 
 /* Callchain handling */
-extern struct perf_callchain_entry *perf_callchain(struct pt_regs *regs);
+extern struct perf_callchain_entry *
+perf_callchain(struct perf_event *event, struct pt_regs *regs);
 extern int get_callchain_buffers(void);
 extern void put_callchain_buffers(void);
 

diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c
index fee3752ae8f66f5348199c67b6ad05c5a41346f5..8a6d2ee2086c2cd721dc29ae92d23082370e10db 100644 (file)
--- a/kernel/trace/trace_event_perf.c
+++ b/kernel/trace/trace_event_perf.c
@@ -281,7 +281,7 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip)
 
        head = this_cpu_ptr(event_function.perf_events);
        perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, 0,
-                             1, &regs, head);
+                             1, &regs, head, NULL);
 
 #undef ENTRY_SIZE
 }

diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index b31d3d5699fea09c235ace4c47e16fc8fb8785e0..1a2117043bb140d06b366d8d6e0798bb69fcad26 100644 (file)
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -1002,7 +1002,8 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
        store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
 
        head = this_cpu_ptr(call->perf_events);
-       perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
+       perf_trace_buf_submit(entry, size, rctx,
+                                       entry->ip, 1, regs, head, NULL);
 }
 
 /* Kretprobe profile handler */
@@ -1033,7 +1034,8 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
        store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize);
 
        head = this_cpu_ptr(call->perf_events);
-       perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head);
+       perf_trace_buf_submit(entry, size, rctx,
+                                       entry->ret_ip, 1, regs, head, NULL);
 }
 #endif /* CONFIG_PERF_EVENTS */
 

diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 96fc733690992bf97dc610c87ff2a6b0ecbb97d8..60e4d78756723189b95d11390bc566a6578a4bbe 100644 (file)
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -532,7 +532,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
                               (unsigned long *)&rec->args);
 
        head = this_cpu_ptr(sys_data->enter_event->perf_events);
-       perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
+       perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
 }
 
 int perf_sysenter_enable(struct ftrace_event_call *call)
@@ -608,7 +608,7 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
        rec->ret = syscall_get_return_value(current, regs);
 
        head = this_cpu_ptr(sys_data->exit_event->perf_events);
-       perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head);
+       perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
 }
 
 int perf_sysexit_enable(struct ftrace_event_call *call)

diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index 2b36ac68549e2e23d148c81b7905407d344fdc80..03003cd7dd962d1ca31b2574f9d09f77a74f47a4 100644 (file)
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -670,7 +670,7 @@ static void uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs)
                call_fetch(&tu->args[i].fetch, regs, data + tu->args[i].offset);
 
        head = this_cpu_ptr(call->perf_events);
-       perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head);
+       perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head, NULL);
 
  out:
        preempt_enable();