perf_counter: Change data head from u32 to u64

Since some people worried that 4G might not be a large enough
as an mmap data window, extend it to 64 bit for capable
platforms.

Reported-by: Stephane Eranian <eranian@googlemail.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/include/linux/perf_counter.h b/include/linux/perf_counter.h
index cef9931793fd0d8783d90cd8338e85fea4a90acf..c046f7d97cfafce277520f591834fbef605dc900 100644 (file)
--- a/include/linux/perf_counter.h
+++ b/include/linux/perf_counter.h
@@ -212,7 +212,7 @@ struct perf_counter_mmap_page {
         * User-space reading this value should issue an rmb(), on SMP capable
         * platforms, after reading this value -- see perf_counter_wakeup().
         */
-       __u32   data_head;              /* head in the data section */
+       __u64   data_head;              /* head in the data section */
 };
 
 #define PERF_EVENT_MISC_CPUMODE_MASK   (3 << 0)
@@ -397,10 +397,11 @@ struct perf_mmap_data {
        int                             nr_locked;      /* nr pages mlocked  */
 
        atomic_t                        poll;           /* POLL_ for wakeups */
-       atomic_t                        head;           /* write position    */
        atomic_t                        events;         /* event limit       */
 
-       atomic_t                        done_head;      /* completed head    */
+       atomic_long_t                   head;           /* write position    */
+       atomic_long_t                   done_head;      /* completed head    */
+
        atomic_t                        lock;           /* concurrent writes */
 
        atomic_t                        wakeup;         /* needs a wakeup    */

diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
index 5ecd9981c03579f1d7e283e391b7e620fa002676..3f11a2bc6c799268ba01fb20538e0fffc292ff00 100644 (file)
--- a/kernel/perf_counter.c
+++ b/kernel/perf_counter.c
@@ -2067,8 +2067,8 @@ __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
 struct perf_output_handle {
        struct perf_counter     *counter;
        struct perf_mmap_data   *data;
-       unsigned int            offset;
-       unsigned int            head;
+       unsigned long           head;
+       unsigned long           offset;
        int                     nmi;
        int                     overflow;
        int                     locked;
@@ -2122,7 +2122,8 @@ static void perf_output_lock(struct perf_output_handle *handle)
 static void perf_output_unlock(struct perf_output_handle *handle)
 {
        struct perf_mmap_data *data = handle->data;
-       int head, cpu;
+       unsigned long head;
+       int cpu;
 
        data->done_head = data->head;
 
@@ -2135,7 +2136,7 @@ again:
         * before we publish the new head, matched by a rmb() in userspace when
         * reading this position.
         */
-       while ((head = atomic_xchg(&data->done_head, 0)))
+       while ((head = atomic_long_xchg(&data->done_head, 0)))
                data->user_page->data_head = head;
 
        /*
@@ -2148,7 +2149,7 @@ again:
        /*
         * Therefore we have to validate we did not indeed do so.
         */
-       if (unlikely(atomic_read(&data->done_head))) {
+       if (unlikely(atomic_long_read(&data->done_head))) {
                /*
                 * Since we had it locked, we can lock it again.
                 */
@@ -2195,7 +2196,7 @@ static int perf_output_begin(struct perf_output_handle *handle,
        do {
                offset = head = atomic_read(&data->head);
                head += size;
-       } while (atomic_cmpxchg(&data->head, offset, head) != offset);
+       } while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
 
        handle->offset  = offset;
        handle->head    = head;
@@ -2246,7 +2247,7 @@ static void perf_output_copy(struct perf_output_handle *handle,
         * Check we didn't copy past our reservation window, taking the
         * possible unsigned int wrap into account.
         */
-       WARN_ON_ONCE(((int)(handle->head - handle->offset)) < 0);
+       WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
 }
 
 #define perf_output_put(handle, x) \