From ec7b092325e24ad7cf19ee116f34eb42e929aa3f Mon Sep 17 00:00:00 2001 From: Tomas Jedlicka Date: Thu, 9 Mar 2017 09:48:56 -0500 Subject: [PATCH] dtrace: Implement high precision walltimestamp There are lock-free implementations for other timers (mono & raw) but lock-free access to realtime clock is missing. This patch allows DTrace to provide CLOCK_REALTIME_COARSE time via walltimestamp without taking any lock. Orabug: 25883559 Signed-off-by: Tomas Jedlicka Reviewed-by: Nick Alcock Acked-by: Somasundaram Krishnasamy --- arch/x86/kernel/fbt_blacklist.h | 4 ++++ include/linux/dtrace_os.h | 11 ++++++++++- kernel/dtrace/dtrace_os.c | 32 ++++++++++++++++++++++++++++++-- kernel/time/timekeeping.c | 2 ++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/fbt_blacklist.h b/arch/x86/kernel/fbt_blacklist.h index db887b4889af..5b14bbf64c99 100644 --- a/arch/x86/kernel/fbt_blacklist.h +++ b/arch/x86/kernel/fbt_blacklist.h @@ -16,6 +16,10 @@ BL_SENTRY(typeof(idr_find_slowpath), idr_find_slowpath) * Functions used to update vtime in probe context. */ BL_SENTRY(typeof(ktime_get_raw_fast_ns), ktime_get_raw_fast_ns) +BL_DENTRY(void *, raw_read_seqcount) +BL_DENTRY(void *, read_seqcount_retry) +BL_DENTRY(void *, __read_seqcount_retry) + /* xen_clocksource */ BL_DENTRY(void *, xen_clocksource_get_cycles) BL_DENTRY(void *, xen_clocksource_read) diff --git a/include/linux/dtrace_os.h b/include/linux/dtrace_os.h index dffe8848a6f9..a9636cb711b7 100644 --- a/include/linux/dtrace_os.h +++ b/include/linux/dtrace_os.h @@ -1,4 +1,6 @@ -/* Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved. */ +/* + * Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved. + */ #ifndef _LINUX_DTRACE_OS_H_ #define _LINUX_DTRACE_OS_H_ @@ -12,6 +14,7 @@ typedef uint32_t dtrace_id_t; #include #include #include +#include #if defined(CONFIG_DT_FASTTRAP) || defined(CONFIG_DT_FASTTRAP_MODULE) #include #endif @@ -39,6 +42,7 @@ typedef enum dtrace_vtime_state { extern dtrace_vtime_state_t dtrace_vtime_active; +extern void dtrace_update_time(struct timekeeper *); extern ktime_t dtrace_get_walltime(void); extern void dtrace_vtime_enable(void); @@ -118,6 +122,11 @@ extern int dtrace_tracepoint_disable(pid_t, fasttrap_machtp_t *); #define dtrace_no_pf(ignore) 0 +/* + * See kernel/timekeeper.c + */ +#define dtrace_update_time(ignore) + #endif /* CONFIG_DTRACE */ #endif /* !HEADERS_CHECK */ diff --git a/kernel/dtrace/dtrace_os.c b/kernel/dtrace/dtrace_os.c index c6962922b8e7..f32d6fd2d55d 100644 --- a/kernel/dtrace/dtrace_os.c +++ b/kernel/dtrace/dtrace_os.c @@ -379,11 +379,39 @@ void dtrace_psinfo_free(struct task_struct *tsk) \*---------------------------------------------------------------------------*/ dtrace_vtime_state_t dtrace_vtime_active = 0; +/* + * Until Linux kernel gains lock-free realtime clock access we are maintaining + * our own version for lock-free access from within a probe context. + */ +static struct dtrace_time_fast { + seqcount_t dtwf_seq; + ktime_t dtwf_offsreal[2]; +} dtrace_time ____cacheline_aligned; + +/* + * Callback from timekeeper code that allows dtrace to update its own time data. + */ +void dtrace_update_time(struct timekeeper *tk) +{ + raw_write_seqcount_latch(&dtrace_time.dtwf_seq); + dtrace_time.dtwf_offsreal[0] = tk->offs_real; + raw_write_seqcount_latch(&dtrace_time.dtwf_seq); + dtrace_time.dtwf_offsreal[1] = tk->offs_real; +} + +/* Lock free walltime */ ktime_t dtrace_get_walltime(void) { - struct timespec t = __current_kernel_time(); + u64 nsec = ktime_get_mono_fast_ns(); + unsigned int seq; + ktime_t *offset; + + do { + seq = raw_read_seqcount(&dtrace_time.dtwf_seq); + offset = dtrace_time.dtwf_offsreal + (seq & 0x1); + } while (read_seqcount_retry(&dtrace_time.dtwf_seq, seq)); - return ns_to_ktime(timespec64_to_ns(&t)); + return ktime_add_ns(*offset, nsec); } EXPORT_SYMBOL(dtrace_get_walltime); diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 65dbf8aee751..a491243ffbb2 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "tick-internal.h" #include "ntp_internal.h" @@ -593,6 +594,7 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action) tk_update_ktime_data(tk); update_vsyscall(tk); + dtrace_update_time(tk); update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); if (action & TK_MIRROR) -- 2.50.1