dtrace: Implement high precision walltimestamp

There are lock-free implementations for other timers (mono & raw) but
lock-free access to realtime clock is missing. This patch allows DTrace
to provide CLOCK_REALTIME_COARSE time via walltimestamp without taking
any lock.

Orabug: 25883559

Signed-off-by: Tomas Jedlicka <tomas.jedlicka@oracle.com>
Reviewed-by: Nick Alcock <nick.alcock@oracle.com>
Acked-by: Somasundaram Krishnasamy <somasundaram.krishnasamy@oracle.com>

diff --git a/arch/x86/kernel/fbt_blacklist.h b/arch/x86/kernel/fbt_blacklist.h
index db887b4889af624a9e91864fd69a6c88edecc660..5b14bbf64c99ad9a772df9600974246dbed11c0c 100644 (file)
--- a/arch/x86/kernel/fbt_blacklist.h
+++ b/arch/x86/kernel/fbt_blacklist.h
@@ -16,6 +16,10 @@ BL_SENTRY(typeof(idr_find_slowpath), idr_find_slowpath)
  * Functions used to update vtime in probe context.
  */
 BL_SENTRY(typeof(ktime_get_raw_fast_ns), ktime_get_raw_fast_ns)
+BL_DENTRY(void *, raw_read_seqcount)
+BL_DENTRY(void *, read_seqcount_retry)
+BL_DENTRY(void *, __read_seqcount_retry)
+
 /* xen_clocksource */
 BL_DENTRY(void *, xen_clocksource_get_cycles)
 BL_DENTRY(void *, xen_clocksource_read)

diff --git a/include/linux/dtrace_os.h b/include/linux/dtrace_os.h
index dffe8848a6f9091e8f65ff9ccf0dd3c770d38c0c..a9636cb711b766a96132548a49c90a88c34311cd 100644 (file)
--- a/include/linux/dtrace_os.h
+++ b/include/linux/dtrace_os.h
@@ -1,4 +1,6 @@
-/* Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved. */
+/*
+ * Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved.
+ */
 
 #ifndef _LINUX_DTRACE_OS_H_
 #define _LINUX_DTRACE_OS_H_
@@ -12,6 +14,7 @@ typedef uint32_t dtrace_id_t;
 #include <linux/ktime.h>
 #include <linux/mm.h>
 #include <linux/notifier.h>
+#include <linux/timekeeper_internal.h>
 #if defined(CONFIG_DT_FASTTRAP) || defined(CONFIG_DT_FASTTRAP_MODULE)
 #include <linux/uprobes.h>
 #endif
@@ -39,6 +42,7 @@ typedef enum dtrace_vtime_state {
 
 extern dtrace_vtime_state_t dtrace_vtime_active;
 
+extern void dtrace_update_time(struct timekeeper *);
 extern ktime_t dtrace_get_walltime(void);
 
 extern void dtrace_vtime_enable(void);
@@ -118,6 +122,11 @@ extern int dtrace_tracepoint_disable(pid_t, fasttrap_machtp_t *);
 
 #define dtrace_no_pf(ignore) 0
 
+/*
+ * See kernel/timekeeper.c
+ */
+#define        dtrace_update_time(ignore)
+
 #endif /* CONFIG_DTRACE */
 
 #endif /* !HEADERS_CHECK */

diff --git a/kernel/dtrace/dtrace_os.c b/kernel/dtrace/dtrace_os.c
index c6962922b8e759c55e51ce84af51af1c76906774..f32d6fd2d55d5a8eb7ccfea8498995906192e0a8 100644 (file)
--- a/kernel/dtrace/dtrace_os.c
+++ b/kernel/dtrace/dtrace_os.c
@@ -379,11 +379,39 @@ void dtrace_psinfo_free(struct task_struct *tsk)
 \*---------------------------------------------------------------------------*/
 dtrace_vtime_state_t   dtrace_vtime_active = 0;
 
+/*
+ * Until Linux kernel gains lock-free realtime clock access we are maintaining
+ * our own version for lock-free access from within a probe context.
+ */
+static struct dtrace_time_fast {
+       seqcount_t      dtwf_seq;
+       ktime_t         dtwf_offsreal[2];
+} dtrace_time ____cacheline_aligned;
+
+/*
+ * Callback from timekeeper code that allows dtrace to update its own time data.
+ */
+void dtrace_update_time(struct timekeeper *tk)
+{
+       raw_write_seqcount_latch(&dtrace_time.dtwf_seq);
+       dtrace_time.dtwf_offsreal[0] = tk->offs_real;
+       raw_write_seqcount_latch(&dtrace_time.dtwf_seq);
+       dtrace_time.dtwf_offsreal[1] = tk->offs_real;
+}
+
+/* Lock free walltime */
 ktime_t dtrace_get_walltime(void)
 {
-       struct timespec t = __current_kernel_time();
+       u64 nsec = ktime_get_mono_fast_ns();
+       unsigned int seq;
+       ktime_t *offset;
+
+       do {
+               seq = raw_read_seqcount(&dtrace_time.dtwf_seq);
+               offset = dtrace_time.dtwf_offsreal + (seq & 0x1);
+       } while (read_seqcount_retry(&dtrace_time.dtwf_seq, seq));
 
-       return ns_to_ktime(timespec64_to_ns(&t));
+       return ktime_add_ns(*offset, nsec);
 }
 EXPORT_SYMBOL(dtrace_get_walltime);
 

diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 65dbf8aee751d0468ddaf4b0cb7cb7ce1deae74f..a491243ffbb27de99b0120a790c268c8072c5f26 100644 (file)
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -23,6 +23,7 @@
 #include <linux/stop_machine.h>
 #include <linux/pvclock_gtod.h>
 #include <linux/compiler.h>
+#include <linux/dtrace_os.h>
 
 #include "tick-internal.h"
 #include "ntp_internal.h"
@@ -593,6 +594,7 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action)
        tk_update_ktime_data(tk);
 
        update_vsyscall(tk);
+       dtrace_update_time(tk);
        update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
 
        if (action & TK_MIRROR)