dtrace: revive dtrace_gethrtime()

This patch re-introduces dtrace_gethrtime(). The CDDL workaround is no longer
required.

Orabug: 27409933

Signed-off-by: Tomas Jedlicka <tomas.jedlicka@oracle.com>
Reviewed-by: Nick Alcock <nick.alcock@oracle.com>
Reviewed-by: Kris Van Hees <kris.van.hees@oracle.com>
Conflicts:
	include/linux/dtrace_os.h
	kernel/dtrace/dtrace_os.c

(cherry picked from commit 71f7d587d7ad8aea85019fe3f2190553820422d0)
Signed-off-by: Jack Vogel <jack.vogel@oracle.com>

diff --git a/dtrace/dtrace_dev.c b/dtrace/dtrace_dev.c
index 994a46493815ba62c604d1b7c5abd3e2cc0ed0ae..034c5fc5f081ff1ba7de5c6fbcc68144affc18b6 100644 (file)
--- a/dtrace/dtrace_dev.c
+++ b/dtrace/dtrace_dev.c
@@ -854,9 +854,9 @@ static long dtrace_ioctl(struct file *file,
                 * for setting dts_laststatus to UINT64_MAX before setting
                 * it to the correct value.
                 */
-               state->dts_laststatus = UINT64_MAX;
+               state->dts_laststatus = ns_to_ktime(UINT64_MAX);
                dtrace_membar_producer();
-               state->dts_laststatus = jiffies;
+               state->dts_laststatus = dtrace_gethrtime();
 
                memset(&stat, 0, sizeof(stat));
 

diff --git a/dtrace/dtrace_dif.c b/dtrace/dtrace_dif.c
index 3bf3d51aaeafa4aee05d99bd98b18d49d0271e7a..11d64605b29f833d9f5211c108cd01f1da4914b1 100644 (file)
--- a/dtrace/dtrace_dif.c
+++ b/dtrace/dtrace_dif.c
@@ -23,7 +23,6 @@
 #include <linux/if_infiniband.h>
 #include <linux/in6.h>
 #include <linux/inet.h>
-#include <linux/jiffies.h>
 #include <linux/kdev_t.h>
 #include <linux/slab.h>
 #include <linux/socket.h>
@@ -2092,7 +2091,7 @@ static uint64_t __always_inline dtrace_dif_variable(dtrace_mstate_t *mstate,
 
        case DIF_VAR_TIMESTAMP:
                if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
-                       mstate->dtms_timestamp = current->dtrace_start;
+                       mstate->dtms_timestamp = dtrace_gethrtime();
                        mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP;
                }
 
@@ -2390,7 +2389,7 @@ static void dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
 
        switch (subr) {
        case DIF_SUBR_RAND:
-               regs[rd] = jiffies * 2416 + 374441;
+               regs[rd] = ktime_to_ns(dtrace_gethrtime()) * 2416 + 374441;
                regs[rd] = do_div(regs[rd], 1771875);
                break;
 

diff --git a/dtrace/dtrace_probe.c b/dtrace/dtrace_probe.c
index 89a15b0c83630773d64cb63131a495becaf61e0b..4afdecd80239b1e9191a860e277df4b04a8e1876 100644 (file)
--- a/dtrace/dtrace_probe.c
+++ b/dtrace/dtrace_probe.c
@@ -347,12 +347,49 @@ static void dtrace_action_stop(void)
 
 static void dtrace_action_chill(dtrace_mstate_t *mstate, ktime_t val)
 {
+       ktime_t                 now;
+       volatile uint16_t       *flags;
+       cpu_core_t              *cpu = this_cpu_core;
+
        if (dtrace_destructive_disallow)
                return;
 
-       dtrace_chill(val, dtrace_chill_interval, dtrace_chill_max);
+       flags = (volatile uint16_t *)&cpu->cpuc_dtrace_flags;
+
+       now = dtrace_gethrtime();
+
+       if (ktime_gt(ktime_sub(now, cpu->cpu_dtrace_chillmark),
+                    dtrace_chill_interval)) {
+               /*
+                * We need to advance the mark to current time.
+                */
+               cpu->cpu_dtrace_chillmark = now;
+               cpu->cpu_dtrace_chilled = ktime_set(0, 0);
+       }
 
+       /*
+        * Now check to see if the requested chill time would take us over
+        * the maximum amount of time allowed in the chill interval.  (Or
+        * worse, if the calculation itself induces overflow.)
+        */
+       if (ktime_gt(ktime_add(cpu->cpu_dtrace_chilled, val),
+                    dtrace_chill_max) ||
+           ktime_lt(ktime_add(cpu->cpu_dtrace_chilled, val),
+                    cpu->cpu_dtrace_chilled)) {
+               *flags |= CPU_DTRACE_ILLOP;
+               return;
+       }
+
+       while (ktime_lt(ktime_sub(dtrace_gethrtime(), now), val))
+               continue;
+
+       /*
+        * Normally, we assure that the value of the variable "timestamp" does
+        * not change within an ECB.  The presence of chill() represents an
+        * exception from this rule, however.
+        */
        mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP;
+       cpu->cpu_dtrace_chilled = ktime_add(cpu->cpu_dtrace_chilled, val);
 }
 
 static void dtrace_action_ustack(dtrace_mstate_t *mstate,
@@ -477,7 +514,9 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
        intptr_t                offs;
        size_t                  size;
        int                     onintr;
+       int                     vtime;
        volatile uint16_t       *flags;
+       ktime_t                 now;
        int                     pflag = 0;
        uint32_t                re_entry;
 
@@ -547,8 +586,12 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
        *flags |= CPU_DTRACE_PROBE_CTX;
        this_cpu_core->cpu_dtrace_caller = id;
 
-       if (id != dtrace_probeid_error)
-               dtrace_vtime_suspend();
+       now = dtrace_gethrtime();
+       vtime = (dtrace_vtime_references > 0);
+
+       if (vtime && ktime_nz(current->dtrace_start))
+               current->dtrace_vtime = ktime_add(current->dtrace_vtime,
+                                                 ktime_sub(now, current->dtrace_start));
 
        mstate.dtms_difo = NULL;
        mstate.dtms_probe = probe;
@@ -673,7 +716,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
                        }
                }
 
-               if (ktime_gt(ktime_sub(current->dtrace_start, state->dts_alive),
+               if (ktime_gt(ktime_sub(now, state->dts_alive),
                             dtrace_deadman_timeout)) {
                        /*
                         * We seem to be dead.  Unless we (a) have kernel
@@ -1169,6 +1212,15 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
                                continue;
                        }
 
+                       if (vtime)
+                               /*
+                                * Before recursing on dtrace_probe(), we
+                                * need to explicitly clear out our start
+                                * time to prevent it from being accumulated
+                                * into the dtrace_vtime.
+                                */
+                               current->dtrace_start = ktime_set(0, 0);
+
                        /*
                         * Iterate over the actions to figure out which action
                         * we were processing when we experienced the error.
@@ -1202,7 +1254,8 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
                             id, ecb->dte_epid);
        }
 
-       dtrace_vtime_resume();
+       if (vtime)
+               current->dtrace_start = dtrace_gethrtime();
 
        /*
         * Only clear the flag if this is not the ERROR probe.  We know that

diff --git a/dtrace/dtrace_state.c b/dtrace/dtrace_state.c
index c04be314f8eb2ba0f09ec5ddc7142afff31b008b..92c492752ffdb56263557c64f626bd3266bbe770 100644 (file)
--- a/dtrace/dtrace_state.c
+++ b/dtrace/dtrace_state.c
@@ -43,7 +43,7 @@ dtrace_optval_t               dtrace_jstackframes_default = 50;
 dtrace_optval_t                dtrace_jstackstrsize_default = 512;
 ktime_t                        dtrace_deadman_interval = KTIME_INIT(1, 0);
 ktime_t                        dtrace_deadman_timeout = KTIME_INIT(10, 0);
-uint64_t               dtrace_deadman_user = SECS_TO_JIFFIES(30);
+ktime_t                        dtrace_deadman_user = KTIME_INIT(30, 0);
 uint64_t               dtrace_sync_sample_count = 100; /* Sampling before counting */
 
 dtrace_id_t            dtrace_probeid_begin;
@@ -274,7 +274,7 @@ void dtrace_vstate_fini(dtrace_vstate_t *vstate)
                vfree(vstate->dtvs_locals);
 }
 
-static void dtrace_state_clean(dtrace_state_t *state, ktime_t when)
+static void dtrace_state_clean(dtrace_state_t *state)
 {
        dtrace_optval_t         *opt = state->dts_options;
 
@@ -289,12 +289,16 @@ static void dtrace_state_clean(dtrace_state_t *state, ktime_t when)
                                                opt[DTRACEOPT_CLEANRATE]));
 }
 
-static void dtrace_state_deadman(dtrace_state_t *state, ktime_t when)
+static void dtrace_state_deadman(dtrace_state_t *state)
 {
+       ktime_t                 now;
+
        dtrace_sync();
 
+       now = dtrace_gethrtime();
        if (state != dtrace_anon.dta_state &&
-           time_after_eq(jiffies, state->dts_laststatus + dtrace_deadman_user))
+           ktime_ge(ktime_sub(now, state->dts_laststatus),
+                    dtrace_deadman_user))
                return;
 
        /*
@@ -308,7 +312,7 @@ static void dtrace_state_deadman(dtrace_state_t *state, ktime_t when)
         */
        state->dts_alive = ktime_set(KTIME_SEC_MAX, 0);
        dtrace_membar_producer();
-       state->dts_alive = when;
+       state->dts_alive = now;
 }
 
 dtrace_state_t *dtrace_state_create(struct file *file)
@@ -776,9 +780,8 @@ int dtrace_state_go(dtrace_state_t *state, processorid_t *cpu)
        when.cyt_when = ktime_set(0, 0);
        when.cyt_interval = dtrace_deadman_interval;
 
+       state->dts_alive = state->dts_laststatus = dtrace_gethrtime();
        state->dts_deadman = cyclic_add(&hdlr, &when);
-       state->dts_alive = when.cyt_when;
-       state->dts_laststatus = jiffies;
 
        state->dts_activity = DTRACE_ACTIVITY_WARMUP;
 

diff --git a/dtrace/profile_dev.c b/dtrace/profile_dev.c
index f2abd275f2013b3b06004c2d4d360c815df6b3ce..aea38a17aa815714cf0e9887f92b1a3d4865b61b 100644 (file)
--- a/dtrace/profile_dev.c
+++ b/dtrace/profile_dev.c
@@ -77,7 +77,7 @@ static int    profile_ticks[] = {
 static int     profile_max;            /* maximum number of profile probes */
 static atomic_t        profile_total;          /* current number of profile probes */
 
-static void profile_tick_fn(uintptr_t arg, ktime_t when)
+static void profile_tick_fn(uintptr_t arg)
 {
        profile_probe_t *prof = (profile_probe_t *)arg;
        unsigned long   pc = 0, upc = 0;
@@ -104,7 +104,7 @@ static void profile_tick_fn(uintptr_t arg, ktime_t when)
        dtrace_probe(prof->prof_id, pc, upc, 0, 0, 0);
 }
 
-static void profile_prof_fn(uintptr_t arg, ktime_t when)
+static void profile_prof_fn(uintptr_t arg)
 {
        profile_probe_percpu_t  *pcpu = (profile_probe_percpu_t *)arg;
        profile_probe_t         *prof = pcpu->profc_probe;
@@ -112,7 +112,7 @@ static void profile_prof_fn(uintptr_t arg, ktime_t when)
        struct pt_regs          *regs = get_irq_regs();
        unsigned long           pc = 0, upc = 0;
 
-       late = ktime_sub(when, pcpu->profc_expected);
+       late = ktime_sub(dtrace_gethrtime(), pcpu->profc_expected);
        pcpu->profc_expected = ktime_add(pcpu->profc_expected,
                                         pcpu->profc_interval);
 
@@ -151,7 +151,7 @@ static void profile_online(void *arg, processorid_t cpu, cyc_handler_t *hdlr,
        hdlr->cyh_level = CY_HIGH_LEVEL;
 
        when->cyt_interval = prof->prof_interval;
-       when->cyt_when = ktime_add(when->cyt_when, when->cyt_interval);
+       when->cyt_when = ktime_add(dtrace_gethrtime(), when->cyt_interval);
 
        pcpu->profc_expected = when->cyt_when;
        pcpu->profc_interval = when->cyt_interval;

diff --git a/include/dtrace/dtrace_impl.h b/include/dtrace/dtrace_impl.h
index 663e0a36ebd1167e17a7af4a876903c4aae0c4e5..06542894f5ff011ae94754750dd6c879f561ff6d 100644 (file)
--- a/include/dtrace/dtrace_impl.h
+++ b/include/dtrace/dtrace_impl.h
@@ -267,7 +267,7 @@ struct dtrace_state {
        uint32_t dts_reserve;
        cyclic_id_t dts_cleaner;
        cyclic_id_t dts_deadman;
-       uint64_t dts_laststatus;
+       ktime_t dts_laststatus;
        ktime_t dts_alive;
        char dts_speculates;
        char dts_destructive;

diff --git a/include/dtrace/types.h b/include/dtrace/types.h
index 367406f22eae1dfdcb8ae2ccbd2a4900d98c13cc..afa70e28833637f09da6d4bbdd6a284eb07b3ab8 100644 (file)
--- a/include/dtrace/types.h
+++ b/include/dtrace/types.h
@@ -41,7 +41,6 @@
 
 #include <asm/bitsperlong.h>
 #include <linux/dtrace_os.h>
-#include <linux/jiffies.h>
 
 typedef unsigned char  uchar_t;
 typedef unsigned int   uint_t;
@@ -118,8 +117,6 @@ typedef enum {
 #define ktime_gt(t0, t1)       ((t0).tv64 > (t1).tv64)
 #define ktime_cp(t0, t1)       ((t0).tv64 = (t1).tv64)
 
-#define SECS_TO_JIFFIES(s)     (((s) * SEC_CONVERSION) >> SEC_JIFFIE_SC)
-
 /*
  * Translate between kernel config options and userspace-compatible definitions.
  */

diff --git a/include/linux/cyclic.h b/include/linux/cyclic.h
index 5d8950222a3a64c6006579f09aab7cc136cccfac..eec5317e42e88f7056559d0427c7ae15d2bf3ce9 100644 (file)
--- a/include/linux/cyclic.h
+++ b/include/linux/cyclic.h
@@ -16,7 +16,7 @@
 
 typedef uintptr_t      cyclic_id_t;
 typedef uint16_t       cyc_level_t;
-typedef void           (*cyc_func_t)(uintptr_t, ktime_t);
+typedef void           (*cyc_func_t)(uintptr_t);
 
 #define CYCLIC_NONE    ((cyclic_id_t)0)
 

diff --git a/include/linux/dtrace_os.h b/include/linux/dtrace_os.h
index 145e6783784f6b8ea25fbd2060f2c43534a9729f..31958bb842250f345d5b2157fd8cf5ec6eafb508 100644 (file)
--- a/include/linux/dtrace_os.h
+++ b/include/linux/dtrace_os.h
@@ -35,6 +35,9 @@ extern void dtrace_free_text(void *);
 extern void dtrace_enable(void);
 extern void dtrace_disable(void);
 
+extern ktime_t dtrace_gethrtime(void);
+extern ktime_t dtrace_getwalltime(void);
+
 typedef enum dtrace_vtime_state {
        DTRACE_VTIME_INACTIVE = 0,
        DTRACE_VTIME_ACTIVE
@@ -51,10 +54,6 @@ extern ktime_t dtrace_get_walltime(void);
 extern void dtrace_vtime_enable(void);
 extern void dtrace_vtime_disable(void);
 extern void dtrace_vtime_switch(struct task_struct *, struct task_struct *);
-extern void dtrace_vtime_suspend(void);
-extern void dtrace_vtime_resume(void);
-extern void dtrace_chill(ktime_t, ktime_t, ktime_t);
-extern ktime_t dtrace_gethrtime(void);
 
 extern void dtrace_skip_instruction(struct pt_regs *);
 

diff --git a/kernel/dtrace/cyclic.c b/kernel/dtrace/cyclic.c
index 822bb3c6fbeaf9c3fd26b52508d7848b718ac81d..18b71192ad9e1d433a301b002882d643ebf6f802 100644 (file)
--- a/kernel/dtrace/cyclic.c
+++ b/kernel/dtrace/cyclic.c
@@ -69,8 +69,7 @@ static void cyclic_fire(struct work_struct *work)
                 * We know that the 'pend' counter for the cyclic is non-zero.
                 * So, we can start with calling the handler at least once.
                 */
-               (*cyc->cyc.hdlr.cyh_func)(cyc->cyc.hdlr.cyh_arg,
-                                         ns_to_ktime(ktime_get_raw_fast_ns()));
+               (*cyc->cyc.hdlr.cyh_func)(cyc->cyc.hdlr.cyh_arg);
 
 again:
                /*
@@ -125,8 +124,7 @@ static enum hrtimer_restart cyclic_expire(struct hrtimer *timr)
         * interrupt context.
         */
        if (cyc->cyc.hdlr.cyh_level == CY_HIGH_LEVEL) {
-               (*cyc->cyc.hdlr.cyh_func)(cyc->cyc.hdlr.cyh_arg,
-                                         ns_to_ktime(ktime_get_raw_fast_ns()));
+               (*cyc->cyc.hdlr.cyh_func)(cyc->cyc.hdlr.cyh_arg);
                goto done;
        }
 
@@ -212,11 +210,6 @@ cyclic_id_t cyclic_add(cyc_handler_t *hdlr, cyc_time_t *when)
 
        cyclic_restart(cyc);
 
-       /*
-        * Let the caller know when the cyclic was added.
-        */
-       when->cyt_when = ns_to_ktime(ktime_get_raw_fast_ns());
-
        return (cyclic_id_t)cyc;
 }
 EXPORT_SYMBOL(cyclic_add);
@@ -255,11 +248,6 @@ static void cyclic_omni_start(cyclic_t *omni, int cpu)
        cyc_time_t      when;
        cyc_handler_t   hdlr;
 
-       /*
-        * Let the caller know when the cyclic is being started.
-        */
-       when.cyt_when = ns_to_ktime(ktime_get_raw_fast_ns());
-
        omni->omni.hdlr.cyo_online(omni->omni.hdlr.cyo_arg, cpu, &hdlr, &when);
        cyclic_add_pinned(cpu, omni, &hdlr, &when);
 }

diff --git a/kernel/dtrace/dtrace_os.c b/kernel/dtrace/dtrace_os.c
index 3ddd226fcb20b50831eb54a629b67742af71fe2e..17e52e25df3267316885bccf51cce253775100c2 100644 (file)
--- a/kernel/dtrace/dtrace_os.c
+++ b/kernel/dtrace/dtrace_os.c
@@ -463,6 +463,19 @@ ktime_t dtrace_get_walltime(void)
 }
 EXPORT_SYMBOL(dtrace_get_walltime);
 
+ktime_t dtrace_gethrtime(void)
+{
+       return ns_to_ktime(ktime_get_raw_fast_ns());
+}
+EXPORT_SYMBOL(dtrace_gethrtime);
+
+/* Needed for lockstat probes where we cannot include ktime.h */
+u64 dtrace_gethrtime_ns(void)
+{
+       return ktime_get_raw_fast_ns();
+}
+EXPORT_SYMBOL(dtrace_gethrtime_ns);
+
 void dtrace_vtime_enable(void)
 {
        dtrace_vtime_state_t    old, new;
@@ -497,7 +510,7 @@ EXPORT_SYMBOL(dtrace_vtime_disable);
 
 void dtrace_vtime_switch(struct task_struct *prev, struct task_struct *next)
 {
-       ktime_t now = ns_to_ktime(ktime_get_raw_fast_ns());
+       ktime_t now = dtrace_gethrtime();
 
        if (ktime_nz(prev->dtrace_start)) {
                prev->dtrace_vtime = ktime_add(prev->dtrace_vtime,
@@ -509,66 +522,6 @@ void dtrace_vtime_switch(struct task_struct *prev, struct task_struct *next)
        next->dtrace_start = now;
 }
 
-void dtrace_vtime_suspend(void)
-{
-       ktime_t now = ns_to_ktime(ktime_get_raw_fast_ns());
-
-       current->dtrace_vtime = ktime_add(current->dtrace_vtime,
-                                 ktime_sub(now, current->dtrace_start));
-       current->dtrace_start = now;
-}
-EXPORT_SYMBOL(dtrace_vtime_suspend);
-
-void dtrace_vtime_resume(void)
-{
-       current->dtrace_start = ns_to_ktime(ktime_get_raw_fast_ns());
-}
-EXPORT_SYMBOL(dtrace_vtime_resume);
-
-#define ktime_lt(t0, t1)       ((t0).tv64 < (t1).tv64)
-#define ktime_gt(t0, t1)       ((t0).tv64 > (t1).tv64)
-
-void dtrace_chill(ktime_t val, ktime_t interval, ktime_t int_max)
-{
-       ktime_t                 now = ns_to_ktime(ktime_get_raw_fast_ns());
-       cpu_core_t              *cpu = this_cpu_core;
-       volatile uint16_t       *flags;
-
-       flags = (volatile uint16_t *)&cpu->cpuc_dtrace_flags;
-
-       if (ktime_gt(ktime_sub(now, cpu->cpu_dtrace_chillmark), interval)) {
-               cpu->cpu_dtrace_chillmark = now;
-               cpu->cpu_dtrace_chilled = ktime_set(0, 0);
-       }
-
-       if (ktime_gt(ktime_add(cpu->cpu_dtrace_chilled, val), int_max) ||
-           ktime_lt(ktime_add(cpu->cpu_dtrace_chilled, val),
-                    cpu->cpu_dtrace_chilled)) {
-               *flags |= CPU_DTRACE_ILLOP;
-               return;
-       }
-
-       while (ktime_lt(ktime_sub(ns_to_ktime(ktime_get_raw_fast_ns()), now),
-                       val))
-               continue;
-
-       cpu->cpu_dtrace_chilled = ktime_add(cpu->cpu_dtrace_chilled, val);
-}
-EXPORT_SYMBOL(dtrace_chill);
-
-/* Needed for lockstat probes where we cannot include ktime.h */
-u64 dtrace_gethrtime_ns(void)
-{
-       return ktime_get_raw_fast_ns();
-}
-EXPORT_SYMBOL(dtrace_gethrtime_ns);
-
-ktime_t dtrace_gethrtime(void)
-{
-       return ns_to_ktime(dtrace_gethrtime_ns());
-}
-EXPORT_SYMBOL(dtrace_gethrtime);
-
 void dtrace_stacktrace(stacktrace_state_t *st)
 {
        struct stack_trace      trace;