From: Tomas Jedlicka Date: Wed, 20 Sep 2017 09:29:09 +0000 (-0400) Subject: dtrace: revive dtrace_gethrtime() X-Git-Tag: v4.1.12-124.31.3~1255 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=9e4f3720fcf5f56e5e96ba994182147d1d6a16c9;p=users%2Fjedix%2Flinux-maple.git dtrace: revive dtrace_gethrtime() This patch re-introduces dtrace_gethrtime(). The CDDL workaround is no longer required. Orabug: 27409933 Signed-off-by: Tomas Jedlicka Reviewed-by: Nick Alcock Reviewed-by: Kris Van Hees Conflicts: include/linux/dtrace_os.h kernel/dtrace/dtrace_os.c (cherry picked from commit 71f7d587d7ad8aea85019fe3f2190553820422d0) Signed-off-by: Jack Vogel --- diff --git a/dtrace/dtrace_dev.c b/dtrace/dtrace_dev.c index 994a46493815..034c5fc5f081 100644 --- a/dtrace/dtrace_dev.c +++ b/dtrace/dtrace_dev.c @@ -854,9 +854,9 @@ static long dtrace_ioctl(struct file *file, * for setting dts_laststatus to UINT64_MAX before setting * it to the correct value. */ - state->dts_laststatus = UINT64_MAX; + state->dts_laststatus = ns_to_ktime(UINT64_MAX); dtrace_membar_producer(); - state->dts_laststatus = jiffies; + state->dts_laststatus = dtrace_gethrtime(); memset(&stat, 0, sizeof(stat)); diff --git a/dtrace/dtrace_dif.c b/dtrace/dtrace_dif.c index 3bf3d51aaeaf..11d64605b29f 100644 --- a/dtrace/dtrace_dif.c +++ b/dtrace/dtrace_dif.c @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -2092,7 +2091,7 @@ static uint64_t __always_inline dtrace_dif_variable(dtrace_mstate_t *mstate, case DIF_VAR_TIMESTAMP: if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) { - mstate->dtms_timestamp = current->dtrace_start; + mstate->dtms_timestamp = dtrace_gethrtime(); mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP; } @@ -2390,7 +2389,7 @@ static void dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, switch (subr) { case DIF_SUBR_RAND: - regs[rd] = jiffies * 2416 + 374441; + regs[rd] = ktime_to_ns(dtrace_gethrtime()) * 2416 + 374441; regs[rd] = do_div(regs[rd], 1771875); break; diff --git a/dtrace/dtrace_probe.c b/dtrace/dtrace_probe.c index 89a15b0c8363..4afdecd80239 100644 --- a/dtrace/dtrace_probe.c +++ b/dtrace/dtrace_probe.c @@ -347,12 +347,49 @@ static void dtrace_action_stop(void) static void dtrace_action_chill(dtrace_mstate_t *mstate, ktime_t val) { + ktime_t now; + volatile uint16_t *flags; + cpu_core_t *cpu = this_cpu_core; + if (dtrace_destructive_disallow) return; - dtrace_chill(val, dtrace_chill_interval, dtrace_chill_max); + flags = (volatile uint16_t *)&cpu->cpuc_dtrace_flags; + + now = dtrace_gethrtime(); + + if (ktime_gt(ktime_sub(now, cpu->cpu_dtrace_chillmark), + dtrace_chill_interval)) { + /* + * We need to advance the mark to current time. + */ + cpu->cpu_dtrace_chillmark = now; + cpu->cpu_dtrace_chilled = ktime_set(0, 0); + } + /* + * Now check to see if the requested chill time would take us over + * the maximum amount of time allowed in the chill interval. (Or + * worse, if the calculation itself induces overflow.) + */ + if (ktime_gt(ktime_add(cpu->cpu_dtrace_chilled, val), + dtrace_chill_max) || + ktime_lt(ktime_add(cpu->cpu_dtrace_chilled, val), + cpu->cpu_dtrace_chilled)) { + *flags |= CPU_DTRACE_ILLOP; + return; + } + + while (ktime_lt(ktime_sub(dtrace_gethrtime(), now), val)) + continue; + + /* + * Normally, we assure that the value of the variable "timestamp" does + * not change within an ECB. The presence of chill() represents an + * exception from this rule, however. + */ mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP; + cpu->cpu_dtrace_chilled = ktime_add(cpu->cpu_dtrace_chilled, val); } static void dtrace_action_ustack(dtrace_mstate_t *mstate, @@ -477,7 +514,9 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1, intptr_t offs; size_t size; int onintr; + int vtime; volatile uint16_t *flags; + ktime_t now; int pflag = 0; uint32_t re_entry; @@ -547,8 +586,12 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1, *flags |= CPU_DTRACE_PROBE_CTX; this_cpu_core->cpu_dtrace_caller = id; - if (id != dtrace_probeid_error) - dtrace_vtime_suspend(); + now = dtrace_gethrtime(); + vtime = (dtrace_vtime_references > 0); + + if (vtime && ktime_nz(current->dtrace_start)) + current->dtrace_vtime = ktime_add(current->dtrace_vtime, + ktime_sub(now, current->dtrace_start)); mstate.dtms_difo = NULL; mstate.dtms_probe = probe; @@ -673,7 +716,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1, } } - if (ktime_gt(ktime_sub(current->dtrace_start, state->dts_alive), + if (ktime_gt(ktime_sub(now, state->dts_alive), dtrace_deadman_timeout)) { /* * We seem to be dead. Unless we (a) have kernel @@ -1169,6 +1212,15 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1, continue; } + if (vtime) + /* + * Before recursing on dtrace_probe(), we + * need to explicitly clear out our start + * time to prevent it from being accumulated + * into the dtrace_vtime. + */ + current->dtrace_start = ktime_set(0, 0); + /* * Iterate over the actions to figure out which action * we were processing when we experienced the error. @@ -1202,7 +1254,8 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1, id, ecb->dte_epid); } - dtrace_vtime_resume(); + if (vtime) + current->dtrace_start = dtrace_gethrtime(); /* * Only clear the flag if this is not the ERROR probe. We know that diff --git a/dtrace/dtrace_state.c b/dtrace/dtrace_state.c index c04be314f8eb..92c492752ffd 100644 --- a/dtrace/dtrace_state.c +++ b/dtrace/dtrace_state.c @@ -43,7 +43,7 @@ dtrace_optval_t dtrace_jstackframes_default = 50; dtrace_optval_t dtrace_jstackstrsize_default = 512; ktime_t dtrace_deadman_interval = KTIME_INIT(1, 0); ktime_t dtrace_deadman_timeout = KTIME_INIT(10, 0); -uint64_t dtrace_deadman_user = SECS_TO_JIFFIES(30); +ktime_t dtrace_deadman_user = KTIME_INIT(30, 0); uint64_t dtrace_sync_sample_count = 100; /* Sampling before counting */ dtrace_id_t dtrace_probeid_begin; @@ -274,7 +274,7 @@ void dtrace_vstate_fini(dtrace_vstate_t *vstate) vfree(vstate->dtvs_locals); } -static void dtrace_state_clean(dtrace_state_t *state, ktime_t when) +static void dtrace_state_clean(dtrace_state_t *state) { dtrace_optval_t *opt = state->dts_options; @@ -289,12 +289,16 @@ static void dtrace_state_clean(dtrace_state_t *state, ktime_t when) opt[DTRACEOPT_CLEANRATE])); } -static void dtrace_state_deadman(dtrace_state_t *state, ktime_t when) +static void dtrace_state_deadman(dtrace_state_t *state) { + ktime_t now; + dtrace_sync(); + now = dtrace_gethrtime(); if (state != dtrace_anon.dta_state && - time_after_eq(jiffies, state->dts_laststatus + dtrace_deadman_user)) + ktime_ge(ktime_sub(now, state->dts_laststatus), + dtrace_deadman_user)) return; /* @@ -308,7 +312,7 @@ static void dtrace_state_deadman(dtrace_state_t *state, ktime_t when) */ state->dts_alive = ktime_set(KTIME_SEC_MAX, 0); dtrace_membar_producer(); - state->dts_alive = when; + state->dts_alive = now; } dtrace_state_t *dtrace_state_create(struct file *file) @@ -776,9 +780,8 @@ int dtrace_state_go(dtrace_state_t *state, processorid_t *cpu) when.cyt_when = ktime_set(0, 0); when.cyt_interval = dtrace_deadman_interval; + state->dts_alive = state->dts_laststatus = dtrace_gethrtime(); state->dts_deadman = cyclic_add(&hdlr, &when); - state->dts_alive = when.cyt_when; - state->dts_laststatus = jiffies; state->dts_activity = DTRACE_ACTIVITY_WARMUP; diff --git a/dtrace/profile_dev.c b/dtrace/profile_dev.c index f2abd275f201..aea38a17aa81 100644 --- a/dtrace/profile_dev.c +++ b/dtrace/profile_dev.c @@ -77,7 +77,7 @@ static int profile_ticks[] = { static int profile_max; /* maximum number of profile probes */ static atomic_t profile_total; /* current number of profile probes */ -static void profile_tick_fn(uintptr_t arg, ktime_t when) +static void profile_tick_fn(uintptr_t arg) { profile_probe_t *prof = (profile_probe_t *)arg; unsigned long pc = 0, upc = 0; @@ -104,7 +104,7 @@ static void profile_tick_fn(uintptr_t arg, ktime_t when) dtrace_probe(prof->prof_id, pc, upc, 0, 0, 0); } -static void profile_prof_fn(uintptr_t arg, ktime_t when) +static void profile_prof_fn(uintptr_t arg) { profile_probe_percpu_t *pcpu = (profile_probe_percpu_t *)arg; profile_probe_t *prof = pcpu->profc_probe; @@ -112,7 +112,7 @@ static void profile_prof_fn(uintptr_t arg, ktime_t when) struct pt_regs *regs = get_irq_regs(); unsigned long pc = 0, upc = 0; - late = ktime_sub(when, pcpu->profc_expected); + late = ktime_sub(dtrace_gethrtime(), pcpu->profc_expected); pcpu->profc_expected = ktime_add(pcpu->profc_expected, pcpu->profc_interval); @@ -151,7 +151,7 @@ static void profile_online(void *arg, processorid_t cpu, cyc_handler_t *hdlr, hdlr->cyh_level = CY_HIGH_LEVEL; when->cyt_interval = prof->prof_interval; - when->cyt_when = ktime_add(when->cyt_when, when->cyt_interval); + when->cyt_when = ktime_add(dtrace_gethrtime(), when->cyt_interval); pcpu->profc_expected = when->cyt_when; pcpu->profc_interval = when->cyt_interval; diff --git a/include/dtrace/dtrace_impl.h b/include/dtrace/dtrace_impl.h index 663e0a36ebd1..06542894f5ff 100644 --- a/include/dtrace/dtrace_impl.h +++ b/include/dtrace/dtrace_impl.h @@ -267,7 +267,7 @@ struct dtrace_state { uint32_t dts_reserve; cyclic_id_t dts_cleaner; cyclic_id_t dts_deadman; - uint64_t dts_laststatus; + ktime_t dts_laststatus; ktime_t dts_alive; char dts_speculates; char dts_destructive; diff --git a/include/dtrace/types.h b/include/dtrace/types.h index 367406f22eae..afa70e288336 100644 --- a/include/dtrace/types.h +++ b/include/dtrace/types.h @@ -41,7 +41,6 @@ #include #include -#include typedef unsigned char uchar_t; typedef unsigned int uint_t; @@ -118,8 +117,6 @@ typedef enum { #define ktime_gt(t0, t1) ((t0).tv64 > (t1).tv64) #define ktime_cp(t0, t1) ((t0).tv64 = (t1).tv64) -#define SECS_TO_JIFFIES(s) (((s) * SEC_CONVERSION) >> SEC_JIFFIE_SC) - /* * Translate between kernel config options and userspace-compatible definitions. */ diff --git a/include/linux/cyclic.h b/include/linux/cyclic.h index 5d8950222a3a..eec5317e42e8 100644 --- a/include/linux/cyclic.h +++ b/include/linux/cyclic.h @@ -16,7 +16,7 @@ typedef uintptr_t cyclic_id_t; typedef uint16_t cyc_level_t; -typedef void (*cyc_func_t)(uintptr_t, ktime_t); +typedef void (*cyc_func_t)(uintptr_t); #define CYCLIC_NONE ((cyclic_id_t)0) diff --git a/include/linux/dtrace_os.h b/include/linux/dtrace_os.h index 145e6783784f..31958bb84225 100644 --- a/include/linux/dtrace_os.h +++ b/include/linux/dtrace_os.h @@ -35,6 +35,9 @@ extern void dtrace_free_text(void *); extern void dtrace_enable(void); extern void dtrace_disable(void); +extern ktime_t dtrace_gethrtime(void); +extern ktime_t dtrace_getwalltime(void); + typedef enum dtrace_vtime_state { DTRACE_VTIME_INACTIVE = 0, DTRACE_VTIME_ACTIVE @@ -51,10 +54,6 @@ extern ktime_t dtrace_get_walltime(void); extern void dtrace_vtime_enable(void); extern void dtrace_vtime_disable(void); extern void dtrace_vtime_switch(struct task_struct *, struct task_struct *); -extern void dtrace_vtime_suspend(void); -extern void dtrace_vtime_resume(void); -extern void dtrace_chill(ktime_t, ktime_t, ktime_t); -extern ktime_t dtrace_gethrtime(void); extern void dtrace_skip_instruction(struct pt_regs *); diff --git a/kernel/dtrace/cyclic.c b/kernel/dtrace/cyclic.c index 822bb3c6fbea..18b71192ad9e 100644 --- a/kernel/dtrace/cyclic.c +++ b/kernel/dtrace/cyclic.c @@ -69,8 +69,7 @@ static void cyclic_fire(struct work_struct *work) * We know that the 'pend' counter for the cyclic is non-zero. * So, we can start with calling the handler at least once. */ - (*cyc->cyc.hdlr.cyh_func)(cyc->cyc.hdlr.cyh_arg, - ns_to_ktime(ktime_get_raw_fast_ns())); + (*cyc->cyc.hdlr.cyh_func)(cyc->cyc.hdlr.cyh_arg); again: /* @@ -125,8 +124,7 @@ static enum hrtimer_restart cyclic_expire(struct hrtimer *timr) * interrupt context. */ if (cyc->cyc.hdlr.cyh_level == CY_HIGH_LEVEL) { - (*cyc->cyc.hdlr.cyh_func)(cyc->cyc.hdlr.cyh_arg, - ns_to_ktime(ktime_get_raw_fast_ns())); + (*cyc->cyc.hdlr.cyh_func)(cyc->cyc.hdlr.cyh_arg); goto done; } @@ -212,11 +210,6 @@ cyclic_id_t cyclic_add(cyc_handler_t *hdlr, cyc_time_t *when) cyclic_restart(cyc); - /* - * Let the caller know when the cyclic was added. - */ - when->cyt_when = ns_to_ktime(ktime_get_raw_fast_ns()); - return (cyclic_id_t)cyc; } EXPORT_SYMBOL(cyclic_add); @@ -255,11 +248,6 @@ static void cyclic_omni_start(cyclic_t *omni, int cpu) cyc_time_t when; cyc_handler_t hdlr; - /* - * Let the caller know when the cyclic is being started. - */ - when.cyt_when = ns_to_ktime(ktime_get_raw_fast_ns()); - omni->omni.hdlr.cyo_online(omni->omni.hdlr.cyo_arg, cpu, &hdlr, &when); cyclic_add_pinned(cpu, omni, &hdlr, &when); } diff --git a/kernel/dtrace/dtrace_os.c b/kernel/dtrace/dtrace_os.c index 3ddd226fcb20..17e52e25df32 100644 --- a/kernel/dtrace/dtrace_os.c +++ b/kernel/dtrace/dtrace_os.c @@ -463,6 +463,19 @@ ktime_t dtrace_get_walltime(void) } EXPORT_SYMBOL(dtrace_get_walltime); +ktime_t dtrace_gethrtime(void) +{ + return ns_to_ktime(ktime_get_raw_fast_ns()); +} +EXPORT_SYMBOL(dtrace_gethrtime); + +/* Needed for lockstat probes where we cannot include ktime.h */ +u64 dtrace_gethrtime_ns(void) +{ + return ktime_get_raw_fast_ns(); +} +EXPORT_SYMBOL(dtrace_gethrtime_ns); + void dtrace_vtime_enable(void) { dtrace_vtime_state_t old, new; @@ -497,7 +510,7 @@ EXPORT_SYMBOL(dtrace_vtime_disable); void dtrace_vtime_switch(struct task_struct *prev, struct task_struct *next) { - ktime_t now = ns_to_ktime(ktime_get_raw_fast_ns()); + ktime_t now = dtrace_gethrtime(); if (ktime_nz(prev->dtrace_start)) { prev->dtrace_vtime = ktime_add(prev->dtrace_vtime, @@ -509,66 +522,6 @@ void dtrace_vtime_switch(struct task_struct *prev, struct task_struct *next) next->dtrace_start = now; } -void dtrace_vtime_suspend(void) -{ - ktime_t now = ns_to_ktime(ktime_get_raw_fast_ns()); - - current->dtrace_vtime = ktime_add(current->dtrace_vtime, - ktime_sub(now, current->dtrace_start)); - current->dtrace_start = now; -} -EXPORT_SYMBOL(dtrace_vtime_suspend); - -void dtrace_vtime_resume(void) -{ - current->dtrace_start = ns_to_ktime(ktime_get_raw_fast_ns()); -} -EXPORT_SYMBOL(dtrace_vtime_resume); - -#define ktime_lt(t0, t1) ((t0).tv64 < (t1).tv64) -#define ktime_gt(t0, t1) ((t0).tv64 > (t1).tv64) - -void dtrace_chill(ktime_t val, ktime_t interval, ktime_t int_max) -{ - ktime_t now = ns_to_ktime(ktime_get_raw_fast_ns()); - cpu_core_t *cpu = this_cpu_core; - volatile uint16_t *flags; - - flags = (volatile uint16_t *)&cpu->cpuc_dtrace_flags; - - if (ktime_gt(ktime_sub(now, cpu->cpu_dtrace_chillmark), interval)) { - cpu->cpu_dtrace_chillmark = now; - cpu->cpu_dtrace_chilled = ktime_set(0, 0); - } - - if (ktime_gt(ktime_add(cpu->cpu_dtrace_chilled, val), int_max) || - ktime_lt(ktime_add(cpu->cpu_dtrace_chilled, val), - cpu->cpu_dtrace_chilled)) { - *flags |= CPU_DTRACE_ILLOP; - return; - } - - while (ktime_lt(ktime_sub(ns_to_ktime(ktime_get_raw_fast_ns()), now), - val)) - continue; - - cpu->cpu_dtrace_chilled = ktime_add(cpu->cpu_dtrace_chilled, val); -} -EXPORT_SYMBOL(dtrace_chill); - -/* Needed for lockstat probes where we cannot include ktime.h */ -u64 dtrace_gethrtime_ns(void) -{ - return ktime_get_raw_fast_ns(); -} -EXPORT_SYMBOL(dtrace_gethrtime_ns); - -ktime_t dtrace_gethrtime(void) -{ - return ns_to_ktime(dtrace_gethrtime_ns()); -} -EXPORT_SYMBOL(dtrace_gethrtime); - void dtrace_stacktrace(stacktrace_state_t *st) { struct stack_trace trace;