From d007abf2161da09b5d631fa1b7bb65f9ddb7097e Mon Sep 17 00:00:00 2001 From: Kris Van Hees Date: Wed, 23 Jan 2013 04:53:14 -0500 Subject: [PATCH] USDT implementation (Phase 1). This rather large patch provides the implementation for USDT support at the DTrace kernel core level, and the fasttrap provider level. It ensures that executables can register their embedded providers (with USDT probes), that the probes are visible to the dtrace userspace utility, and that probes are properly removed upon executable completion, execve() invocation, or any unexpected executable termination. The following parts are provided by this patch: - meta-provider support (dtrace_ptofapi) - helper ioctl interface (dtrace_dev) - DIF validation for helper objects (dtrace_dif) - DOF processing for helpers for provider and probe definitions (dtrace_dof) - fasttrap meta-provider for USDT only (fasttrap*) The dtrace_helper.c file was removed because this code belongs in dtrace_dof.c instead. Minimal changes were made to the core kernel in exec.c, sched.h, exit.c, and fork.c to add support for process-specific helpers (and those encapsulate providers and probes). Signed-off-by: Kris Van Hees --- dtrace/dtrace.h | 50 +- dtrace/dtrace_dev.c | 228 +++++--- dtrace/dtrace_dif.c | 154 ++++- dtrace/dtrace_dof.c | 1177 ++++++++++++++++++++++++++++++++++++++- dtrace/dtrace_helper.c | 193 ------- dtrace/dtrace_ptofapi.c | 185 +++++- dtrace/fasttrap.h | 7 - dtrace/fasttrap_dev.c | 941 ++++++++++++++++++++++++++++++- dtrace/fasttrap_impl.h | 132 +++++ dtrace/fasttrap_mod.c | 14 +- 10 files changed, 2797 insertions(+), 284 deletions(-) delete mode 100644 dtrace/dtrace_helper.c delete mode 100644 dtrace/fasttrap.h create mode 100644 dtrace/fasttrap_impl.h diff --git a/dtrace/dtrace.h b/dtrace/dtrace.h index 2c4ee746904c..09868ac6d248 100644 --- a/dtrace/dtrace.h +++ b/dtrace/dtrace.h @@ -414,7 +414,7 @@ typedef struct dtrace_helper_provdesc { typedef struct dtrace_mops { void (*dtms_create_probe)(void *, void *, dtrace_helper_probedesc_t *); - void (*dtms_provide_pid)(void *, dtrace_helper_provdesc_t *, pid_t); + void *(*dtms_provide_pid)(void *, dtrace_helper_provdesc_t *, pid_t); void (*dtms_remove_pid)(void *, dtrace_helper_provdesc_t *, pid_t); } dtrace_mops_t; @@ -1203,10 +1203,19 @@ extern void dtrace_probekey(const dtrace_probedesc_t *, dtrace_probekey_t *); typedef uintptr_t dtrace_provider_id_t; typedef uintptr_t dtrace_meta_provider_id_t; +typedef struct dtrace_meta { + dtrace_mops_t dtm_mops; + char *dtm_name; + void *dtm_arg; + uint64_t dtm_count; +} dtrace_meta_t; + extern dtrace_provider_t *dtrace_provider; +extern dtrace_meta_t *dtrace_meta_pid; +extern dtrace_helpers_t *dtrace_deferred_pid; extern int dtrace_register(const char *, const dtrace_pattr_t *, uint32_t, - cred_t *, const dtrace_pops_t *, void *, + const cred_t *, const dtrace_pops_t *, void *, dtrace_provider_id_t *); extern int dtrace_unregister(dtrace_provider_id_t); extern void dtrace_invalidate(dtrace_provider_id_t); @@ -1543,6 +1552,7 @@ extern int dtrace_vcanload(void *, dtrace_diftype_t *, dtrace_mstate_t *, extern int dtrace_difo_validate(dtrace_difo_t *, dtrace_vstate_t *, uint_t, const cred_t *); +extern int dtrace_difo_validate_helper(dtrace_difo_t *); extern int dtrace_difo_cacheable(dtrace_difo_t *); extern void dtrace_difo_hold(dtrace_difo_t *); extern void dtrace_difo_init(dtrace_difo_t *, dtrace_vstate_t *); @@ -1576,6 +1586,7 @@ extern void dtrace_actdesc_release(dtrace_actdesc_t *, dtrace_vstate_t *); /* * DTrace Helper Functions */ +extern void dtrace_helpers_destroy(struct task_struct *); extern uint64_t dtrace_helper(int, dtrace_mstate_t *, dtrace_state_t *, uint64_t, uint64_t); @@ -2080,6 +2091,7 @@ typedef struct dof_xlref { uint32_t dofxr_argn; } dof_xlref_t; +extern void dtrace_dof_error(dof_hdr_t *, const char *); extern dof_hdr_t *dtrace_dof_create(dtrace_state_t *); extern dof_hdr_t *dtrace_dof_copyin(void __user *, int *); extern dof_hdr_t *dtrace_dof_property(const char *); @@ -2087,6 +2099,9 @@ extern void dtrace_dof_destroy(dof_hdr_t *); extern int dtrace_dof_slurp(dof_hdr_t *, dtrace_vstate_t *, const cred_t *, dtrace_enabling_t **, uint64_t, int); extern int dtrace_dof_options(dof_hdr_t *, dtrace_state_t *); +extern void dtrace_helper_provide(dof_helper_t *dhp, pid_t pid); +extern int dtrace_helper_slurp(dof_hdr_t *, dof_helper_t *); +extern int dtrace_helper_destroygen(int); /* * DTrace Anonymous Enabling Functions @@ -2200,6 +2215,37 @@ extern void ctf_forceload(void); module_init(name##_init); \ module_exit(name##_exit); +#define DT_META_PROVIDER_MODULE(name) \ + dtrace_meta_provider_id_t name##_id; \ + \ + static int __init name##_init(void) \ + { \ + int ret = 0; \ + \ + ret = name##_dev_init(); \ + if (ret) \ + goto failed; \ + \ + ret = dtrace_meta_register(__stringify(name), &name##_mops, \ + NULL, &name##_id); \ + if (ret) \ + goto failed; \ + \ + return 0; \ + \ + failed: \ + return ret; \ + } \ + \ + static void __exit name##_exit(void) \ + { \ + dtrace_meta_unregister(name##_id); \ + name##_dev_exit(); \ + } \ + \ + module_init(name##_init); \ + module_exit(name##_exit); + typedef struct dtrace_mprovider { char *dtmp_name; char *dtmp_pref; diff --git a/dtrace/dtrace_dev.c b/dtrace/dtrace_dev.c index 9ef630105a8e..f681ffe9afe4 100644 --- a/dtrace/dtrace_dev.c +++ b/dtrace/dtrace_dev.c @@ -93,6 +93,67 @@ int dtrace_enable_nullop(void) return 0; } +static int dtrace_open(struct inode *inode, struct file *file) +{ + dtrace_state_t *state; + uint32_t priv; + uid_t uid; + + dtrace_cred2priv(file->f_cred, &priv, &uid); + if (priv == DTRACE_PRIV_NONE) + return -EACCES; + + mutex_lock(&dtrace_provider_lock); + dtrace_probe_provide(NULL, NULL); + mutex_unlock(&dtrace_provider_lock); + + mutex_lock(&cpu_lock); + mutex_lock(&dtrace_lock); + dtrace_opens++; + dtrace_membar_producer(); + +#ifdef FIXME + /* + * Is this relevant for Linux? Is there an equivalent? + */ + if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) { + dtrace_opens--; + mutex_unlock(&cpu_lock); + mutex_unlock(&dtrace_lock); + return -EBUSY; + } +#endif + + state = dtrace_state_create(file); + mutex_unlock(&cpu_lock); + + if (state == NULL) { +#ifdef FIXME + if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL) + (void)kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); +#endif + + mutex_unlock(&dtrace_lock); + + return -EAGAIN; + } + + file->private_data = state; + + /* + * We only want to enable trap handling once, so we'll do it for the + * first open of the DTrace core device file. + * FIXME: If anonymous tracing is enabled, that would have enabled trap + * handling already, so we should not do it here again. + */ + if (dtrace_opens == 1) + dtrace_enable(); + + mutex_unlock(&dtrace_lock); + + return 0; +} + static long dtrace_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -849,67 +910,6 @@ static long dtrace_ioctl(struct file *file, return -ENOTTY; } -static int dtrace_open(struct inode *inode, struct file *file) -{ - dtrace_state_t *state; - uint32_t priv; - uid_t uid; - - dtrace_cred2priv(file->f_cred, &priv, &uid); - if (priv == DTRACE_PRIV_NONE) - return -EACCES; - - mutex_lock(&dtrace_provider_lock); - dtrace_probe_provide(NULL, NULL); - mutex_unlock(&dtrace_provider_lock); - - mutex_lock(&cpu_lock); - mutex_lock(&dtrace_lock); - dtrace_opens++; - dtrace_membar_producer(); - -#ifdef FIXME - /* - * Is this relevant for Linux? Is there an equivalent? - */ - if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) { - dtrace_opens--; - mutex_unlock(&cpu_lock); - mutex_unlock(&dtrace_lock); - return -EBUSY; - } -#endif - - state = dtrace_state_create(file); - mutex_unlock(&cpu_lock); - - if (state == NULL) { -#ifdef FIXME - if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL) - (void)kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); -#endif - - mutex_unlock(&dtrace_lock); - - return -EAGAIN; - } - - file->private_data = state; - - /* - * We only want to enable trap handling once, so we'll do it for the - * first open of the DTrace core device file. - * FIXME: If anonymous tracing is enabled, that would have enabled trap - * handling already, so we should not do it here again. - */ - if (dtrace_opens == 1) - dtrace_enable(); - - mutex_unlock(&dtrace_lock); - - return 0; -} - static int dtrace_close(struct inode *inode, struct file *file) { dtrace_state_t *state = file->private_data; @@ -950,6 +950,80 @@ static int dtrace_close(struct inode *inode, struct file *file) return 0; } +static int dtrace_helper_open(struct inode *inode, struct file *file) +{ + return 0; +} + +static long dtrace_helper_ioctl(struct file *file, + unsigned int cmd, unsigned long arg) +{ + int rval; + dof_helper_t help, *dhp = NULL; + void __user *argp = (void __user *)arg; + + switch (cmd) { + case DTRACEHIOC_ADDDOF: + if (copy_from_user(&help, argp, sizeof(help)) != 0) { + dtrace_dof_error(NULL, "failed to copy DOF helper"); + return -EFAULT; + } + + dhp = &help; + argp = (void __user *)help.dofhp_dof; + + /* fallthrough */ + + case DTRACEHIOC_ADD: { + dof_hdr_t *dof = dtrace_dof_copyin(argp, &rval); + + if (dof == NULL) + return rval; + + dt_dbg_ioctl("Helper IOCTL: %s\n", + cmd == DTRACEHIOC_ADD ? "AddProbe" : "AddDOF"); + + mutex_lock(&dtrace_lock); + + /* + * The dtrace_helper_slurp() routine takes responsibility for + * the dof -- it may free it now, or it may save it and free it + * later. + */ + if ((rval = dtrace_helper_slurp(dof, dhp)) == -1) + rval = -EINVAL; + + mutex_unlock(&dtrace_lock); + + dt_dbg_ioctl("Helper IOCTL: %s returning %d\n", + cmd == DTRACEHIOC_ADD ? "AddProbe" : "AddDOF", + rval); + + return rval; + } + + case DTRACEHIOC_REMOVE: + dt_dbg_ioctl("Helper IOCTL: Remove gen %ld\n", (uintptr_t)argp); + + mutex_lock(&dtrace_lock); + + rval = dtrace_helper_destroygen((uintptr_t)argp); + + mutex_unlock(&dtrace_lock); + + return rval; + default: + break; + } + + return -ENOTTY; +} + +static int dtrace_helper_close(struct inode *inode, struct file *file) +{ + return 0; +} + static const struct file_operations dtrace_fops = { .owner = THIS_MODULE, .unlocked_ioctl = dtrace_ioctl, @@ -957,6 +1031,13 @@ static const struct file_operations dtrace_fops = { .release = dtrace_close, }; +static const struct file_operations helper_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = dtrace_helper_ioctl, + .open = dtrace_helper_open, + .release = dtrace_helper_close, +}; + static struct miscdevice dtrace_dev = { .minor = DT_DEV_DTRACE_MINOR, .name = "dtrace", @@ -964,6 +1045,13 @@ static struct miscdevice dtrace_dev = { .fops = &dtrace_fops, }; +static struct miscdevice helper_dev = { + .minor = DT_DEV_HELPER_MINOR, + .name = "helper", + .nodename = "dtrace/helper", + .fops = &helper_fops, +}; + static void dtrace_module_loaded(struct module *module) { @@ -1213,13 +1301,28 @@ int dtrace_dev_init(void) return rc; } + /* + * Register the device for the DTrace helper. + */ + rc = misc_register(&helper_dev); + if (rc) { + pr_err("%s: Can't register misc device %d\n", + helper_dev.name, helper_dev.minor); + + mutex_unlock(&cpu_lock); + mutex_unlock(&dtrace_provider_lock); + mutex_unlock(&dtrace_lock); + + return rc; + } + ctf_forceload(); dtrace_modload = dtrace_module_loaded; dtrace_modunload = dtrace_module_unloaded; + dtrace_helpers_cleanup = dtrace_helpers_destroy; #ifdef FIXME dtrace_cpu_init = dtrace_cpu_setup_initial; - dtrace_helpers_cleanup = dtrace_helpers_destroy; dtrace_helpers_fork = dtrace_helpers_duplicate; dtrace_cpustart_init = dtrace_suspend; dtrace_cpustart_fini = dtrace_resume; @@ -1327,6 +1430,7 @@ int dtrace_dev_init(void) void dtrace_dev_exit(void) { kmem_cache_destroy(dtrace_state_cache); + misc_deregister(&helper_dev); misc_deregister(&dtrace_dev); dtrace_probe_exit(); diff --git a/dtrace/dtrace_dif.c b/dtrace/dtrace_dif.c index 572f37e37a37..c23b5f7c3f63 100644 --- a/dtrace/dtrace_dif.c +++ b/dtrace/dtrace_dif.c @@ -464,6 +464,158 @@ int dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, return err; } +/* + * Validate a DTrace DIF object that it is to be used as a helper. Helpers + * are much more constrained than normal DIFOs. Specifically, they may + * not: + * + * 1. Make calls to subroutines other than copyin(), copyinstr() or + * miscellaneous string routines + * 2. Access DTrace variables other than the args[] array, and the + * curthread, pid, ppid, tid, execname, zonename, uid and gid variables. + * 3. Have thread-local variables. + * 4. Have dynamic variables. + */ +int dtrace_difo_validate_helper(dtrace_difo_t *dp) +{ + int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err; + int err = 0; + uint_t pc; + + for (pc = 0; pc < dp->dtdo_len; pc++) { + dif_instr_t instr = dp->dtdo_buf[pc]; + uint_t v = DIF_INSTR_VAR(instr); + uint_t subr = DIF_INSTR_SUBR(instr); + uint_t op = DIF_INSTR_OP(instr); + + switch (op) { + case DIF_OP_OR: + case DIF_OP_XOR: + case DIF_OP_AND: + case DIF_OP_SLL: + case DIF_OP_SRL: + case DIF_OP_SRA: + case DIF_OP_SUB: + case DIF_OP_ADD: + case DIF_OP_MUL: + case DIF_OP_SDIV: + case DIF_OP_UDIV: + case DIF_OP_SREM: + case DIF_OP_UREM: + case DIF_OP_COPYS: + case DIF_OP_NOT: + case DIF_OP_MOV: + case DIF_OP_RLDSB: + case DIF_OP_RLDSH: + case DIF_OP_RLDSW: + case DIF_OP_RLDUB: + case DIF_OP_RLDUH: + case DIF_OP_RLDUW: + case DIF_OP_RLDX: + case DIF_OP_ULDSB: + case DIF_OP_ULDSH: + case DIF_OP_ULDSW: + case DIF_OP_ULDUB: + case DIF_OP_ULDUH: + case DIF_OP_ULDUW: + case DIF_OP_ULDX: + case DIF_OP_STB: + case DIF_OP_STH: + case DIF_OP_STW: + case DIF_OP_STX: + case DIF_OP_ALLOCS: + case DIF_OP_CMP: + case DIF_OP_SCMP: + case DIF_OP_TST: + case DIF_OP_BA: + case DIF_OP_BE: + case DIF_OP_BNE: + case DIF_OP_BG: + case DIF_OP_BGU: + case DIF_OP_BGE: + case DIF_OP_BGEU: + case DIF_OP_BL: + case DIF_OP_BLU: + case DIF_OP_BLE: + case DIF_OP_BLEU: + case DIF_OP_RET: + case DIF_OP_NOP: + case DIF_OP_POPTS: + case DIF_OP_FLUSHTS: + case DIF_OP_SETX: + case DIF_OP_SETS: + case DIF_OP_LDGA: + case DIF_OP_LDLS: + case DIF_OP_STGS: + case DIF_OP_STLS: + case DIF_OP_PUSHTR: + case DIF_OP_PUSHTV: + break; + + case DIF_OP_LDGS: + if (v >= DIF_VAR_OTHER_UBASE) + break; + + if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) + break; + + if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID || + v == DIF_VAR_PPID || v == DIF_VAR_TID || + v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME || + v == DIF_VAR_UID || v == DIF_VAR_GID) + break; + + err += efunc(pc, "illegal variable %u\n", v); + break; + + case DIF_OP_LDTA: + case DIF_OP_LDGAA: + case DIF_OP_LDTAA: + err += efunc(pc, "illegal dynamic variable load\n"); + break; + + case DIF_OP_STTS: + case DIF_OP_STGAA: + case DIF_OP_STTAA: + err += efunc(pc, "illegal dynamic variable store\n"); + break; + + case DIF_OP_CALL: + if (subr == DIF_SUBR_ALLOCA || + subr == DIF_SUBR_BCOPY || + subr == DIF_SUBR_COPYIN || + subr == DIF_SUBR_COPYINTO || + subr == DIF_SUBR_COPYINSTR || + subr == DIF_SUBR_INDEX || + subr == DIF_SUBR_INET_NTOA || + subr == DIF_SUBR_INET_NTOA6 || + subr == DIF_SUBR_INET_NTOP || + subr == DIF_SUBR_LLTOSTR || + subr == DIF_SUBR_RINDEX || + subr == DIF_SUBR_STRCHR || + subr == DIF_SUBR_STRJOIN || + subr == DIF_SUBR_STRRCHR || + subr == DIF_SUBR_STRSTR || + subr == DIF_SUBR_HTONS || + subr == DIF_SUBR_HTONL || + subr == DIF_SUBR_HTONLL || + subr == DIF_SUBR_NTOHS || + subr == DIF_SUBR_NTOHL || + subr == DIF_SUBR_NTOHLL) + break; + + err += efunc(pc, "invalid subr %u\n", subr); + break; + + default: + err += efunc(pc, "invalid opcode %u\n", + DIF_INSTR_OP(instr)); + } + } + + return err; +} + /* * Returns 1 if the expression in the DIF object can be cached on a per-thread * basis; 0 if not. @@ -521,7 +673,7 @@ int dtrace_difo_cacheable(dtrace_difo_t *dp) */ static void dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate) { - uint64_t sval; + uint64_t sval = 0; dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* + thread + id */ const dif_instr_t *text = dp->dtdo_buf; uint_t pc, srd = 0; diff --git a/dtrace/dtrace_dof.c b/dtrace/dtrace_dof.c index 999344536537..833213eed3e3 100644 --- a/dtrace/dtrace_dof.c +++ b/dtrace/dtrace_dof.c @@ -31,14 +31,31 @@ #include "dtrace.h" -size_t dtrace_difo_maxsize = 256 * 1024; -dtrace_optval_t dtrace_dof_maxsize = 256 * 1024; -size_t dtrace_actions_max = 16 * 1024; +size_t dtrace_difo_maxsize = 256 * 1024; +dtrace_optval_t dtrace_dof_maxsize = 256 * 1024; +size_t dtrace_actions_max = 16 * 1024; +dtrace_optval_t dtrace_helper_actions_max = 32; +dtrace_optval_t dtrace_helper_providers_max = 32; -static void dtrace_dof_error(dof_hdr_t *dof, const char *str) +static int dtrace_helpers; + +static uint32_t dtrace_helptrace_next = 0; +static uint32_t dtrace_helptrace_nlocals; +static char *dtrace_helptrace_buffer; +static int dtrace_helptrace_bufsize = 512 * 1024; + +#ifdef CONFIG_DT_DEBUG +static int dtrace_helptrace_enabled = 1; +#else +static int dtrace_helptrace_enabled = 0; +#endif + +void dtrace_dof_error(dof_hdr_t *dof, const char *str) { if (dtrace_err_verbose) pr_warning("failed to process DOF: %s", str); + else + dt_dbg_dof("Failed to process DOF: %s\n", str); #ifdef DTRACE_ERRDEBUG dtrace_errdebug(str); @@ -309,6 +326,10 @@ static dtrace_probedesc_t *dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name), min((size_t)DTRACE_NAMELEN - 1, size - probe->dofp_name)); + dt_dbg_dof(" ECB Probe %s:%s:%s:%s\n", + desc->dtpd_provider, desc->dtpd_mod, desc->dtpd_func, + desc->dtpd_name); + return desc; } @@ -816,6 +837,10 @@ int dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, const cred_t *cr, ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(dof->dofh_loadsz >= sizeof(dof_hdr_t)); + dt_dbg_dof(" DOF 0x%p Slurping...\n", dof); + + dt_dbg_dof(" DOF 0x%p Validating...\n", dof); + /* * Check the DOF header identification bytes. In addition to checking * valid settings, we also verify that unused bits/bytes are zeroed so @@ -903,6 +928,8 @@ int dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, const cred_t *cr, * the headers don't have stray offsets. If the 'noprobes' flag is * set, do not permit sections relating to providers, probes, or args. */ + dt_dbg_dof(" DOF 0x%p Checking section offsets...\n", dof); + for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(daddr + @@ -962,6 +989,8 @@ int dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, const cred_t *cr, * relocations that are present. We do this after the first pass to * be sure that all sections have had their headers validated. */ + dt_dbg_dof(" DOF 0x%p Performing relocations...\n", dof); + for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(daddr + @@ -982,6 +1011,8 @@ int dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, const cred_t *cr, } } + dt_dbg_dof(" DOF 0x%p Processing enablings...\n", dof); + if ((enab = *enabp) == NULL) enab = *enabp = dtrace_enabling_create(vstate); @@ -1068,3 +1099,1141 @@ int dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state) return 0; } + +static dtrace_helpers_t *dtrace_helpers_create(struct task_struct *curr) +{ + dtrace_helpers_t *dth; + + ASSERT(MUTEX_HELD(&dtrace_lock)); + ASSERT(curr->dtrace_helpers == NULL); + + dth = vzalloc(sizeof(dtrace_helpers_t)); + dth->dthps_actions = vzalloc(sizeof(dtrace_helper_action_t *) * + DTRACE_NHELPER_ACTIONS); + + curr->dtrace_helpers = dth; + dtrace_helpers++; + + dt_dbg_dof(" Helpers allocated for task 0x%p (%d system-wide)\n", + curr, dtrace_helpers); + + return dth; +} + +static int dtrace_helper_validate(dtrace_helper_action_t *helper) +{ + int err = 0, i; + dtrace_difo_t *dp; + + if ((dp = helper->dtha_predicate) != NULL) + err += dtrace_difo_validate_helper(dp); + + for (i = 0; i < helper->dtha_nactions; i++) + err += dtrace_difo_validate_helper(helper->dtha_actions[i]); + + return (err == 0); +} + +static int dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec) +{ + uintptr_t daddr = (uintptr_t)dof; + dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec; + dof_provider_t *prov; + dof_probe_t *prb; + uint8_t *arg; + char *strtab, *typestr; + dof_stridx_t typeidx; + size_t typesz; + uint_t nprobes, j, k; + + ASSERT(sec->dofs_type == DOF_SECT_PROVIDER); + + if (sec->dofs_offset & (sizeof(uint_t) - 1)) { + dtrace_dof_error(dof, "misaligned section offset"); + return -1; + } + + /* + * The section needs to be large enough to contain the DOF provider + * structure appropriate for the given version. + */ + if (sec->dofs_size < + ((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) + ? offsetof(dof_provider_t, dofpv_prenoffs) + : sizeof(dof_provider_t))) { + dtrace_dof_error(dof, "provider section too small"); + return -1; + } + + prov = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); + str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, prov->dofpv_strtab); + prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, prov->dofpv_probes); + arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, prov->dofpv_prargs); + off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, prov->dofpv_proffs); + + if (str_sec == NULL || prb_sec == NULL || + arg_sec == NULL || off_sec == NULL) + return -1; + + enoff_sec = NULL; + + if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && + prov->dofpv_prenoffs != DOF_SECT_NONE && + (enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS, + prov->dofpv_prenoffs)) == NULL) + return -1; + + strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); + + if (prov->dofpv_name >= str_sec->dofs_size || + strlen(strtab + prov->dofpv_name) >= DTRACE_PROVNAMELEN) { + dtrace_dof_error(dof, "invalid provider name"); + return -1; + } + + if (prb_sec->dofs_entsize == 0 || + prb_sec->dofs_entsize > prb_sec->dofs_size) { + dtrace_dof_error(dof, "invalid entry size"); + return -1; + } + + if (prb_sec->dofs_entsize & (sizeof(uintptr_t) - 1)) { + dtrace_dof_error(dof, "misaligned entry size"); + return -1; + } + + if (off_sec->dofs_entsize != sizeof(uint32_t)) { + dtrace_dof_error(dof, "invalid entry size"); + return -1; + } + + if (off_sec->dofs_offset & (sizeof(uint32_t) - 1)) { + dtrace_dof_error(dof, "misaligned section offset"); + return -1; + } + + if (arg_sec->dofs_entsize != sizeof(uint8_t)) { + dtrace_dof_error(dof, "invalid entry size"); + return -1; + } + + arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset); + nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize; + + dt_dbg_dof(" DOF 0x%p %s::: with %d probes\n", + dof, strtab + prov->dofpv_name, nprobes); + + /* + * Take a pass through the probes to check for errors. + */ + for (j = 0; j < nprobes; j++) { + prb = (dof_probe_t *)(uintptr_t) + (daddr + prb_sec->dofs_offset + + j * prb_sec->dofs_entsize); + + if (prb->dofpr_func >= str_sec->dofs_size) { + dtrace_dof_error(dof, "invalid function name"); + return -1; + } + + if (strlen(strtab + prb->dofpr_func) >= DTRACE_FUNCNAMELEN) { + dtrace_dof_error(dof, "function name too long"); + return -1; + } + + if (prb->dofpr_name >= str_sec->dofs_size || + strlen(strtab + prb->dofpr_name) >= DTRACE_NAMELEN) { + dtrace_dof_error(dof, "invalid probe name"); + return -1; + } + + /* + * The offset count must not wrap the index, and the offsets + * must also not overflow the section's data. + */ + if (prb->dofpr_offidx + prb->dofpr_noffs < prb->dofpr_offidx || + (prb->dofpr_offidx + prb->dofpr_noffs) * + off_sec->dofs_entsize > off_sec->dofs_size) { + dtrace_dof_error(dof, "invalid probe offset"); + return -1; + } + + if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) { + /* + * If there's no is-enabled offset section, make sure + * there aren't any is-enabled offsets. Otherwise + * perform the same checks as for probe offsets + * (immediately above). + */ + if (enoff_sec == NULL) { + if (prb->dofpr_enoffidx != 0 || + prb->dofpr_nenoffs != 0) { + dtrace_dof_error(dof, + "is-enabled offsets " + "with null section"); + return -1; + } + } else if (prb->dofpr_enoffidx + prb->dofpr_nenoffs < + prb->dofpr_enoffidx || + (prb->dofpr_enoffidx + prb->dofpr_nenoffs) * + enoff_sec->dofs_entsize > + enoff_sec->dofs_size) { + dtrace_dof_error(dof, "invalid is-enabled " + "offset"); + return -1; + } + + if (prb->dofpr_noffs + prb->dofpr_nenoffs == 0) { + dtrace_dof_error(dof, "zero probe and " + "is-enabled offsets"); + return -1; + } + } else if (prb->dofpr_noffs == 0) { + dtrace_dof_error(dof, "zero probe offsets"); + return -1; + } + + if (prb->dofpr_argidx + prb->dofpr_xargc < prb->dofpr_argidx || + (prb->dofpr_argidx + prb->dofpr_xargc) * + arg_sec->dofs_entsize > arg_sec->dofs_size) { + dtrace_dof_error(dof, "invalid args"); + return -1; + } + + typeidx = prb->dofpr_nargv; + typestr = strtab + prb->dofpr_nargv; + for (k = 0; k < prb->dofpr_nargc; k++) { + if (typeidx >= str_sec->dofs_size) { + dtrace_dof_error(dof, "bad native argument " + "type"); + return -1; + } + + typesz = strlen(typestr) + 1; + if (typesz > DTRACE_ARGTYPELEN) { + dtrace_dof_error(dof, "native argument type " + "too long"); + return -1; + } + + typeidx += typesz; + typestr += typesz; + } + + typeidx = prb->dofpr_xargv; + typestr = strtab + prb->dofpr_xargv; + for (k = 0; k < prb->dofpr_xargc; k++) { + if (arg[prb->dofpr_argidx + k] > prb->dofpr_nargc) { + dtrace_dof_error(dof, "bad native argument " + "index"); + return -1; + } + + if (typeidx >= str_sec->dofs_size) { + dtrace_dof_error(dof, "bad translated " + "argument type"); + return -1; + } + + typesz = strlen(typestr) + 1; + if (typesz > DTRACE_ARGTYPELEN) { + dtrace_dof_error(dof, "translated argument " + "type too long"); + return -1; + } + + typeidx += typesz; + typestr += typesz; + } + + dt_dbg_dof(" Probe %d %s:%s:%s:%s with %d offsets, " + "%d is-enabled offsets\n", j, + strtab + prov->dofpv_name, "", + strtab + prb->dofpr_func, strtab + prb->dofpr_name, + prb->dofpr_noffs, prb->dofpr_nenoffs); + } + + return 0; +} + +static void dtrace_helper_action_destroy(dtrace_helper_action_t *helper, + dtrace_vstate_t *vstate) +{ + int i; + + if (helper->dtha_predicate != NULL) + dtrace_difo_release(helper->dtha_predicate, vstate); + + for (i = 0; i < helper->dtha_nactions; i++) { + ASSERT(helper->dtha_actions[i] != NULL); + dtrace_difo_release(helper->dtha_actions[i], vstate); + } + + vfree(helper->dtha_actions); + vfree(helper); +} + +static int dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep) +{ + dtrace_helpers_t *dth; + dtrace_helper_action_t *helper, *last; + dtrace_actdesc_t *act; + dtrace_vstate_t *vstate; + dtrace_predicate_t *pred; + int count = 0, nactions = 0, i; + + if (which < 0 || which >= DTRACE_NHELPER_ACTIONS) + return -EINVAL; + + dth = current->dtrace_helpers; + last = dth->dthps_actions[which]; + vstate = &dth->dthps_vstate; + + for (count = 0; last != NULL; last = last->dtha_next) { + count++; + if (last->dtha_next == NULL) + break; + } + + /* + * If we already have dtrace_helper_actions_max helper actions for this + * helper action type, we'll refuse to add a new one. + */ + if (count >= dtrace_helper_actions_max) + return -ENOSPC; + + helper = vzalloc(sizeof(dtrace_helper_action_t)); + helper->dtha_generation = dth->dthps_generation; + + if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) { + ASSERT(pred->dtp_difo != NULL); + dtrace_difo_hold(pred->dtp_difo); + helper->dtha_predicate = pred->dtp_difo; + } + + for (act = ep->dted_action; act != NULL; act = act->dtad_next) { + if (act->dtad_kind != DTRACEACT_DIFEXPR) + goto err; + + if (act->dtad_difo == NULL) + goto err; + + nactions++; + } + + helper->dtha_actions = vzalloc(sizeof(dtrace_difo_t *) * + (helper->dtha_nactions = nactions)); + + for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) { + dtrace_difo_hold(act->dtad_difo); + helper->dtha_actions[i++] = act->dtad_difo; + } + + if (!dtrace_helper_validate(helper)) + goto err; + + if (last == NULL) + dth->dthps_actions[which] = helper; + else + last->dtha_next = helper; + + if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) { + dtrace_helptrace_nlocals = vstate->dtvs_nlocals; + dtrace_helptrace_next = 0; + } + + return 0; + +err: + dtrace_helper_action_destroy(helper, vstate); + return -EINVAL; +} + +static int dtrace_helper_provider_add(dof_helper_t *dofhp, int gen) +{ + dtrace_helpers_t *dth; + dtrace_helper_provider_t *hprov, **tmp_provs; + uint_t tmp_maxprovs, i; + + ASSERT(MUTEX_HELD(&dtrace_lock)); + + dth = current->dtrace_helpers; + ASSERT(dth != NULL); + + /* + * If we already have dtrace_helper_providers_max helper providers, + * we're refuse to add a new one. + */ + if (dth->dthps_nprovs >= dtrace_helper_providers_max) + return -ENOSPC; + + /* + * Check to make sure this isn't a duplicate. + */ + for (i = 0; i < dth->dthps_nprovs; i++) { + if (dofhp->dofhp_addr == + dth->dthps_provs[i]->dthp_prov.dofhp_addr) + return -EALREADY; + } + + hprov = vzalloc(sizeof(dtrace_helper_provider_t)); + hprov->dthp_prov = *dofhp; + hprov->dthp_ref = 1; + hprov->dthp_generation = gen; + + /* + * Allocate a bigger table for helper providers if it's already full. + */ + if (dth->dthps_maxprovs == dth->dthps_nprovs) { + tmp_maxprovs = dth->dthps_maxprovs; + tmp_provs = dth->dthps_provs; + + if (dth->dthps_maxprovs == 0) + dth->dthps_maxprovs = 2; + else + dth->dthps_maxprovs *= 2; + + if (dth->dthps_maxprovs > dtrace_helper_providers_max) + dth->dthps_maxprovs = dtrace_helper_providers_max; + + ASSERT(tmp_maxprovs < dth->dthps_maxprovs); + + dth->dthps_provs = vzalloc(dth->dthps_maxprovs * + sizeof(dtrace_helper_provider_t *)); + + if (tmp_provs != NULL) { + memcpy(dth->dthps_provs, tmp_provs, + tmp_maxprovs * + sizeof(dtrace_helper_provider_t *)); + vfree(tmp_provs); + } + } + + dth->dthps_provs[dth->dthps_nprovs] = hprov; + dth->dthps_nprovs++; + + return 0; +} + +static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov) +{ + mutex_lock(&dtrace_lock); + + if (--hprov->dthp_ref == 0) { + dof_hdr_t *dof; + + mutex_unlock(&dtrace_lock); + + dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof; + dtrace_dof_destroy(dof); + vfree(hprov); + } else + mutex_unlock(&dtrace_lock); +} + +static void dtrace_dofattr2attr(dtrace_attribute_t *attr, + const dof_attr_t dofattr) +{ + attr->dtat_name = DOF_ATTR_NAME(dofattr); + attr->dtat_data = DOF_ATTR_DATA(dofattr); + attr->dtat_class = DOF_ATTR_CLASS(dofattr); +} + +static void dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov, + const dof_provider_t *dofprov, char *strtab) +{ + hprov->dthpv_provname = strtab + dofprov->dofpv_name; + dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider, + dofprov->dofpv_provattr); + dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod, + dofprov->dofpv_modattr); + dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func, + dofprov->dofpv_funcattr); + dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name, + dofprov->dofpv_nameattr); + dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args, + dofprov->dofpv_argsattr); +} + +static void dtrace_helper_provider_remove_one(dof_helper_t *dhp, + dof_sec_t *sec, pid_t pid) +{ + uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; + dof_hdr_t *dof = (dof_hdr_t *)daddr; + dof_sec_t *str_sec; + dof_provider_t *prov; + char *strtab; + dtrace_helper_provdesc_t dhpv; + dtrace_meta_t *meta = dtrace_meta_pid; + dtrace_mops_t *mops = &meta->dtm_mops; + + prov = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); + str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + + prov->dofpv_strtab * + dof->dofh_secsize); + + strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); + + /* + * Create the provider. + */ + dtrace_dofprov2hprov(&dhpv, prov, strtab); + + dt_dbg_dof(" Removing provider %s for PID %d\n", + dhpv.dthpv_provname, pid); + + mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid); + + meta->dtm_count--; +} + +static void dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid) +{ + uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; + dof_hdr_t *dof = (dof_hdr_t *)daddr; + int i; + + ASSERT(MUTEX_HELD(&dtrace_meta_lock)); + + for (i = 0; i < dof->dofh_secnum; i++) { + dof_sec_t *sec = (dof_sec_t *)(uintptr_t) + (daddr + dof->dofh_secoff + + i * dof->dofh_secsize); + + if (sec->dofs_type != DOF_SECT_PROVIDER) + continue; + + dtrace_helper_provider_remove_one(dhp, sec, pid); + } +} + +static void dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, + pid_t pid) +{ + uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; + dof_hdr_t *dof = (dof_hdr_t *)daddr; + dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, + *enoff_sec; + dof_provider_t *prov; + dof_probe_t *probe; + uint32_t *off, *enoff; + uint8_t *arg; + char *strtab; + uint_t i, nprobes; + dtrace_helper_provdesc_t dhpv; + dtrace_helper_probedesc_t dhpb; + dtrace_meta_t *meta = dtrace_meta_pid; + dtrace_mops_t *mops = &meta->dtm_mops; + void *parg; + + prov = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); + str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + + prov->dofpv_strtab * + dof->dofh_secsize); + prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + + prov->dofpv_probes * + dof->dofh_secsize); + arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + + prov->dofpv_prargs * + dof->dofh_secsize); + off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + + prov->dofpv_proffs * + dof->dofh_secsize); + + strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); + off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset); + arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset); + enoff = NULL; + + /* + * See dtrace_helper_provider_validate(). + */ + if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && + prov->dofpv_prenoffs != DOF_SECT_NONE) { + enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + + prov->dofpv_proffs * + dof->dofh_secsize); + enoff = (uint32_t *)(uintptr_t)(daddr + + enoff_sec->dofs_offset); + } + + nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize; + + /* + * Create the provider. + */ + dtrace_dofprov2hprov(&dhpv, prov, strtab); + + if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL) + return; + + dt_dbg_dof(" Created provider %s for PID %d\n", + strtab + prov->dofpv_name, pid); + + meta->dtm_count++; + + /* + * Create the probes. + */ + for (i = 0; i < nprobes; i++) { + probe = (dof_probe_t *)(uintptr_t)(daddr + + prb_sec->dofs_offset + + i * prb_sec->dofs_entsize); + + dhpb.dthpb_mod = dhp->dofhp_mod; + dhpb.dthpb_func = strtab + probe->dofpr_func; + dhpb.dthpb_name = strtab + probe->dofpr_name; + dhpb.dthpb_base = probe->dofpr_addr; + dhpb.dthpb_offs = off + probe->dofpr_offidx; + dhpb.dthpb_noffs = probe->dofpr_noffs; + + if (enoff != NULL) { + dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx; + dhpb.dthpb_nenoffs = probe->dofpr_nenoffs; + } else { + dhpb.dthpb_enoffs = NULL; + dhpb.dthpb_nenoffs = 0; + } + + dhpb.dthpb_args = arg + probe->dofpr_argidx; + dhpb.dthpb_nargc = probe->dofpr_nargc; + dhpb.dthpb_xargc = probe->dofpr_xargc; + dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv; + dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv; + + mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb); + + dt_dbg_dof(" Created probe %s:%s:%s:%s\n", + strtab + prov->dofpv_name, "", dhpb.dthpb_func, + dhpb.dthpb_name); + } +} + +void dtrace_helper_provide(dof_helper_t *dhp, pid_t pid) +{ + uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; + dof_hdr_t *dof = (dof_hdr_t *)daddr; + int i; + + ASSERT(MUTEX_HELD(&dtrace_meta_lock)); + + for (i = 0; i < dof->dofh_secnum; i++) { + dof_sec_t *sec = (dof_sec_t *)(uintptr_t) + (daddr + dof->dofh_secoff + + i * dof->dofh_secsize); + + if (sec->dofs_type != DOF_SECT_PROVIDER) + continue; + + dtrace_helper_provide_one(dhp, sec, pid); + } + + /* + * We may have just created probes, so we must now rematch against any + * retained enablings. Note that this call will acquire both cpu_lock + * and dtrace_lock; the fact that we are holding dtrace_meta_lock now + * is what defines the ordering with respect to these three locks. + */ + dt_dbg_dof(" Re-matching against any retained enablings\n"); + dtrace_enabling_matchall(); +} + +static void dtrace_helper_provider_register(struct task_struct *curr, + dtrace_helpers_t *dth, + dof_helper_t *dofhp) +{ + ASSERT(!MUTEX_HELD(&dtrace_lock)); + + mutex_lock(&dtrace_meta_lock); + mutex_lock(&dtrace_lock); + + if (!dtrace_attached() || dtrace_meta_pid == NULL) { + dt_dbg_dof(" No meta provider registered -- deferred\n"); + + /* + * If the dtrace module is loaded but not attached, or if there + * isn't a meta provider registered to deal with these provider + * descriptions, we need to postpone creating the actual + * providers until later. + */ + if (dth->dthps_next == NULL && dth->dthps_prev == NULL && + dtrace_deferred_pid != dth) { + dth->dthps_deferred = 1; + dth->dthps_pid = current->pid; + dth->dthps_next = dtrace_deferred_pid; + dth->dthps_prev = NULL; + if (dtrace_deferred_pid != NULL) + dtrace_deferred_pid->dthps_prev = dth; + dtrace_deferred_pid = dth; + } + + mutex_unlock(&dtrace_lock); + } else if (dofhp != NULL) { + /* + * If the dtrace module is loaded and we have a particular + * helper provider description, pass that off to the meta + * provider. + */ + mutex_unlock(&dtrace_lock); + + dtrace_helper_provide(dofhp, current->pid); + } else { + /* + * Otherwise, just pass all the helper provider descriptions + * off to the meta provider. + */ + int i; + + mutex_unlock(&dtrace_lock); + + for (i = 0; i < dth->dthps_nprovs; i++) { + dtrace_helper_provide(&dth->dthps_provs[i]->dthp_prov, + current->pid); + } + } + + mutex_unlock(&dtrace_meta_lock); +} + +int dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp) +{ + dtrace_helpers_t *dth; + dtrace_vstate_t *vstate; + dtrace_enabling_t *enab = NULL; + int i, gen, rv; + int nhelpers = 0, nprovs = 0,destroy = 1; + uintptr_t daddr = (uintptr_t)dof; + + ASSERT(MUTEX_HELD(&dtrace_lock)); + + if ((dth = current->dtrace_helpers) == NULL) + dth = dtrace_helpers_create(current); + + vstate = &dth->dthps_vstate; + + if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab, + dhp != NULL ? dhp->dofhp_addr : 0, + FALSE)) != 0) { + dtrace_dof_destroy(dof); + return rv; + } + + /* + * Look for helper providers and validate their descriptions. + */ + if (dhp != NULL) { + dt_dbg_dof(" DOF 0x%p Validating providers...\n", dof); + + for (i = 0; i < dof->dofh_secnum; i++) { + dof_sec_t *sec = (dof_sec_t *)(uintptr_t) + (daddr + dof->dofh_secoff + + i * dof->dofh_secsize); + + if (sec->dofs_type != DOF_SECT_PROVIDER) + continue; + + if (dtrace_helper_provider_validate(dof, sec) != 0) { + dtrace_enabling_destroy(enab); + dtrace_dof_destroy(dof); + return -1; + } + + nprovs++; + } + } + + /* + * Now we need to walk through the ECB descriptions in the enabling. + */ + for (i = 0; i < enab->dten_ndesc; i++) { + dtrace_ecbdesc_t *ep = enab->dten_desc[i]; + dtrace_probedesc_t *desc = &ep->dted_probe; + + dt_dbg_dof(" ECB Desc %s:%s:%s:%s\n", + desc->dtpd_provider, desc->dtpd_mod, + desc->dtpd_func, desc->dtpd_name); + if (strcmp(desc->dtpd_provider, "dtrace") != 0) + continue; + + if (strcmp(desc->dtpd_mod, "helper") != 0) + continue; + + if (strcmp(desc->dtpd_func, "ustack") != 0) + continue; + + if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK, + ep)) != 0) { + /* + * Adding this helper action failed -- we are now going + * to rip out the entire generation and return failure. + */ + dtrace_helper_destroygen(dth->dthps_generation); + dtrace_enabling_destroy(enab); + dtrace_dof_destroy(dof); + return -1; + } + + nhelpers++; + } + + if (nhelpers < enab->dten_ndesc) + dtrace_dof_error(dof, "unmatched helpers"); + + gen = dth->dthps_generation++; + dtrace_enabling_destroy(enab); + + if (dhp != NULL && nprovs > 0) { + dt_dbg_dof(" DOF 0x%p Adding and registering providers\n", + dof); + + dhp->dofhp_dof = (uint64_t)(uintptr_t)dof; + if (dtrace_helper_provider_add(dhp, gen) == 0) { + mutex_unlock(&dtrace_lock); + dtrace_helper_provider_register(current, dth, dhp); + mutex_lock(&dtrace_lock); + + destroy = 0; + } + } + + if (destroy) + dtrace_dof_destroy(dof); + + return gen; +} + +void dtrace_helpers_destroy(struct task_struct *tsk) +{ + dtrace_helpers_t *help; + dtrace_vstate_t *vstate; + int i; + + mutex_lock(&dtrace_lock); + + ASSERT(tsk->dtrace_helpers != NULL); + ASSERT(dtrace_helpers > 0); + + dt_dbg_dof("Helper cleanup: PID %d\n", tsk->pid); + + help = tsk->dtrace_helpers; + vstate = &help->dthps_vstate; + + /* + * We're now going to lose the help from this process. + */ + tsk->dtrace_helpers = NULL; + dtrace_sync(); + + /* + * Destory the helper actions. + */ + for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { + dtrace_helper_action_t *h, *next; + + for (h = help->dthps_actions[i]; h != NULL; h = next) { + next = h->dtha_next; + dtrace_helper_action_destroy(h, vstate); + h = next; + } + } + + mutex_unlock(&dtrace_lock); + + /* + * Destroy the helper providers. + */ + if (help->dthps_maxprovs > 0) { + mutex_lock(&dtrace_meta_lock); + if (dtrace_meta_pid != NULL) { + ASSERT(dtrace_deferred_pid == NULL); + + for (i = 0; i < help->dthps_nprovs; i++) { + dtrace_helper_provider_remove( + &help->dthps_provs[i]->dthp_prov, + tsk->pid); + } + } else { + mutex_lock(&dtrace_lock); + ASSERT(help->dthps_deferred == 0 || + help->dthps_next != NULL || + help->dthps_prev != NULL || + help == dtrace_deferred_pid); + + /* + * Remove the helper from the deferred list. + */ + if (help->dthps_next != NULL) + help->dthps_next->dthps_prev = help->dthps_prev; + if (help->dthps_prev != NULL) + help->dthps_prev->dthps_next = help->dthps_next; + if (dtrace_deferred_pid == help) { + dtrace_deferred_pid = help->dthps_next; + ASSERT(help->dthps_prev == NULL); + } + + mutex_unlock(&dtrace_lock); + } + + mutex_unlock(&dtrace_meta_lock); + + for (i = 0; i < help->dthps_nprovs; i++) + dtrace_helper_provider_destroy(help->dthps_provs[i]); + + vfree(help->dthps_provs); + } + + mutex_lock(&dtrace_lock); + + dtrace_vstate_fini(&help->dthps_vstate); + vfree(help->dthps_actions); + vfree(help); + + --dtrace_helpers; + mutex_unlock(&dtrace_lock); +} + +int dtrace_helper_destroygen(int gen) +{ + struct task_struct *p = current; + dtrace_helpers_t *dth = p->dtrace_helpers; + dtrace_vstate_t *vstate; + int i; + + ASSERT(MUTEX_HELD(&dtrace_lock)); + + if (dth == NULL || gen > dth->dthps_generation) + return -EINVAL; + + vstate = &dth->dthps_vstate; + + for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { + dtrace_helper_action_t *last = NULL, *h, *next; + + for (h = dth->dthps_actions[i]; h != NULL; h = next) { + next = h->dtha_next; + + dt_dbg_dof(" Comparing action (agen %d vs rgen %d)\n", + h->dtha_generation, gen); + + if (h->dtha_generation == gen) { + if (last != NULL) + last->dtha_next = next; + else + dth->dthps_actions[i] = next; + + dtrace_helper_action_destroy(h, vstate); + } else + last = h; + } + } + + /* + * Iterate until we've cleared out all helper providers with the given + * generation number. + */ + for (;;) { + dtrace_helper_provider_t *prov = NULL; + + /* + * Look for a helper provider with the right generation. We + * have to start back at the beginning of the list each time + * because we drop dtrace_lock. It's unlikely that we'll make + * more than two passes. + */ + for (i = 0; i < dth->dthps_nprovs; i++) { + prov = dth->dthps_provs[i]; + + if (prov->dthp_generation == gen) + break; + } + + /* + * If there were no matches, we are done. + */ + if (i == dth->dthps_nprovs) + break; + + dt_dbg_dof(" Found provider with gen %d\n", gen); + + /* + * Move the last helper provider into this slot. + */ + dth->dthps_nprovs--; + dth->dthps_provs[i] = dth->dthps_provs[dth->dthps_nprovs]; + dth->dthps_provs[dth->dthps_nprovs] = NULL; + + mutex_unlock(&dtrace_lock); + + /* + * If we have a meta provider, remove this helper provider. + */ + mutex_lock(&dtrace_meta_lock); + + if (dtrace_meta_pid != NULL) { + ASSERT(dtrace_deferred_pid == NULL); + + dtrace_helper_provider_remove(&prov->dthp_prov, + p->pid); + } + + mutex_unlock(&dtrace_meta_lock); + + dtrace_helper_provider_destroy(prov); + + mutex_lock(&dtrace_lock); + } + + return 0; +} + +static void dtrace_helper_trace(dtrace_helper_action_t *helper, + dtrace_mstate_t *mstate, + dtrace_vstate_t *vstate, int where) +{ + uint32_t size, next, nnext, i; + dtrace_helptrace_t *ent; + uint16_t flags = this_cpu_core->cpuc_dtrace_flags; + + if (!dtrace_helptrace_enabled) + return; + + ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals); + + /* + * What would a tracing framework be without its own tracing + * framework? (Well, a hell of a lot simpler, for starters...) + */ + size = sizeof(dtrace_helptrace_t) + dtrace_helptrace_nlocals * + sizeof(uint64_t) - sizeof(uint64_t); + + /* + * Iterate until we can allocate a slot in the trace buffer. + */ + do { + next = dtrace_helptrace_next; + + if (next + size < dtrace_helptrace_bufsize) + nnext = next + size; + else + nnext = size; + } while (cmpxchg(&dtrace_helptrace_next, next, nnext) != next); + + /* + * We have our slot; fill it in. + */ + if (nnext == size) + next = 0; + + ent = (dtrace_helptrace_t *)&dtrace_helptrace_buffer[next]; + ent->dtht_helper = helper; + ent->dtht_where = where; + ent->dtht_nlocals = vstate->dtvs_nlocals; + + ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) + ? mstate->dtms_fltoffs + : -1; + ent->dtht_fault = DTRACE_FLAGS2FLT(flags); + ent->dtht_illval = this_cpu_core->cpuc_dtrace_illval; + + for (i = 0; i < vstate->dtvs_nlocals; i++) { + dtrace_statvar_t *svar; + + if ((svar = vstate->dtvs_locals[i]) == NULL) + continue; + + ASSERT(svar->dtsv_size >= NR_CPUS * sizeof(uint64_t)); + ent->dtht_locals[i] = + ((uint64_t *)(uintptr_t)svar->dtsv_data)[ + smp_processor_id()]; + } +} + +uint64_t dtrace_helper(int which, dtrace_mstate_t *mstate, + dtrace_state_t *state, uint64_t arg0, uint64_t arg1) +{ + uint16_t *flags = &this_cpu_core->cpuc_dtrace_flags; + uint64_t sarg0 = mstate->dtms_arg[0]; + uint64_t sarg1 = mstate->dtms_arg[1]; + uint64_t rval = 0; + dtrace_helpers_t *helpers = current->dtrace_helpers; + dtrace_helper_action_t *helper; + dtrace_vstate_t *vstate; + dtrace_difo_t *pred; + int i, trace = dtrace_helptrace_enabled; + + ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS); + + if (helpers == NULL) + return 0; + + if ((helper = helpers->dthps_actions[which]) == NULL) + return 0; + + vstate = &helpers->dthps_vstate; + mstate->dtms_arg[0] = arg0; + mstate->dtms_arg[1] = arg1; + + /* + * Now iterate over each helper. If its predicate evaluates to 'true', + * we'll call the corresponding actions. Note that the below calls + * to dtrace_dif_emulate() may set faults in machine state. This is + * okay: our caller (the outer dtrace_dif_emulate()) will simply plow + * the stored DIF offset with its own (which is the desired behavior). + * Also, note the calls to dtrace_dif_emulate() may allocate scratch + * from machine state; this is okay, too. + */ + for (; helper != NULL; helper = helper->dtha_next) { + if ((pred = helper->dtha_predicate) != NULL) { + if (trace) + dtrace_helper_trace(helper, mstate, vstate, 0); + + if (!dtrace_dif_emulate(pred, mstate, vstate, state)) + goto next; + + if (*flags & CPU_DTRACE_FAULT) + goto err; + } + + for (i = 0; i < helper->dtha_nactions; i++) { + if (trace) + dtrace_helper_trace(helper, mstate, vstate, + i + 1); + + rval = dtrace_dif_emulate(helper->dtha_actions[i], + mstate, vstate, state); + + if (*flags & CPU_DTRACE_FAULT) + goto err; + } + +next: + if (trace) + dtrace_helper_trace(helper, mstate, vstate, + DTRACE_HELPTRACE_NEXT); + } + + if (trace) + dtrace_helper_trace(helper, mstate, vstate, + DTRACE_HELPTRACE_DONE); + + /* + * Restore the arg0 that we saved upon entry. + */ + mstate->dtms_arg[0] = sarg0; + mstate->dtms_arg[1] = sarg1; + + return rval; + +err: + if (trace) + dtrace_helper_trace(helper, mstate, vstate, + DTRACE_HELPTRACE_ERR); + + /* + * Restore the arg0 that we saved upon entry. + */ + mstate->dtms_arg[0] = sarg0; + mstate->dtms_arg[1] = sarg1; + + return 0; +} diff --git a/dtrace/dtrace_helper.c b/dtrace/dtrace_helper.c deleted file mode 100644 index 589fd4add1d1..000000000000 --- a/dtrace/dtrace_helper.c +++ /dev/null @@ -1,193 +0,0 @@ -/* - * FILE: dtrace_helper.c - * DESCRIPTION: Dynamic Tracing: helper functions - * - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - * - * Copyright 2010, 2011 Oracle, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include - -#include "dtrace.h" - -static uint32_t dtrace_helptrace_next = 0; -static uint32_t dtrace_helptrace_nlocals; -static char *dtrace_helptrace_buffer; -static int dtrace_helptrace_bufsize = 512 * 1024; - -#ifdef CONFIG_DT_DEBUG -static int dtrace_helptrace_enabled = 1; -#else -static int dtrace_helptrace_enabled = 0; -#endif - -static void dtrace_helper_trace(dtrace_helper_action_t *helper, - dtrace_mstate_t *mstate, - dtrace_vstate_t *vstate, int where) -{ - uint32_t size, next, nnext, i; - dtrace_helptrace_t *ent; - uint16_t flags = this_cpu_core->cpuc_dtrace_flags; - - if (!dtrace_helptrace_enabled) - return; - - ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals); - - /* - * What would a tracing framework be without its own tracing - * framework? (Well, a hell of a lot simpler, for starters...) - */ - size = sizeof(dtrace_helptrace_t) + dtrace_helptrace_nlocals * - sizeof(uint64_t) - sizeof(uint64_t); - - /* - * Iterate until we can allocate a slot in the trace buffer. - */ - do { - next = dtrace_helptrace_next; - - if (next + size < dtrace_helptrace_bufsize) - nnext = next + size; - else - nnext = size; - } while (cmpxchg(&dtrace_helptrace_next, next, nnext) != next); - - /* - * We have our slot; fill it in. - */ - if (nnext == size) - next = 0; - - ent = (dtrace_helptrace_t *)&dtrace_helptrace_buffer[next]; - ent->dtht_helper = helper; - ent->dtht_where = where; - ent->dtht_nlocals = vstate->dtvs_nlocals; - - ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) - ? mstate->dtms_fltoffs - : -1; - ent->dtht_fault = DTRACE_FLAGS2FLT(flags); - ent->dtht_illval = this_cpu_core->cpuc_dtrace_illval; - - for (i = 0; i < vstate->dtvs_nlocals; i++) { - dtrace_statvar_t *svar; - - if ((svar = vstate->dtvs_locals[i]) == NULL) - continue; - - ASSERT(svar->dtsv_size >= NR_CPUS * sizeof(uint64_t)); - ent->dtht_locals[i] = - ((uint64_t *)(uintptr_t)svar->dtsv_data)[ - smp_processor_id()]; - } -} - -uint64_t dtrace_helper(int which, dtrace_mstate_t *mstate, - dtrace_state_t *state, uint64_t arg0, uint64_t arg1) -{ - uint16_t *flags = &this_cpu_core->cpuc_dtrace_flags; - uint64_t sarg0 = mstate->dtms_arg[0]; - uint64_t sarg1 = mstate->dtms_arg[1]; - uint64_t rval = 0; - dtrace_helpers_t *helpers = current->dtrace_helpers; - dtrace_helper_action_t *helper; - dtrace_vstate_t *vstate; - dtrace_difo_t *pred; - int i, trace = dtrace_helptrace_enabled; - - ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS); - - if (helpers == NULL) - return 0; - - if ((helper = helpers->dthps_actions[which]) == NULL) - return 0; - - vstate = &helpers->dthps_vstate; - mstate->dtms_arg[0] = arg0; - mstate->dtms_arg[1] = arg1; - - /* - * Now iterate over each helper. If its predicate evaluates to 'true', - * we'll call the corresponding actions. Note that the below calls - * to dtrace_dif_emulate() may set faults in machine state. This is - * okay: our caller (the outer dtrace_dif_emulate()) will simply plow - * the stored DIF offset with its own (which is the desired behavior). - * Also, note the calls to dtrace_dif_emulate() may allocate scratch - * from machine state; this is okay, too. - */ - for (; helper != NULL; helper = helper->dtha_next) { - if ((pred = helper->dtha_predicate) != NULL) { - if (trace) - dtrace_helper_trace(helper, mstate, vstate, 0); - - if (!dtrace_dif_emulate(pred, mstate, vstate, state)) - goto next; - - if (*flags & CPU_DTRACE_FAULT) - goto err; - } - - for (i = 0; i < helper->dtha_nactions; i++) { - if (trace) - dtrace_helper_trace(helper, mstate, vstate, - i + 1); - - rval = dtrace_dif_emulate(helper->dtha_actions[i], - mstate, vstate, state); - - if (*flags & CPU_DTRACE_FAULT) - goto err; - } - -next: - if (trace) - dtrace_helper_trace(helper, mstate, vstate, - DTRACE_HELPTRACE_NEXT); - } - - if (trace) - dtrace_helper_trace(helper, mstate, vstate, - DTRACE_HELPTRACE_DONE); - - /* - * Restore the arg0 that we saved upon entry. - */ - mstate->dtms_arg[0] = sarg0; - mstate->dtms_arg[1] = sarg1; - - return rval; - -err: - if (trace) - dtrace_helper_trace(helper, mstate, vstate, - DTRACE_HELPTRACE_ERR); - - /* - * Restore the arg0 that we saved upon entry. - */ - mstate->dtms_arg[0] = sarg0; - mstate->dtms_arg[1] = sarg1; - - return 0; -} diff --git a/dtrace/dtrace_ptofapi.c b/dtrace/dtrace_ptofapi.c index 5c388ceb4cf2..ed3645494c7e 100644 --- a/dtrace/dtrace_ptofapi.c +++ b/dtrace/dtrace_ptofapi.c @@ -32,6 +32,8 @@ #include "dtrace.h" dtrace_provider_t *dtrace_provider; +dtrace_meta_t *dtrace_meta_pid; +dtrace_helpers_t *dtrace_deferred_pid; DEFINE_MUTEX(dtrace_lock); DEFINE_MUTEX(dtrace_provider_lock); @@ -42,7 +44,7 @@ DEFINE_MUTEX(dtrace_meta_lock); * be called by providers during module initialization. */ int dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv, - cred_t *cr, const dtrace_pops_t *pops, void *arg, + const cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp) { dtrace_provider_t *provider; @@ -201,8 +203,9 @@ static int dtrace_unregister_check(int id, void *p, void *data) } /* - * Remove the given probe from the hash tables and the probe IDR. The probes - * are chained for further processing. + * Remove the given probe from the hash tables and the probe IDR, if it is + * associated with the given provider. The probes are chained for further + * processing. */ static int dtrace_unregister_probe(int id, void *p, void *data) { @@ -227,6 +230,37 @@ static int dtrace_unregister_probe(int id, void *p, void *data) return 0; } +/* + * Remove the given probe from the hash tables and the probe IDR, if it is + * associated with the given provider and if it does not have any enablings. + * The probes are chained for further processing. + */ +static int dtrace_condense_probe(int id, void *p, void *data) +{ + dtrace_probe_t *probe = (dtrace_probe_t *)p; + struct unreg_state *st = (struct unreg_state *)data; + + if (probe->dtpr_provider != st->prov) + return 0; + + if (probe->dtpr_ecb == NULL) + return 0; + + dtrace_hash_remove(dtrace_bymod, probe); + dtrace_hash_remove(dtrace_byfunc, probe); + dtrace_hash_remove(dtrace_byname, probe); + + if (st->first == NULL) { + st->first = probe; + probe->dtpr_nextmod = NULL; + } else { + probe->dtpr_nextmod = st->first; + st->first = probe; + } + + return 0; +} + /* * Unregister the specified provider from the DTrace core. This should be * called by provider during module cleanup. @@ -410,8 +444,11 @@ EXPORT_SYMBOL(dtrace_attached); int dtrace_condense(dtrace_provider_id_t id) { dtrace_provider_t *prov = (dtrace_provider_t *)id; - int i; dtrace_probe_t *probe; + struct unreg_state st = { + prov, + NULL + }; /* * Make sure this isn't the DTrace provider itself. @@ -422,7 +459,35 @@ int dtrace_condense(dtrace_provider_id_t id) mutex_lock(&dtrace_provider_lock); mutex_lock(&dtrace_lock); - /* FIXME - INCOMPLETE */ + /* + * Attempt to destroy the probes associated with this provider. + */ + dtrace_probe_for_each(dtrace_condense_probe, &st); + + /* + * The probes associated with the provider have been removed. Ensure + * synchronization on probe IDR processing. + */ + dtrace_sync(); + + /* + * Now get rid of the actual probes. + */ + for (probe = st.first; probe != NULL; probe = st.first) { + int probe_id = probe->dtpr_id; + + st.first = probe->dtpr_nextmod; + + prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe_id, + probe->dtpr_arg); + + kfree(probe->dtpr_mod); + kfree(probe->dtpr_func); + kfree(probe->dtpr_name); + kfree(probe); + + dtrace_probe_remove_id(probe_id); + } mutex_unlock(&dtrace_lock); mutex_unlock(&dtrace_provider_lock); @@ -430,3 +495,113 @@ int dtrace_condense(dtrace_provider_id_t id) return 0; } EXPORT_SYMBOL(dtrace_condense); + +int dtrace_meta_register(const char *name, const dtrace_mops_t *mops, + void *arg, dtrace_meta_provider_id_t *idp) +{ + dtrace_meta_t *meta; + dtrace_helpers_t *help, *next; + int i; + + *idp = DTRACE_METAPROVNONE; + + /* + * We strictly don't need the name, but we hold onto it for + * debuggability. All hail error queues! + */ + if (name == NULL) { + pr_warn("failed to register meta-provider: invalid name\n"); + return -EINVAL; + } + + if (mops == NULL || + mops->dtms_create_probe == NULL || + mops->dtms_provide_pid == NULL || + mops->dtms_remove_pid == NULL) { + pr_warn("failed to register meta-register %s: invalid ops\n", + name); + return -EINVAL; + } + + meta = kzalloc(sizeof(dtrace_meta_t), GFP_KERNEL); + meta->dtm_mops = *mops; + meta->dtm_name = kmalloc(strlen(name) + 1, GFP_KERNEL); + strcpy(meta->dtm_name, name); + meta->dtm_arg = arg; + + mutex_lock(&dtrace_meta_lock); + mutex_lock(&dtrace_lock); + + if (dtrace_meta_pid != NULL) { + mutex_unlock(&dtrace_lock); + mutex_unlock(&dtrace_meta_lock); + pr_warn("failed to register meta-register %s: user-land " + "meta-provider exists", name); + kfree(meta->dtm_name); + kfree(meta); + return -EINVAL; + } + + dtrace_meta_pid = meta; + *idp = (dtrace_meta_provider_id_t)meta; + + /* + * If there are providers and probes ready to go, pass them + * off to the new meta provider now. + */ + help = dtrace_deferred_pid; + dtrace_deferred_pid = NULL; + + mutex_unlock(&dtrace_lock); + + while (help != NULL) { + for (i = 0; i < help->dthps_nprovs; i++) { + dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov, + help->dthps_pid); + } + + next = help->dthps_next; + help->dthps_next = NULL; + help->dthps_prev = NULL; + help->dthps_deferred = 0; + help = next; + } + + mutex_unlock(&dtrace_meta_lock); + + return 0; +} +EXPORT_SYMBOL(dtrace_meta_register); + +int dtrace_meta_unregister(dtrace_meta_provider_id_t id) +{ + dtrace_meta_t **pp, *old = (dtrace_meta_t *)id; + + mutex_lock(&dtrace_meta_lock); + mutex_lock(&dtrace_lock); + + if (old == dtrace_meta_pid) { + pp = &dtrace_meta_pid; + } else { + pr_err("Attempt to unregister non-existent DTrace meta-" + "provider %p\n", (void *)old); + BUG(); + } + + if (old->dtm_count != 0) { + mutex_unlock(&dtrace_lock); + mutex_unlock(&dtrace_meta_lock); + return -EBUSY; + } + + *pp = NULL; + + mutex_unlock(&dtrace_lock); + mutex_unlock(&dtrace_meta_lock); + + kfree(old->dtm_name); + kfree(old); + + return 0; +} +EXPORT_SYMBOL(dtrace_meta_unregister); diff --git a/dtrace/fasttrap.h b/dtrace/fasttrap.h deleted file mode 100644 index 665a26c506e0..000000000000 --- a/dtrace/fasttrap.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef _FASTTRAP_H_ -#define _FASTTRAP_H_ - -extern int fasttrap_dev_init(void); -extern void fasttrap_dev_exit(void); - -#endif /* _FASTTRAP_H_ */ diff --git a/dtrace/fasttrap_dev.c b/dtrace/fasttrap_dev.c index bf88a79e1017..7dc08d844cd0 100644 --- a/dtrace/fasttrap_dev.c +++ b/dtrace/fasttrap_dev.c @@ -21,15 +21,772 @@ * * CDDL HEADER END * - * Copyright 2010, 2011 Oracle, Inc. All rights reserved. + * Copyright 2010, 2011, 2012, 2013 Oracle, Inc. All rights reserved. * Use is subject to license terms. */ +#include #include #include +#include +#include +#include #include "dtrace.h" #include "dtrace_dev.h" +#include "fasttrap_impl.h" + +#define FASTTRAP_MAX_DEFAULT 250000 +static uint32_t fasttrap_max; +static atomic_t fasttrap_total; + +#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 +#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 +#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 + +#define FASTTRAP_PID_NAME "pid" + +fasttrap_hash_t fasttrap_tpoints; +static fasttrap_hash_t fasttrap_provs; +static fasttrap_hash_t fasttrap_procs; + +#define FASTTRAP_PROVS_INDEX(pid, name) \ + ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) +#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) + +#define CLEANUP_NONE 0 +#define CLEANUP_SCHEDULED 1 +#define CLEANUP_DEFERRED 2 + +DEFINE_MUTEX(fasttrap_cleanup_mtx); +static uint_t fasttrap_cleanup_state; +static uint_t fasttrap_cleanup_work; + +static void fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc) +{ + /* + * There are no "default" pid probes. + */ +} + +static int +fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) +{ + return 0; /* FIXME */ +} + +static void fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) +{ + /* FIXME */ +} + +static void fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, + dtrace_argdesc_t *desc) +{ + /* FIXME */ +} + +static uint64_t fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, + int argno, int aframes) +{ + return 0; /* FIXME */ +} + +static void fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) +{ + /* FIXME */ +} + +static const dtrace_pattr_t pid_attr = { +{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, +{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, +{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, +{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, +{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, +}; + +static dtrace_pops_t pid_pops = { +#ifdef FIXME + fasttrap_pid_provide, + NULL, + fasttrap_pid_enable, + fasttrap_pid_disable, + NULL, + NULL, + fasttrap_pid_getargdesc, + fasttrap_pid_getarg, + NULL, + fasttrap_pid_destroy +#endif +}; + +static dtrace_pops_t usdt_pops = { + fasttrap_pid_provide, + NULL, + fasttrap_pid_enable, + fasttrap_pid_disable, + NULL, + NULL, + fasttrap_pid_getargdesc, + fasttrap_usdt_getarg, + NULL, + fasttrap_pid_destroy +}; + +static uint_t fasttrap_hash_str(const char *p) +{ + unsigned int g; + uint_t hval = 0; + + while (*p) { + hval = (hval << 4) + *p++; + if ((g = (hval & 0xf0000000)) != 0) + hval ^= g >> 24; + hval &= ~g; + } + + return hval; +} + +static int fasttrap_uint32_cmp(const void *ap, const void *bp) +{ + return (*(const uint32_t *)ap - *(const uint32_t *)bp); +} + +static int fasttrap_uint64_cmp(const void *ap, const void *bp) +{ + return (*(const uint64_t *)ap - *(const uint64_t *)bp); +} + +void fasttrap_meta_create_probe(void *arg, void *parg, + dtrace_helper_probedesc_t *dhpb) +{ + fasttrap_provider_t *provider = parg; + fasttrap_probe_t *pp; + fasttrap_tracepoint_t *tp; + int i, j; + uint32_t ntps; + + /* + * Since the meta provider count is non-zero we don't have to worry + * about this provider disappearing. + */ + ASSERT(provider->ftp_mcount > 0); + + /* + * The offsets must be unique. + */ + sort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof(uint32_t), + fasttrap_uint32_cmp, NULL); + for (i = 1; i < dhpb->dthpb_noffs; i++) { + if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= + dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) + return; + } + + sort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof(uint32_t), + fasttrap_uint32_cmp, NULL); + for (i = 1; i < dhpb->dthpb_nenoffs; i++) { + if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= + dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) + return; + } + + /* + * Grab the creation lock to ensure consistency between calls to + * dtrace_probe_lookup() and dtrace_probe_create() in the face of + * other threads creating probes. + */ + mutex_lock(&provider->ftp_cmtx); + + if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, + dhpb->dthpb_func, dhpb->dthpb_name) != 0) { + mutex_unlock(&provider->ftp_cmtx); + return; + } + + ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; + ASSERT(ntps > 0); + + atomic_add(ntps, &fasttrap_total); + + if (atomic_read(&fasttrap_total) > fasttrap_max) { + atomic_add(-ntps, &fasttrap_total); + mutex_unlock(&provider->ftp_cmtx); + return; + } + + pp = kzalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), GFP_KERNEL); + + pp->ftp_prov = provider; + pp->ftp_pid = provider->ftp_pid; + pp->ftp_ntps = ntps; + pp->ftp_nargs = dhpb->dthpb_xargc; + pp->ftp_xtypes = dhpb->dthpb_xtypes; + pp->ftp_ntypes = dhpb->dthpb_ntypes; + + /* + * First create a tracepoint for each actual point of interest. + */ + for (i = 0; i < dhpb->dthpb_noffs; i++) { + tp = kzalloc(sizeof(fasttrap_tracepoint_t), GFP_KERNEL); + + tp->ftt_proc = provider->ftp_proc; + tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; + tp->ftt_pid = provider->ftp_pid; + + pp->ftp_tps[i].fit_tp = tp; + pp->ftp_tps[i].fit_id.fti_probe = pp; +#ifdef __sparc + pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; +#else + pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; +#endif + } + + /* + * Then create a tracepoint for each is-enabled point. + */ + for (j = 0; i < ntps; i++, j++) { + tp = kzalloc(sizeof(fasttrap_tracepoint_t), GFP_KERNEL); + + tp->ftt_proc = provider->ftp_proc; + tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; + tp->ftt_pid = provider->ftp_pid; + + pp->ftp_tps[i].fit_tp = tp; + pp->ftp_tps[i].fit_id.fti_probe = pp; + pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; + } + + /* + * If the arguments are shuffled around we set the argument remapping + * table. Later, when the probe fires, we only remap the arguments + * if the table is non-NULL. + */ + for (i = 0; i < dhpb->dthpb_xargc; i++) { + if (dhpb->dthpb_args[i] != i) { + pp->ftp_argmap = dhpb->dthpb_args; + break; + } + } + + /* + * The probe is fully constructed -- register it with DTrace. + */ + pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, + dhpb->dthpb_func, dhpb->dthpb_name, + FASTTRAP_OFFSET_AFRAMES, pp); + + mutex_unlock(&provider->ftp_cmtx); +} + +static void fasttrap_proc_release(fasttrap_proc_t *proc) +{ + fasttrap_bucket_t *bucket; + fasttrap_proc_t *fprc, **fprcp; + pid_t pid = proc->ftpc_pid; + + mutex_lock(&proc->ftpc_mtx); + + ASSERT(proc->ftpc_rcount != 0); + ASSERT(atomic64_read(&proc->ftpc_acount) <= proc->ftpc_rcount); + + if (--proc->ftpc_rcount != 0) { + mutex_unlock(&proc->ftpc_mtx); + return; + } + + mutex_unlock(&proc->ftpc_mtx); + + /* + * There should definitely be no live providers associated with this + * process at this point. + */ + ASSERT(atomic64_read(&proc->ftpc_acount) == 0); + + bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; + mutex_lock(&bucket->ftb_mtx); + + fprcp = (fasttrap_proc_t **)&bucket->ftb_data; + while ((fprc = *fprcp) != NULL) { + if (fprc == proc) + break; + + fprcp = &fprc->ftpc_next; + } + + /* + * Something strange has happened if we can't find the proc. + */ + ASSERT(fprc != NULL); + + *fprcp = fprc->ftpc_next; + + mutex_unlock(&bucket->ftb_mtx); + + kfree(fprc); +} + +static void fasttrap_provider_free(fasttrap_provider_t *provider) +{ + pid_t pid = provider->ftp_pid; + + /* + * There need to be no associated enabled probes, no consumers + * creating probes, and no meta providers referencing this provider. + */ + ASSERT(provider->ftp_rcount == 0); + ASSERT(provider->ftp_ccount == 0); + ASSERT(provider->ftp_mcount == 0); + + /* + * If this provider hasn't been retired, we need to explicitly drop the + * count of active providers on the associated process structure. + */ + if (!provider->ftp_retired) { + atomic64_add(-1, &provider->ftp_proc->ftpc_acount); + ASSERT(atomic64_read(&provider->ftp_proc->ftpc_acount) < + provider->ftp_proc->ftpc_rcount); + } + + fasttrap_proc_release(provider->ftp_proc); + + kfree(provider); + + unregister_pid_provider(pid); +} + +static fasttrap_proc_t *fasttrap_proc_lookup(pid_t pid) +{ + fasttrap_bucket_t *bucket; + fasttrap_proc_t *fprc, *new_fprc; + + bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; + mutex_lock(&bucket->ftb_mtx); + + for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { + if (fprc->ftpc_pid == pid && + atomic64_read(&fprc->ftpc_acount) != 0) { + mutex_lock(&fprc->ftpc_mtx); + mutex_unlock(&bucket->ftb_mtx); + fprc->ftpc_rcount++; + atomic64_inc(&fprc->ftpc_acount); + ASSERT(atomic64_read(&fprc->ftpc_acount) <= + fprc->ftpc_rcount); + mutex_unlock(&fprc->ftpc_mtx); + + return fprc; + } + } + + /* + * Drop the bucket lock so we don't try to perform a sleeping + * allocation under it. + */ + mutex_unlock(&bucket->ftb_mtx); + + new_fprc = kzalloc(sizeof(fasttrap_proc_t), GFP_KERNEL); + new_fprc->ftpc_pid = pid; + new_fprc->ftpc_rcount = 1; + atomic64_set(&new_fprc->ftpc_acount, 1); + mutex_init(&new_fprc->ftpc_mtx); + + mutex_lock(&bucket->ftb_mtx); + + /* + * Take another lap through the list to make sure a proc hasn't + * been created for this pid while we weren't under the bucket lock. + */ + for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { + if (fprc->ftpc_pid == pid && + atomic64_read(&fprc->ftpc_acount) != 0) { + mutex_lock(&fprc->ftpc_mtx); + mutex_unlock(&bucket->ftb_mtx); + fprc->ftpc_rcount++; + atomic64_inc(&fprc->ftpc_acount); + ASSERT(atomic64_read(&fprc->ftpc_acount) <= + fprc->ftpc_rcount); + mutex_unlock(&fprc->ftpc_mtx); + + kfree(new_fprc); + + return fprc; + } + } + + new_fprc->ftpc_next = bucket->ftb_data; + bucket->ftb_data = new_fprc; + + mutex_unlock(&bucket->ftb_mtx); + + return new_fprc; +} + +/* + * Lookup a fasttrap-managed provider based on its name and associated pid. + * If the pattr argument is non-NULL, this function instantiates the provider + * if it doesn't exist otherwise it returns NULL. The provider is returned + * with its lock held. + */ +static fasttrap_provider_t *fasttrap_provider_lookup(pid_t pid, + const char *name, + const dtrace_pattr_t *pa) +{ + fasttrap_provider_t *fp, *new_fp = NULL; + fasttrap_bucket_t *bucket; + char provname[DTRACE_PROVNAMELEN]; + struct task_struct *p; + const cred_t *cred; + + ASSERT(strlen(name) < sizeof (fp->ftp_name)); + ASSERT(pa != NULL); + + bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; + mutex_lock(&bucket->ftb_mtx); + + /* + * Take a lap through the list and return the match if we find it. + */ + for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { + if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && + !fp->ftp_retired) { + mutex_lock(&fp->ftp_mtx); + mutex_unlock(&bucket->ftb_mtx); + return fp; + } + } + + /* + * Drop the bucket lock so we don't try to perform a sleeping + * allocation under it. + */ + mutex_unlock(&bucket->ftb_mtx); + + if ((p = register_pid_provider(pid)) == NULL) + return NULL; + + /* + * Grab the credentials for this process so we have + * something to pass to dtrace_register(). + */ + cred = get_cred(p->cred); + + new_fp = kzalloc(sizeof(fasttrap_provider_t), GFP_KERNEL); + new_fp->ftp_pid = pid; + new_fp->ftp_proc = fasttrap_proc_lookup(pid); + mutex_init(&new_fp->ftp_mtx); + mutex_init(&new_fp->ftp_cmtx); + + ASSERT(new_fp->ftp_proc != NULL); + + mutex_lock(&bucket->ftb_mtx); + + /* + * Take another lap through the list to make sure a provider hasn't + * been created for this pid while we weren't under the bucket lock. + */ + for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { + if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && + !fp->ftp_retired) { + mutex_lock(&fp->ftp_mtx); + mutex_unlock(&bucket->ftb_mtx); + fasttrap_provider_free(new_fp); + put_cred(cred); + return fp; + } + } + + strcpy(new_fp->ftp_name, name); + + /* + * Fail and return NULL if either the provider name is too long + * or we fail to register this new provider with the DTrace + * framework. Note that this is the only place we ever construct + * the full provider name -- we keep it in pieces in the provider + * structure. + */ + if (snprintf(provname, sizeof(provname), "%s%u", name, (uint_t)pid) >= + sizeof(provname) || + dtrace_register(provname, pa, + DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER, cred, + pa == &pid_attr ? &pid_pops : &usdt_pops, + new_fp, &new_fp->ftp_provid) != 0) { + mutex_unlock(&bucket->ftb_mtx); + fasttrap_provider_free(new_fp); + put_cred(cred); + return NULL; + } + + new_fp->ftp_next = bucket->ftb_data; + bucket->ftb_data = new_fp; + + mutex_lock(&new_fp->ftp_mtx); + mutex_unlock(&bucket->ftb_mtx); + + put_cred(cred); + return new_fp; +} + +void *fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, + pid_t pid) +{ + fasttrap_provider_t *provider; + + if (strlen(dhpv->dthpv_provname) + 10 >= sizeof (provider->ftp_name)) { + pr_warn("Failed to instantiate provider %s: name too long " + "to accomodate pid\n", dhpv->dthpv_provname); + return NULL; + } + + /* + * Don't let folks spoof the true pid provider. + */ + if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { + pr_warn("Failed to instantiate provider %s: %s is an invalid " + "name\n", dhpv->dthpv_provname, FASTTRAP_PID_NAME); + return NULL; + } + + /* + * The highest stability class that fasttrap supports is ISA; cap + * the stability of the new provider accordingly. + */ + if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) + dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; + if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) + dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; + if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) + dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; + if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) + dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; + if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) + dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; + + provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, + &dhpv->dthpv_pattr); + if (provider == NULL) { + pr_warn("Failed to instantiate provider %s for process %u\n", + dhpv->dthpv_provname, (uint_t)pid); + return NULL; + } + + /* + * Up the meta provider count so this provider isn't removed until the + * meta provider has been told to remove it. + */ + provider->ftp_mcount++; + + mutex_unlock(&provider->ftp_mtx); + + return provider; +} + +static void fasttrap_pid_cleanup_cb(struct work_struct *work) +{ + fasttrap_provider_t **fpp, *fp; + fasttrap_bucket_t *bucket; + dtrace_provider_id_t provid; + int i, later = 0; + + static volatile int in = 0; + + ASSERT(in == 0); + in = 1; + + mutex_lock(&fasttrap_cleanup_mtx); + while (fasttrap_cleanup_work) { + fasttrap_cleanup_work = 0; + mutex_unlock(&fasttrap_cleanup_mtx); + + later = 0; + + /* + * Iterate over all the providers trying to remove the marked + * ones. If a provider is marked but not retired, we just + * have to take a crack at removing it -- it's no big deal if + * we can't. + */ + for (i = 0; i < fasttrap_provs.fth_nent; i++) { + bucket = &fasttrap_provs.fth_table[i]; + mutex_lock(&bucket->ftb_mtx); + fpp = (fasttrap_provider_t **)&bucket->ftb_data; + + while ((fp = *fpp) != NULL) { + if (!fp->ftp_marked) { + fpp = &fp->ftp_next; + continue; + } + + mutex_lock(&fp->ftp_mtx); + + /* + * If this provider has consumers actively + * creating probes (ftp_ccount) or is a USDT + * provider (ftp_mcount), we can't unregister + * or even condense. + */ + if (fp->ftp_ccount != 0 || + fp->ftp_mcount != 0) { + mutex_unlock(&fp->ftp_mtx); + fp->ftp_marked = 0; + continue; + } + + if (!fp->ftp_retired || fp->ftp_rcount != 0) + fp->ftp_marked = 0; + + mutex_unlock(&fp->ftp_mtx); + + /* + * If we successfully unregister this + * provider we can remove it from the hash + * chain and free the memory. If our attempt + * to unregister fails and this is a retired + * provider, increment our flag to try again + * pretty soon. If we've consumed more than + * half of our total permitted number of + * probes call dtrace_condense() to try to + * clean out the unenabled probes. + */ + provid = fp->ftp_provid; + if (dtrace_unregister(provid) != 0) { + if (atomic_read(&fasttrap_total) > + fasttrap_max / 2) + dtrace_condense(provid); + later += fp->ftp_marked; + fpp = &fp->ftp_next; + } else { + *fpp = fp->ftp_next; + fasttrap_provider_free(fp); + } + } + + mutex_unlock(&bucket->ftb_mtx); + } + + mutex_lock(&fasttrap_cleanup_mtx); + } + + ASSERT(fasttrap_cleanup_state != CLEANUP_NONE); + + /* + * If we were unable to remove a retired provider, try again after + * a second. This situation can occur in certain circumstances where + * providers cannot be unregistered even though they have no probes + * enabled because of an execution of dtrace -l or something similar. + * If the timeout has been disabled (set to 1 because we're trying + * to detach), we set fasttrap_cleanup_work to ensure that we'll + * get a chance to do that work if and when the timeout is reenabled + * (if detach fails). + */ + if (later > 0 && fasttrap_cleanup_state != CLEANUP_DEFERRED) { + struct delayed_work *dw = container_of(work, + struct delayed_work, + work); + + fasttrap_cleanup_state = CLEANUP_SCHEDULED; + schedule_delayed_work(dw, HZ); + } else if (later > 0) { + fasttrap_cleanup_work = 1; + } else + fasttrap_cleanup_state = CLEANUP_NONE; + + mutex_unlock(&fasttrap_cleanup_mtx); + in = 0; +} + +static DECLARE_DELAYED_WORK(fasttrap_cleanup, fasttrap_pid_cleanup_cb); + +/* + * Activate the asynchronous cleanup mechanism. + */ +static void fasttrap_pid_cleanup(void) +{ + mutex_lock(&fasttrap_cleanup_mtx); + fasttrap_cleanup_work = 1; + fasttrap_cleanup_state = CLEANUP_SCHEDULED; + schedule_delayed_work(&fasttrap_cleanup, 3); + mutex_unlock(&fasttrap_cleanup_mtx); +} + +void fasttrap_provider_retire(pid_t pid, const char *name, int mprov) +{ + fasttrap_provider_t *fp; + fasttrap_bucket_t *bucket; + dtrace_provider_id_t provid; + + ASSERT(strlen(name) < sizeof (fp->ftp_name)); + + bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; + mutex_lock(&bucket->ftb_mtx); + + for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { + if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && + !fp->ftp_retired) + break; + } + + if (fp == NULL) { + mutex_unlock(&bucket->ftb_mtx); + return; + } + + mutex_lock(&fp->ftp_mtx); + ASSERT(!mprov || fp->ftp_mcount > 0); + if (mprov && --fp->ftp_mcount != 0) { + mutex_unlock(&fp->ftp_mtx); + mutex_unlock(&bucket->ftb_mtx); + return; + } + + /* + * Mark the provider to be removed in our post-processing step, mark it + * retired, and drop the active count on its proc. Marking it indicates + * that we should try to remove it; setting the retired flag indicates + * that we're done with this provider; dropping the active count on the + * proc releases our hold, and when this reaches zero (as it will + * during exit or exec) the proc and associated providers become + * defunct. + * + * We obviously need to take the bucket lock before the provider lock + * to perform the lookup, but we need to drop the provider lock + * before calling into the DTrace framework since we acquire the + * provider lock in callbacks invoked from the DTrace framework. The + * bucket lock therefore protects the integrity of the provider hash + * table. + */ + atomic64_dec(&fp->ftp_proc->ftpc_acount); + ASSERT(atomic64_read(&fp->ftp_proc->ftpc_acount) < + fp->ftp_proc->ftpc_rcount); + + fp->ftp_retired = 1; + fp->ftp_marked = 1; + provid = fp->ftp_provid; + mutex_unlock(&fp->ftp_mtx); + + /* + * We don't have to worry about invalidating the same provider twice + * since fasttrap_provider_lookup() will ignore provider that have + * been marked as retired. + */ + dtrace_invalidate(provid); + + mutex_unlock(&bucket->ftb_mtx); + + fasttrap_pid_cleanup(); +} + +void fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) +{ + /* + * Clean up the USDT provider. There may be active consumers of the + * provider busy adding probes, no damage will actually befall the + * provider until that count has dropped to zero. This just puts + * the provider on death row. + */ + fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); +} static long fasttrap_ioctl(struct file *file, unsigned int cmd, unsigned long arg) @@ -61,19 +818,197 @@ static struct miscdevice fasttrap_dev = { .fops = &fasttrap_fops, }; +static void fasttrap_init_htable(fasttrap_hash_t *fth, ulong_t nent) +{ + ulong_t i; + + if ((nent & (nent - 1)) == 0) + fth->fth_nent = nent; + else + fth->fth_nent = 1 << fls(nent); + + ASSERT(fth->fth_nent > 0); + + fth->fth_mask = fth->fth_nent - 1; + fth->fth_table = kzalloc(fth->fth_nent * sizeof(fasttrap_bucket_t), + GFP_KERNEL); + + for (i = 0; i < fth->fth_nent; i++) + mutex_init(&fth->fth_table[i].ftb_mtx); +} + int fasttrap_dev_init(void) { - int ret = 0; + int ret = 0; + ulong_t nent; ret = misc_register(&fasttrap_dev); - if (ret) + if (ret) { pr_err("%s: Can't register misc device %d\n", fasttrap_dev.name, fasttrap_dev.minor); + goto fail; + } + +#ifdef FIXME + dtrace_fasttrap_fork_ptr = &fasttrap_fork; + dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit; + dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit; +#endif + + fasttrap_max = FASTTRAP_MAX_DEFAULT; + atomic_set(&fasttrap_total, 0); + + /* + * Conjure up the tracepoints hashtable... + */ + nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; + if (nent == 0 || nent > 0x1000000) + nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; + + fasttrap_init_htable(&fasttrap_tpoints, nent); + + /* + * ... and the providers hash table... + */ + nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; + fasttrap_init_htable(&fasttrap_provs, nent); + + /* + * ... and the procs hash table. + */ + nent = FASTTRAP_PROCS_DEFAULT_SIZE; + fasttrap_init_htable(&fasttrap_procs, nent); + +fail: return ret; } void fasttrap_dev_exit(void) { + int fail = 0; + ulong_t i; + + /* + * Prevent any new timeouts from running by setting fasttrap_timeout + * to a non-zero value, and wait for the current timeout to complete. + */ + mutex_lock(&fasttrap_cleanup_mtx); + fasttrap_cleanup_work = 0; + + while (fasttrap_cleanup_state != CLEANUP_DEFERRED) { + uint_t tmp; + + tmp = fasttrap_cleanup_state; + fasttrap_cleanup_state = CLEANUP_DEFERRED; + + if (tmp != CLEANUP_NONE) { + mutex_unlock(&fasttrap_cleanup_mtx); + flush_delayed_work_sync(&fasttrap_cleanup); + mutex_lock(&fasttrap_cleanup_mtx); + } + } + + fasttrap_cleanup_work = 0; + mutex_unlock(&fasttrap_cleanup_mtx); + + /* + * Iterate over all of our providers. If there's still a process + * that corresponds to that pid, fail to detach. + */ + for (i = 0; i < fasttrap_provs.fth_nent; i++) { + fasttrap_provider_t **fpp, *fp; + fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; + + mutex_lock(&bucket->ftb_mtx); + fpp = (fasttrap_provider_t **)&bucket->ftb_data; + while ((fp = *fpp) != NULL) { + /* + * Acquire and release the lock as a simple way of + * waiting for any other consumer to finish with + * this provider. A thread must first acquire the + * bucket lock so there's no chance of another thread + * blocking on the provider's lock. + */ + mutex_lock(&fp->ftp_mtx); + mutex_unlock(&fp->ftp_mtx); + + if (dtrace_unregister(fp->ftp_provid) != 0) { + fail = 1; + fpp = &fp->ftp_next; + } else { + *fpp = fp->ftp_next; + fasttrap_provider_free(fp); + } + } + + mutex_unlock(&bucket->ftb_mtx); + } + +#ifdef FIXME + if (fail) { + uint_t work; + + /* + * If we're failing to detach, we need to unblock timeouts + * and start a new timeout if any work has accumulated while + * we've been unsuccessfully trying to detach. + */ + mutex_lock(&fasttrap_cleanup_mtx); + fasttrap_cleanup_state = CLEANUP_NONE; + work = fasttrap_cleanup_work; + mutex_unlock(&fasttrap_cleanup_mtx); + + if (work) + fasttrap_pid_cleanup(); + + dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, + &fasttrap_meta_id); + + return (DDI_FAILURE); + } +#else + /* + * If we fail to detach, we are in lots of trouble because Linux does + * not allow us to fail on a module unload. We try to continue anyway + * since triggering an OOPS at this point is even worse. + */ + if (fail) + pr_err("Unrecoverable error removing the fasttrap provider\n"); +#endif + +#ifdef DEBUG + mutex_lock(&fasttrap_count_mtx); + ASSERT(fasttrap_pid_count == 0); + mutex_unlock(&fasttrap_count_mtx); +#endif + + kfree(fasttrap_tpoints.fth_table); + fasttrap_tpoints.fth_nent = 0; + + kfree(fasttrap_provs.fth_table); + fasttrap_provs.fth_nent = 0; + + kfree(fasttrap_procs.fth_table); + fasttrap_procs.fth_nent = 0; + + /* + * We know there are no tracepoints in any process anywhere in + * the system so there is no process which has its p_dtrace_count + * greater than zero, therefore we know that no thread can actively + * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes + * and fasttrap_exec() and fasttrap_exit(). + */ +#ifdef FIXME + ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork); + dtrace_fasttrap_fork_ptr = NULL; + + ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit); + dtrace_fasttrap_exec_ptr = NULL; + + ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit); + dtrace_fasttrap_exit_ptr = NULL; +#endif + misc_deregister(&fasttrap_dev); } diff --git a/dtrace/fasttrap_impl.h b/dtrace/fasttrap_impl.h new file mode 100644 index 000000000000..e9b242463993 --- /dev/null +++ b/dtrace/fasttrap_impl.h @@ -0,0 +1,132 @@ +#ifndef _FASTTRAP_IMPL_H_ +#define _FASTTRAP_IMPL_H_ + +#include + +/* + * Fasttrap Providers, Probes and Tracepoints + * + * Each Solaris process can have multiple providers -- the pid provider as + * well as any number of user-level statically defined tracing (USDT) + * providers. Those providers are each represented by a fasttrap_provider_t. + * All providers for a given process have a pointer to a shared + * fasttrap_proc_t. The fasttrap_proc_t has two states: active or defunct. + * When the count of active providers goes to zero it becomes defunct; a + * provider drops its active count when it is removed individually or as part + * of a mass removal when a process exits or performs an exec. + * + * Each probe is represented by a fasttrap_probe_t which has a pointer to + * its associated provider as well as a list of fasttrap_id_tp_t structures + * which are tuples combining a fasttrap_id_t and a fasttrap_tracepoint_t. + * A fasttrap_tracepoint_t represents the actual point of instrumentation + * and it contains two lists of fasttrap_id_t structures (to be fired pre- + * and post-instruction emulation) that identify the probes attached to the + * tracepoint. Tracepoints also have a pointer to the fasttrap_proc_t for the + * process they trace which is used when looking up a tracepoint both when a + * probe fires and when enabling and disabling probes. + * + * It's important to note that probes are preallocated with the necessary + * number of tracepoints, but that tracepoints can be shared by probes and + * swapped between probes. If a probe's preallocated tracepoint is enabled + * (and, therefore, the associated probe is enabled), and that probe is + * then disabled, ownership of that tracepoint may be exchanged for an + * unused tracepoint belonging to another probe that was attached to the + * enabled tracepoint. + */ +typedef struct fasttrap_proc { + pid_t ftpc_pid; /* process ID for this proc */ + atomic64_t ftpc_acount; /* count of active providers */ + uint64_t ftpc_rcount; /* count of extant providers */ + struct mutex ftpc_mtx; /* lock on all but acount */ + struct fasttrap_proc *ftpc_next; /* next proc in hash chain */ +} fasttrap_proc_t; + +typedef struct fasttrap_provider { + pid_t ftp_pid; /* process ID for this prov */ + char ftp_name[DTRACE_PROVNAMELEN]; /* prov name (w/o the pid) */ + dtrace_provider_id_t ftp_provid; /* DTrace provider handle */ + uint_t ftp_marked; /* mark for possible removal */ + uint_t ftp_retired; /* mark when retired */ + struct mutex ftp_mtx; /* provider lock */ + struct mutex ftp_cmtx; /* lock on creating probes */ + uint64_t ftp_rcount; /* enabled probes ref count */ + uint64_t ftp_ccount; /* consumers creating probes */ + uint64_t ftp_mcount; /* meta provider count */ + fasttrap_proc_t *ftp_proc; /* shared proc for all provs */ + struct fasttrap_provider *ftp_next; /* next prov in hash chain */ +} fasttrap_provider_t; + +typedef struct fasttrap_id fasttrap_id_t; +typedef struct fasttrap_probe fasttrap_probe_t; +typedef struct fasttrap_tracepoint fasttrap_tracepoint_t; + +struct fasttrap_id { + fasttrap_probe_t *fti_probe; /* referrring probe */ + fasttrap_id_t *fti_next; /* enabled probe list on tp */ + fasttrap_probe_type_t fti_ptype; /* probe type */ +}; + +typedef unsigned long fasttrap_machtp_t; /* FIXME */ + +struct fasttrap_tracepoint { + fasttrap_proc_t *ftt_proc; /* associated process struct */ + uintptr_t ftt_pc; /* address of tracepoint */ + pid_t ftt_pid; /* pid of tracepoint */ + fasttrap_machtp_t ftt_mtp; /* ISA-specific portion */ + fasttrap_id_t *ftt_ids; /* NULL-terminated list */ + fasttrap_id_t *ftt_retids; /* NULL-terminated list */ + fasttrap_tracepoint_t *ftt_next; /* link in global hash */ +}; + +typedef struct fasttrap_id_tp { + fasttrap_id_t fit_id; + fasttrap_tracepoint_t *fit_tp; +} fasttrap_id_tp_t; + +struct fasttrap_probe { + dtrace_id_t ftp_id; /* DTrace probe identifier */ + pid_t ftp_pid; /* pid for this probe */ + fasttrap_provider_t *ftp_prov; /* this probe's provider */ + uintptr_t ftp_faddr; /* associated function's addr */ + size_t ftp_fsize; /* associated function's size */ + uint64_t ftp_gen; /* modification generation */ + uint64_t ftp_ntps; /* number of tracepoints */ + uint8_t *ftp_argmap; /* native to translated args */ + uint8_t ftp_nargs; /* translated argument count */ + uint8_t ftp_enabled; /* is this probe enabled */ + char *ftp_xtypes; /* translated types index */ + char *ftp_ntypes; /* native types index */ + fasttrap_id_tp_t ftp_tps[1]; /* flexible array */ +}; + +typedef struct fasttrap_bucket { + struct mutex ftb_mtx; /* bucket lock */ + void *ftb_data; /* data payload */ + + uint8_t ftb_pad[64 - sizeof(struct mutex) - sizeof(void *)]; +} fasttrap_bucket_t; + +typedef struct fasttrap_hash { + ulong_t fth_nent; /* power-of-2 num. of entries */ + ulong_t fth_mask; /* fth_nent - 1 */ + fasttrap_bucket_t *fth_table; /* array of buckets */ +} fasttrap_hash_t; + +#define FASTTRAP_ID_INDEX(id) \ + ((fasttrap_id_tp_t *)(((char *)(id) - \ + offsetof(fasttrap_id_tp_t, fit_id))) - \ + &(id)->fti_probe->ftp_tps[0]) + +#define FASTTRAP_OFFSET_AFRAMES 3 + +extern void fasttrap_meta_create_probe(void *, void *, + dtrace_helper_probedesc_t *); +extern void *fasttrap_meta_provide(void *, dtrace_helper_provdesc_t *, pid_t); +extern void fasttrap_meta_remove(void *, dtrace_helper_provdesc_t *, pid_t); + +extern dtrace_meta_provider_id_t fasttrap_id; + +extern int fasttrap_dev_init(void); +extern void fasttrap_dev_exit(void); + +#endif /* _FASTTRAP_IMPL_H_ */ diff --git a/dtrace/fasttrap_mod.c b/dtrace/fasttrap_mod.c index df1c98566a31..fc85d48405d5 100644 --- a/dtrace/fasttrap_mod.c +++ b/dtrace/fasttrap_mod.c @@ -21,7 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010, 2011 Oracle, Inc. All rights reserved. + * Copyright 2010, 2011, 2012, 2013 Oracle, Inc. All rights reserved. * Use is subject to license terms. */ @@ -29,17 +29,17 @@ #include "dtrace.h" #include "dtrace_dev.h" -#include "fasttrap.h" +#include "fasttrap_impl.h" MODULE_AUTHOR("Kris Van Hees (kris.van.hees@oracle.com)"); MODULE_DESCRIPTION("Fasttrap Tracing"); MODULE_VERSION("v0.1"); MODULE_LICENSE("CDDL"); -static const dtrace_pattr_t fasttrap_attr = { +static dtrace_mops_t fasttrap_mops = { + fasttrap_meta_create_probe, + fasttrap_meta_provide, + fasttrap_meta_remove }; -static dtrace_pops_t fasttrap_pops = { -}; - -DT_PROVIDER_MODULE(fasttrap, DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER) +DT_META_PROVIDER_MODULE(fasttrap) -- 2.50.1