From: Tomas Jedlicka Date: Fri, 7 Apr 2017 22:23:04 +0000 (-0400) Subject: dtrace: FBT module support and SPARCs return probes X-Git-Tag: v4.1.12-111.0.20170907_2225~3^2~3^2~4 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=98320804470d626a34ea8f1f0e40635180cab7ba;p=users%2Fjedix%2Flinux-maple.git dtrace: FBT module support and SPARCs return probes This fix contains following changes: - Modification of provider ops vector - Move of fbt hash table to x86 platform specific code - Instrumentation of return probes on SPARC DTrace provider has semantics that allows to create a probe on the fly or to provide probes for given module. There is no way how a provider can attach its own per-module data through the framework. With this change a provider my allocate per-module data inside provide_module() call. Once module is about to go it will be notified by framework by destroy_module() op. To stay binary compatible I extended the ops vector on its end but used C99 style to init ops structures to keep callbacks grouped per their logic. SPARC now does two passes over available kernel symbols. The first one is used to count how many symbols are present to be able to allocate correctly sized trampoline. Second pass performs actuall disassembly and creates return probes. It is possible that not every probe is instrumentable so we may end up wasting some memory. It is a tradeoff between speed and memory consumption. It is not possible to instrument arbitrary return places so we support only some variants that are used in the stream. Current implementation relies on usage of JMPL thus it is not possible to instrument return from tail call optimized code. Another change is in patching of the code. The JMPL requires to store NOP in its delay slot. This prevents us to do this atomically on the running kernel and must stop CPUs for safety reasons. Linux probes may fire from non-standart context like TL1 so it is not safe to assume anything about %g registers. Thanks to having few free %l we are able to temporarly store %gs and restor them back to avoid breaking up trap handlers. Orabug: 26384179 Orabug: 26384765 Signed-off-by: Tomas Jedlicka Reviewed-by: Kris Van Hees --- diff --git a/dtrace/dt_perf_mod.c b/dtrace/dt_perf_mod.c index 1fd67d812f4c8..1365cb4875fe9 100644 --- a/dtrace/dt_perf_mod.c +++ b/dtrace/dt_perf_mod.c @@ -47,16 +47,17 @@ static const dtrace_pattr_t dt_perf_attr = { DT_PROVIDER_POPS(dt_perf) static dtrace_pops_t dt_perf_pops = { - dt_perf_provide, - NULL, - dt_perf_enable, - dt_perf_disable, - NULL, - NULL, - NULL, - NULL, - NULL, - dt_perf_destroy + .dtps_provide = dt_perf_provide, + .dtps_provide_module = NULL, + .dtps_destroy_module = NULL, + .dtps_enable = dt_perf_enable, + .dtps_disable = dt_perf_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = NULL, + .dtps_getargval = NULL, + .dtps_usermode = NULL, + .dtps_destroy = dt_perf_destroy }; DT_PROVIDER_MODULE(dt_perf, DTRACE_PRIV_USER) diff --git a/dtrace/dt_test_mod.c b/dtrace/dt_test_mod.c index b7e059bd57ead..57ea5fe9e542d 100644 --- a/dtrace/dt_test_mod.c +++ b/dtrace/dt_test_mod.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2011, 2012 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -47,16 +46,17 @@ static const dtrace_pattr_t dt_test_attr = { DT_PROVIDER_POPS(dt_test) static dtrace_pops_t dt_test_pops = { - dt_test_provide, - NULL, - dt_test_enable, - dt_test_disable, - NULL, - NULL, - NULL, - NULL, - NULL, - dt_test_destroy + .dtps_provide = dt_test_provide, + .dtps_provide_module = NULL, + .dtps_destroy_module = NULL, + .dtps_enable = dt_test_enable, + .dtps_disable = dt_test_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = NULL, + .dtps_getargval = NULL, + .dtps_usermode = NULL, + .dtps_destroy = dt_test_destroy }; DT_PROVIDER_MODULE(dt_test, DTRACE_PRIV_USER) diff --git a/dtrace/dtrace_dev.c b/dtrace/dtrace_dev.c index 000dcc70b2c3a..08006eaa07205 100644 --- a/dtrace/dtrace_dev.c +++ b/dtrace/dtrace_dev.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010-2016 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -58,7 +57,8 @@ dtrace_pops_t dtrace_provider_ops = { NULL, NULL, NULL, - (void (*)(void *, dtrace_id_t, void *))dtrace_nullop + (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, + (void (*)(void *, struct module *))dtrace_nullop, }; static size_t dtrace_retain_max = 1024; @@ -1135,10 +1135,11 @@ static void module_del_pdata(void *dmy, struct module *mp) { dtrace_module_t *pdata = mp->pdata; - if (!pdata) + if (pdata == NULL) return; pdata_cleanup(pdata, mp); + mp->pdata = NULL; kmem_cache_free(dtrace_pdata_cachep, pdata); } @@ -1257,10 +1258,17 @@ static void dtrace_module_unloaded(struct module *mp) kfree(probe); } - mutex_unlock(&dtrace_lock); - mutex_unlock(&dtrace_provider_lock); + /* + * Notify providers to cleanup per-module data for this module. + */ + for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next) + if (prv->dtpv_pops.dtps_destroy_module != NULL) + prv->dtpv_pops.dtps_destroy_module(prv->dtpv_arg, mp); module_del_pdata(NULL, mp); + + mutex_unlock(&dtrace_lock); + mutex_unlock(&dtrace_provider_lock); } /* @@ -1452,6 +1460,7 @@ int dtrace_dev_init(void) * a pdata object. Modules loaded after this one will get their pdata * object assigned using the module notifier hook. */ + module_add_pdata(NULL, dtrace_kmod); dtrace_for_each_module(module_add_pdata, NULL); /* @@ -1603,6 +1612,7 @@ void dtrace_dev_exit(void) * point had their pdata object cleaned up using the module notifier * hook. */ + module_del_pdata(NULL, dtrace_kmod); dtrace_for_each_module(module_del_pdata, NULL); /* diff --git a/dtrace/dtrace_isa_sparc64.c b/dtrace/dtrace_isa_sparc64.c index 8901fb1037761..66f0a9f9fcea5 100644 --- a/dtrace/dtrace_isa_sparc64.c +++ b/dtrace/dtrace_isa_sparc64.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010-2014 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -156,18 +155,22 @@ ulong_t dtrace_getreg(struct task_struct *task, uint_t reg) void pdata_init(dtrace_module_t *pdata, struct module *mp) { - if (mp->pdata) { - pdata->sdt_tab = (asm_instr_t *)((uintptr_t)mp->pdata + - DTRACE_PD_SDT_OFF(mp)); - pdata->fbt_tab = (asm_instr_t *)((uintptr_t)mp->pdata + - DTRACE_PD_FBT_OFF(mp)); - } else { - pdata->sdt_tab = NULL; - pdata->fbt_tab = NULL; + /* + * Throw away existing data as we don't support reusal at + * the moment. + */ + if (mp->pdata != NULL) { + pdata_cleanup(pdata, mp); } + + pdata->sdt_tab = NULL; + pdata->fbt_tab = NULL; } void pdata_cleanup(dtrace_module_t *pdata, struct module *mp) { - mp->pdata = (void *)((uintptr_t)pdata->sdt_tab - DTRACE_PD_SDT_OFF(mp)); + if (pdata->sdt_tab != NULL) + dtrace_free_text(pdata->sdt_tab); + if (pdata->fbt_tab != NULL) + dtrace_free_text(pdata->fbt_tab); } diff --git a/dtrace/dtrace_probe_ctx.c b/dtrace/dtrace_probe_ctx.c index a38b6eed5b00d..3d92503f72a8a 100644 --- a/dtrace/dtrace_probe_ctx.c +++ b/dtrace/dtrace_probe_ctx.c @@ -39,6 +39,7 @@ void dtrace_panic(const char *fmt, ...) BUG(); } +EXPORT_SYMBOL(dtrace_panic); int dtrace_assfail(const char *a, const char *f, int l) { diff --git a/dtrace/dtrace_ptofapi.c b/dtrace/dtrace_ptofapi.c index 532a1e9bf1912..d5e05cd97c8d2 100644 --- a/dtrace/dtrace_ptofapi.c +++ b/dtrace/dtrace_ptofapi.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010, 2011, 2012, 2013 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -222,6 +221,11 @@ int dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv, (void (*)(void *, struct module *))dtrace_nullop; } + if (pops->dtps_destroy_module == NULL) { + provider->dtpv_pops.dtps_destroy_module = + (void (*)(void *, struct module *))dtrace_nullop; + } + if (pops->dtps_suspend == NULL) { ASSERT(pops->dtps_resume == NULL); provider->dtpv_pops.dtps_suspend = diff --git a/dtrace/fasttrap_dev.c b/dtrace/fasttrap_dev.c index 3d8b9b0bae3e4..25201f41585bb 100644 --- a/dtrace/fasttrap_dev.c +++ b/dtrace/fasttrap_dev.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010 -- 2016 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -709,6 +708,7 @@ static dtrace_pops_t pid_pops = { fasttrap_pid_provide, NULL, NULL, + NULL, fasttrap_pid_enable, fasttrap_pid_disable, NULL, @@ -721,16 +721,17 @@ static dtrace_pops_t pid_pops = { }; static dtrace_pops_t usdt_pops = { - fasttrap_pid_provide, - NULL, - fasttrap_pid_enable, - fasttrap_pid_disable, - NULL, - NULL, - fasttrap_pid_getargdesc, - fasttrap_usdt_getarg, - NULL, - fasttrap_pid_destroy + .dtps_provide = fasttrap_pid_provide, + .dtps_provide_module = NULL, + .dtps_destroy_module = NULL, + .dtps_enable = fasttrap_pid_enable, + .dtps_disable = fasttrap_pid_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = fasttrap_pid_getargdesc, + .dtps_getargval = fasttrap_usdt_getarg, + .dtps_usermode = NULL, + .dtps_destroy = fasttrap_pid_destroy }; static uint_t fasttrap_hash_str(const char *p) diff --git a/dtrace/fbt_dev.c b/dtrace/fbt_dev.c index 9d0ddd9487998..d2014cac130f5 100644 --- a/dtrace/fbt_dev.c +++ b/dtrace/fbt_dev.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010-2017 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -44,7 +43,7 @@ int fbt_probetab_mask; static void *fbt_provide_probe(struct module *mp, char *func, int type, int stype, asm_instr_t *addr, uintptr_t off, - void *pfbt) + void *pfbt, void *arg) { fbt_probe_t *fbp; fbt_probe_t *prev; @@ -61,14 +60,19 @@ static void *fbt_provide_probe(struct module *mp, char *func, int type, int fbp->fbp_roffset = off; fbp->fbp_patchpoint = addr; fbt_provide_probe_arch(fbp, type, stype); - fbp->fbp_hashnext = fbt_probetab[FBT_ADDR2NDX(addr)]; - fbt_probetab[FBT_ADDR2NDX(addr)] = fbp; + fbp->fbp_hashnext = fbt_probetab[FBT_ADDR2NDX(fbp->fbp_patchpoint)]; + fbt_probetab[FBT_ADDR2NDX(fbp->fbp_patchpoint)] = fbp; PDATA(mp)->fbt_probe_cnt++; return fbp; case FBT_RETURN: + + /* Check if we are able to patch this return probe. */ + if (!fbt_can_patch_return_arch(addr)) + return pfbt; + fbp = kzalloc(sizeof(fbt_probe_t), GFP_KERNEL); fbp->fbp_name = kstrdup(func, GFP_KERNEL); @@ -88,9 +92,9 @@ static void *fbt_provide_probe(struct module *mp, char *func, int type, int fbp->fbp_roffset = off; fbp->fbp_patchpoint = addr; fbt_provide_probe_arch(fbp, type, stype); - fbp->fbp_hashnext = fbt_probetab[FBT_ADDR2NDX(addr)]; - fbt_probetab[FBT_ADDR2NDX(addr)] = fbp; + fbp->fbp_hashnext = fbt_probetab[FBT_ADDR2NDX(fbp->fbp_patchpoint)]; + fbt_probetab[FBT_ADDR2NDX(fbp->fbp_patchpoint)] = fbp; PDATA(mp)->fbt_probe_cnt++; @@ -105,16 +109,35 @@ static void *fbt_provide_probe(struct module *mp, char *func, int type, int void fbt_provide_module(void *arg, struct module *mp) { + struct module_use *use; + /* * Nothing to do if the module FBT probes were already created. */ if (PDATA(mp)->fbt_probe_cnt != 0) return; - if (strncmp(mp->name, "vmlinux", 7)) + /* + * Do not try to instrument DTrace itself and its modules: + * - dtrace module + * - ctf module + * - all modules depending on dtrace + */ + if (!strncmp(mp->name, "dtrace", 7)) + return; + + list_for_each_entry(use, &mp->target_list, target_list) { + if (!strncmp(use->target->name, "dtrace", 7)) + return; + } + + /* + * Provide probes. + */ + if (!fbt_provide_module_arch(arg, mp)) return; - dtrace_fbt_init((fbt_add_probe_fn)fbt_provide_probe); + dtrace_fbt_init((fbt_add_probe_fn)fbt_provide_probe, mp, NULL); } int _fbt_enable(void *arg, dtrace_id_t id, void *parg) @@ -165,13 +188,12 @@ void _fbt_disable(void *arg, dtrace_id_t id, void *parg) void fbt_destroy(void *arg, dtrace_id_t id, void *parg) { fbt_probe_t *fbp = parg; - fbt_probe_t *nxt, *hbp, *lst; - struct module *mp = fbp->fbp_module; + fbt_probe_t *hbp, *lst, *nxt; int ndx; + struct module *mp = fbp->fbp_module; do { - if (mp != NULL) - PDATA(mp)->fbt_probe_cnt--; + nxt = fbp->fbp_next; ndx = FBT_ADDR2NDX(fbp->fbp_patchpoint); lst = NULL; @@ -189,10 +211,11 @@ void fbt_destroy(void *arg, dtrace_id_t id, void *parg) else fbt_probetab[ndx] = fbp->fbp_hashnext; - nxt = fbp->fbp_next; - + kfree(fbp->fbp_name); kfree(fbp); + PDATA(mp)->fbt_probe_cnt--; + fbp = nxt; } while (fbp != NULL); } @@ -231,18 +254,12 @@ int fbt_dev_init(void) { int ret = 0; - fbt_probetab_mask = fbt_probetab_size - 1; - fbt_probetab = dtrace_vzalloc_try(fbt_probetab_size * - sizeof (fbt_probe_t *)); - ret = misc_register(&fbt_dev); if (ret) pr_err("%s: Can't register misc device %d\n", fbt_dev.name, fbt_dev.minor); - fbt_dev_init_arch(); - - return ret; + return fbt_dev_init_arch(); } void fbt_dev_exit(void) @@ -250,8 +267,4 @@ void fbt_dev_exit(void) fbt_dev_exit_arch(); misc_deregister(&fbt_dev); - - vfree(fbt_probetab); - fbt_probetab_mask = 0; - fbt_probetab_size = 0; } diff --git a/dtrace/fbt_impl.h b/dtrace/fbt_impl.h index 530342d6256e1..bda21e096ce78 100644 --- a/dtrace/fbt_impl.h +++ b/dtrace/fbt_impl.h @@ -4,21 +4,6 @@ #include #include -typedef struct fbt_probe { - char *fbp_name; /* name of probe */ - dtrace_id_t fbp_id; /* probe ID */ - struct module *fbp_module; /* defining module */ - int fbp_loadcnt; /* load count for module */ - int fbp_primary; /* non-zero if primary mod */ - asm_instr_t *fbp_patchpoint;/* patch point */ - asm_instr_t fbp_patchval; /* instruction to patch */ - asm_instr_t fbp_savedval; /* saved instruction value */ - uintptr_t fbp_roffset; - int fbp_rval; - struct fbt_probe *fbp_next; /* next probe */ - struct fbt_probe *fbp_hashnext; /* next on hash */ -} fbt_probe_t; - #define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & \ fbt_probetab_mask) @@ -29,8 +14,11 @@ extern int fbt_probetab_mask; extern void fbt_provide_probe_arch(fbt_probe_t *, int, int); extern void fbt_enable_arch(fbt_probe_t *, dtrace_id_t, void *); extern void fbt_disable_arch(fbt_probe_t *, dtrace_id_t, void *); +extern int fbt_can_patch_return_arch(asm_instr_t *); +extern int fbt_provide_module_arch(void *, struct module *); extern void fbt_provide_module(void *, struct module *); +extern void fbt_destroy_module(void *, struct module *); extern int _fbt_enable(void *, dtrace_id_t, void *); extern void _fbt_disable(void *, dtrace_id_t, void *); extern void fbt_destroy(void *, dtrace_id_t, void *); diff --git a/dtrace/fbt_mod.c b/dtrace/fbt_mod.c index fd04cf8df9028..e7d6fd18f30a4 100644 --- a/dtrace/fbt_mod.c +++ b/dtrace/fbt_mod.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010, 2011, 2012, 2013 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -47,16 +46,17 @@ static const dtrace_pattr_t fbt_attr = { DT_PROVIDER_POPS(fbt) static dtrace_pops_t fbt_pops = { - NULL, - fbt_provide_module, - fbt_enable, - fbt_disable, - NULL, - NULL, - NULL, - NULL, - NULL, - fbt_destroy + .dtps_provide = NULL, + .dtps_provide_module = fbt_provide_module, + .dtps_destroy_module = fbt_destroy_module, + .dtps_enable = fbt_enable, + .dtps_disable = fbt_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = NULL, + .dtps_getargval = NULL, + .dtps_usermode = NULL, + .dtps_destroy = fbt_destroy }; DT_PROVIDER_MODULE(fbt, DTRACE_PRIV_KERNEL) diff --git a/dtrace/fbt_sparc64.c b/dtrace/fbt_sparc64.c index 437536135822c..3a6367cd4debe 100644 --- a/dtrace/fbt_sparc64.c +++ b/dtrace/fbt_sparc64.c @@ -21,11 +21,12 @@ * * CDDL HEADER END * - * Copyright 2010-2014 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ +#include #include +#include #include #include @@ -65,11 +66,8 @@ * retl * nop */ -#ifndef FBT_TRAMP_SIZE -# error The kernel must define FBT_TRAMP_SIZE! -#elif FBT_TRAMP_SIZE < 13 -# error FBT_TRAMP_SIZE must be at least 13 instructions! -#endif +#define FBT_TRAMP_SIZE 19 + static void add_entry_tramp(fbt_probe_t *fbp, int nargs) { @@ -118,6 +116,173 @@ static void add_entry_tramp(fbt_probe_t *fbp, int nargs) fbp->fbp_patchval = ASM_CALL(fbp->fbp_patchpoint, trampoline); fbp->fbp_savedval = *fbp->fbp_patchpoint; + fbp->fbp_isret = 0; +} + +/* + * This function should always succeed as we have removed all problematic cases + * in advance. + */ +static void add_return_tramp(fbt_probe_t *fbp, int stype) +{ + struct module *mp = fbp->fbp_module; + dtrace_id_t id = fbp->fbp_id; + uint64_t roffset = fbp->fbp_roffset; + size_t idx = PDATA(mp)->fbt_probe_cnt; + asm_instr_t *trampoline = &(PDATA(mp)->fbt_tab[idx * + FBT_TRAMP_SIZE]); + asm_instr_t *instr = trampoline; + asm_instr_t instr_ret = *fbp->fbp_patchpoint; + asm_instr_t instr_delay = *(fbp->fbp_patchpoint + 1); + + uint32_t locals[ASM_REG_NLOCALS]; + uint32_t local, tmpreg, saved_g1, saved_g2; + + /* + * The RETURN case is bit more complex compared to others. The RETURN + * performs ret and restore at the same time. Thus the instruction in + * delay slot operates on callers register window. In case there is + * something else than NOP in the delay slot we have to do following: + * + * 1) RESTORE + * 2) Execute Delay slot + * 3) SAVE + * 4) Handle probe + * 5) Simulate return by pair of ret/restore + */ + if (ASM_FMT3_OP(instr_ret) == ASM_OP_RETURN) { + if (instr_delay != ASM_NOP) { + *instr++ = ASM_RESTORE(ASM_REG_G0, ASM_REG_G0, ASM_REG_G0); + *instr++ = instr_delay; + *instr++ = ASM_SAVEIMM(ASM_REG_O6, -ASM_MINFRAME, ASM_REG_O6); + } + + /* This is safe as we support only return %i7 + 8. */ + instr_ret = ASM_RET; + instr_delay = ASM_RESTORE(ASM_REG_G0, ASM_REG_G0, ASM_REG_G0); + } + + /* + * Now we need to remap arguments so we use safe location that is not + * destroyed during call to dtrace_probe(). This is done in multiple steps: + * + * 1) Mark currently used locals + * 2) Move non-locals to unused locals + * 3) Update instruction to use locals only + */ + ASM_REG_INITLOCALS(local, locals); + + /* Only JMPL needs extra care here. */ + if (ASM_FMT3_OP(instr_ret) == ASM_OP_JMPL) { + ASM_REG_MARKLOCAL(locals, ASM_FMT3_RS1(instr_ret)); + if (!ASM_FMT3_ISIMM(instr_ret)) { + ASM_REG_MARKLOCAL(locals, ASM_FMT3_RS2(instr_ret)); + } + } + + /* At this point isntr_delay can hold RESTORE only. */ + ASM_REG_MARKLOCAL(locals, ASM_FMT3_RS1(instr_delay)); + if (!ASM_FMT3_ISIMM(instr_delay)) { + ASM_REG_MARKLOCAL(locals, ASM_FMT3_RS2(instr_delay)); + } + + /* Remap */ + if (ASM_FMT3_OP(instr_ret) == ASM_OP_JMPL) { + tmpreg = ASM_FMT3_RS1(instr_ret); + + if (ASM_REG_ISVOLATILE(tmpreg)) { + ASM_REG_ALLOCLOCAL(local, locals); + *instr++ = ASM_MOV(tmpreg, local); + ASM_FMT3_RS1_SET(instr_ret, local); + } + + if (!ASM_FMT3_ISIMM(instr_ret)) { + tmpreg = ASM_FMT3_RS2(instr_ret); + + if (ASM_REG_ISVOLATILE(tmpreg)) { + ASM_REG_ALLOCLOCAL(local, locals); + *instr++ = ASM_MOV(tmpreg, local); + ASM_FMT3_RS2_SET(instr_ret, local); + } + } + } + + tmpreg = ASM_FMT3_RS1(instr_delay); + if (ASM_REG_ISVOLATILE(tmpreg)) { + ASM_REG_ALLOCLOCAL(local, locals); + *instr++ = ASM_MOV(tmpreg, local); + ASM_FMT3_RS1_SET(instr_delay, local); + } + + if (!ASM_FMT3_ISIMM(instr_delay)) { + tmpreg = ASM_FMT3_RS2(instr_delay); + + if (ASM_REG_ISVOLATILE(tmpreg)) { + ASM_REG_ALLOCLOCAL(local, locals); + *instr++ = ASM_MOV(tmpreg, local); + ASM_FMT3_RS2_SET(instr_delay, local); + } + } + + /* backup globals */ + ASM_REG_ALLOCLOCAL(local, locals); + saved_g1 = local; + *instr++ = ASM_MOV(ASM_REG_G1, saved_g1); + ASM_REG_ALLOCLOCAL(local, locals); + saved_g2 = local; + *instr++ = ASM_MOV(ASM_REG_G2, saved_g2); + + /* prepare arguments */ + if (id > (uint32_t)ASM_SIMM13_MAX) { + *instr++ = ASM_SETHI(id, ASM_REG_O0); + *instr++ = ASM_ORLO(ASM_REG_O0, id, ASM_REG_O0); + } else { + *instr++ = ASM_ORSIMM13(ASM_REG_G0, id, ASM_REG_O0); + } + + if (roffset > (uint32_t)ASM_SIMM13_MAX) { + *instr++ = ASM_SETHI(roffset, ASM_REG_O1); + *instr++ = ASM_ORLO(ASM_REG_O1, roffset, ASM_REG_O1); + } else { + *instr++ = ASM_ORSIMM13(ASM_REG_G0, roffset, ASM_REG_O1); + } + + /* fire probe */ + *instr = ASM_CALL(instr, dtrace_probe); + instr++; + + /* recover return value */ + if (ASM_FMT3_RD(instr_delay) == ASM_REG_O0) { + uint32_t instr_add = (instr_delay & ~ASM_FMT3_OP_MASK) | + ASM_OP_ADD; + instr_add = (instr_add & ~ASM_FMT3_RD_MASK) | + (ASM_REG_O2 << ASM_FMT3_RD_SHIFT); + *instr++ = instr_add; + } else { + *instr++ = ASM_MOV(ASM_REG_I0, ASM_REG_O2); + } + + /* restore globals */ + *instr++ = ASM_MOV(saved_g1, ASM_REG_G1); + *instr++ = ASM_MOV(saved_g2, ASM_REG_G2); + + /* + * Emit original instruction return pair. In case of call update it's + * label to correct value. + */ + if (ASM_FMT1_OP(instr_ret) == ASM_OP_CALL) { + asm_instr_t *dest = fbp->fbp_patchpoint + ASM_FMT1_DISP30(instr_ret); + *instr = ASM_CALL(instr, dest); + instr++; + } else { + *instr++ = instr_ret; + } + *instr++ = instr_delay; + + fbp->fbp_patchval = ASM_TA(0x75); + fbp->fbp_savedval = *fbp->fbp_patchpoint; + fbp->fbp_isret = 1; + fbp->fbp_trampdest = trampoline; } void fbt_provide_probe_arch(fbt_probe_t *fbp, int type, int stype) @@ -125,11 +290,10 @@ void fbt_provide_probe_arch(fbt_probe_t *fbp, int type, int stype) switch (type) { case FBT_ENTRY: add_entry_tramp(fbp, stype); - return; + break; case FBT_RETURN: - pr_info("%s: %s: FBT_RETURN not supported yet\n", - __func__, fbp->fbp_name); - return; + add_return_tramp(fbp, stype); + break; default: pr_info("%s: %s: Unknown FBT type %d\n", __func__, fbp->fbp_name, type); @@ -137,21 +301,168 @@ void fbt_provide_probe_arch(fbt_probe_t *fbp, int type, int stype) } } +/* + * We filetered out unsupported return probes (DCTIs, PC-relative instructions + * in the delay slot) in kernel. Rest of the logic is in the module to give us + * flexibility when we need to alter the logic later. + * + * At the moment we rely on the fact that every supported function has SAVE in + * its prologue. Thus there is no need to support RETL stuff. A call to probe + * may destroy globals and outs so they are not supported. It is in theory + * possible to support RETL but current mechanism of FBT guarantees it is going + * to destroy value in %o7 but that would need bigger changes to how we allocate + * trampolines. + * + * Possible return cases: + * + * 1) ret/restore + * 2) return/delay + * 3) call/restore + * + * The function assumes that it is safe to touch instruction at (addr + 1) to + * access delay slot. + */ +int fbt_can_patch_return_arch(asm_instr_t *addr) +{ + int rd; + + /* RETURN %i7, 8*/ + if (ASM_FMT3_OP(*addr) == ASM_OP_RETURN && + *addr == ASM_RETURN(ASM_REG_I7, 8)) { + return 1; + } + + /* RESTORE in delay */ + if (ASM_FMT3_OP(*(addr + 1)) != ASM_OP_RESTORE) + return 0; + + /* CALL */ + if (ASM_FMT1_OP(*addr) == ASM_OP_CALL) + return 1; + + /* JMPL %i7 + 8, %g0 */ + if (ASM_FMT3_OP(*addr) != ASM_OP_JMPL) + return 0; + + rd = ASM_FMT3_RD(*addr); + if (rd == ASM_REG_G0 || rd == ASM_REG_I7) + return 1; + + /* unsupported */ + return 0; +} + +static void *fbt_count_probe(struct module *mp, char *func, int type, + int stype, asm_instr_t *addr, uint64_t offset, + void *pfbt, void *arg) +{ + static int dummy; + size_t *count = arg; + + switch (type) { + case FBT_ENTRY: + (*count)++; + return NULL; + case FBT_RETURN: + if (!fbt_can_patch_return_arch(addr)) + return pfbt; + + (*count)++; + if (pfbt == NULL) + return &dummy; + return pfbt; + default: + printk(KERN_INFO "FBT: Invalid probe type %d (%d) for %s\n", + type, stype, func); + return NULL; + } +} + +int fbt_provide_module_arch(void *arg, struct module *mp) +{ + size_t probe_cnt = 0; + + /* First estimate the size of trampoline we need */ + dtrace_fbt_init((fbt_add_probe_fn)fbt_count_probe, mp, &probe_cnt); + + if (probe_cnt > 0 && PDATA(mp)->fbt_tab == NULL) { + asm_instr_t *tramp = dtrace_alloc_text(mp, probe_cnt * + FBT_TRAMP_SIZE * + sizeof (asm_instr_t)); + + if (tramp == NULL) { + printk(KERN_INFO "FBT: can't allocate FBT trampoline" + " for %s\n", mp->name); + return 0; + } + + PDATA(mp)->fbt_tab = tramp; + return 1; + } + + return 0; +} + +void fbt_destroy_module(void *arg, struct module *mp) +{ + if (PDATA(mp)->fbt_tab != NULL) { + dtrace_free_text(PDATA(mp)->fbt_tab); + PDATA(mp)->fbt_tab = NULL; + } +} + void fbt_enable_arch(fbt_probe_t *fbp, dtrace_id_t id, void *arg) { *fbp->fbp_patchpoint = fbp->fbp_patchval; + flushi(fbp->fbp_patchpoint); } void fbt_disable_arch(fbt_probe_t *fbp, dtrace_id_t id, void *arg) { *fbp->fbp_patchpoint = fbp->fbp_savedval; + flushi(fbp->fbp_patchpoint); +} + +static void fbt_handler(struct pt_regs *regs) +{ + fbt_probe_t *fbp = fbt_probetab[FBT_ADDR2NDX(regs->tpc)]; + + for(; fbp != NULL; fbp = fbp->fbp_hashnext) { + if ((uintptr_t)fbp->fbp_patchpoint == regs->tpc) { + regs->tpc = (uintptr_t)fbp->fbp_trampdest; + regs->tnpc = regs->tpc + 4; + + return; + } + } + + /* + * The only way that ends here is that we hit our trap in kernel mode. + * The trap is not shared with anyone else so it means we have lost a + * tracpoint somehow. We must die as there is no safe way how we could + * restore original instruction stream. + */ + dtrace_panic(KERN_EMERG, "FBT trap without a probe at %p", + regs->tpc); } int fbt_dev_init_arch(void) { - return 0; + fbt_probetab_mask = fbt_probetab_size - 1; + fbt_probetab = dtrace_vzalloc_try(fbt_probetab_size * + sizeof (fbt_probe_t *)); + + if (fbt_probetab == NULL) + return -ENOMEM; + + return dtrace_fbt_set_handler(fbt_handler); } void fbt_dev_exit_arch(void) { + vfree(fbt_probetab); + fbt_probetab_mask = 0; + fbt_probetab_size = 0; + + (void) dtrace_fbt_set_handler(NULL); } diff --git a/dtrace/fbt_x86_64.c b/dtrace/fbt_x86_64.c index dd3b1527654b4..378ff133df457 100644 --- a/dtrace/fbt_x86_64.c +++ b/dtrace/fbt_x86_64.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010-2014 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -92,6 +91,20 @@ void fbt_provide_probe_arch(fbt_probe_t *fbp, int type, int stype) : DTRACE_INVOP_RET; } +int fbt_can_patch_return_arch(asm_instr_t *addr) +{ + return 1; +} + +int fbt_provide_module_arch(void *arg, struct module *mp) +{ + return 1; +} + +void fbt_destroy_module(void *arg, struct module *mp) +{ +} + void fbt_enable_arch(fbt_probe_t *fbp, dtrace_id_t id, void *arg) { dtrace_invop_enable(fbp->fbp_patchpoint, fbp->fbp_patchval); @@ -104,10 +117,21 @@ void fbt_disable_arch(fbt_probe_t *fbp, dtrace_id_t id, void *arg) int fbt_dev_init_arch(void) { + fbt_probetab_mask = fbt_probetab_size - 1; + fbt_probetab = dtrace_vzalloc_try(fbt_probetab_size * + sizeof (fbt_probe_t *)); + + if (fbt_probetab == NULL) + return -ENOMEM; + return dtrace_invop_add(fbt_invop); } void fbt_dev_exit_arch(void) { + vfree(fbt_probetab); + fbt_probetab_mask = 0; + fbt_probetab_size = 0; + dtrace_invop_remove(fbt_invop); } diff --git a/dtrace/include/dtrace/provider.h b/dtrace/include/dtrace/provider.h index c5a3a4f252d62..7c2a91b35dff0 100644 --- a/dtrace/include/dtrace/provider.h +++ b/dtrace/include/dtrace/provider.h @@ -28,8 +28,7 @@ * * CDDL HEADER END * - * Copyright 2009 -- 2013 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2009, 2017, Oracle and/or its affiliates. All rights reserved. */ /* @@ -57,6 +56,7 @@ * dtps_getargval() <-- Get the value for an argX or args[X] variable * dtps_usermode() <-- Find out if the probe was fired in user mode * dtps_destroy() <-- Destroy all state associated with this probe + * dtps_destroy_module() <-- Destroy per-module data * * 1.2 void dtps_provide(void *arg, const dtrace_probedesc_t *spec) * @@ -355,6 +355,22 @@ * back into at all. mod_lock is held. cpu_lock is not held, and may not be * acquired. * + * 1.12 void dtps_destroy_module(void *arg, struct modctl *mp) + * + * 1.12.1 Overview + * + * Called to notify provider that it can remove any per-module data. + * + * 1.12.2 Arguments and notes + * + * The first argument is the cookie as passed to dtrace_register(). The + * second argument is a pointer to a struct module structure that points to + * the module for which data may be cleared. + * + * 1.12.3 Return value + * + * None. + * * * 2 Provider-to-Framework API * @@ -716,6 +732,7 @@ typedef struct dtrace_pops { uint64_t (*dtps_getargval)(void *, dtrace_id_t, void *, int, int); int (*dtps_usermode)(void *, dtrace_id_t, void *); void (*dtps_destroy)(void *, dtrace_id_t, void *); + void (*dtps_destroy_module)(void *, struct module *); } dtrace_pops_t; typedef struct dtrace_helper_probedesc { diff --git a/dtrace/include/sparc64/dtrace/fbt_arch.h b/dtrace/include/sparc64/dtrace/fbt_arch.h index 473f4a3555368..173a107349583 100644 --- a/dtrace/include/sparc64/dtrace/fbt_arch.h +++ b/dtrace/include/sparc64/dtrace/fbt_arch.h @@ -28,10 +28,26 @@ * * CDDL HEADER END * - * Copyright 2009-2014 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2009, 2017, Oracle and/or its affiliates. All rights reserved. */ #define FBT_AFRAMES 1 +typedef struct fbt_probe { + char *fbp_name; /* name of probe */ + dtrace_id_t fbp_id; /* probe ID */ + struct module *fbp_module; /* defining module */ + int fbp_loadcnt; /* load count for module */ + int fbp_primary; /* non-zero if primary mod */ + asm_instr_t *fbp_patchpoint;/* patch point */ + asm_instr_t fbp_patchval; /* instruction to patch */ + asm_instr_t fbp_savedval; /* saved instruction value */ + uint64_t fbp_roffset; /* relative offset */ + int fbp_rval; + struct fbt_probe *fbp_next; /* next probe */ + struct fbt_probe *fbp_hashnext; /* next on hash */ + int fbp_isret; + asm_instr_t *fbp_trampdest; +} fbt_probe_t; + #endif /* _SPARC64_FBT_ARCH_H */ diff --git a/dtrace/include/sparc64/dtrace/isa_arch.h b/dtrace/include/sparc64/dtrace/isa_arch.h index 1ad496f3c1266..3a762769be3bb 100644 --- a/dtrace/include/sparc64/dtrace/isa_arch.h +++ b/dtrace/include/sparc64/dtrace/isa_arch.h @@ -24,8 +24,7 @@ * * CDDL HEADER END * - * Copyright 2016 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyrigt (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. */ @@ -77,7 +76,7 @@ #define ASM_REG_ISGLOBAL(r) ((r) >= ASM_REG_G0 && (r) <= ASM_REG_G7) #define ASM_REG_ISOUTPUT(r) ((r) >= ASM_REG_O0 && (r) <= ASM_REG_O7) -#define ASM_REG_ISLOCAL(r) ((r) >= ASM_REH_L0 && (r) <= ASM_REG_L7) +#define ASM_REG_ISLOCAL(r) ((r) >= ASM_REG_L0 && (r) <= ASM_REG_L7) #define ASM_REG_ISINPUT(r) ((r) >= ASM_REG_I0 && (r) <= ASM_REG_I7) #define ASM_REG_ISVOLATILE(r) \ ((ASM_REG_ISGLOBAL(r) || ASM_REG_ISOUTPUT(r)) && (r) != ASM_REG_G0) @@ -144,6 +143,8 @@ #define ASM_FMT3_OP_MASK 0xc1f80000 #define ASM_FMT3_OP(val) ((val) & ASM_FMT3_OP_MASK) +#define ASM_FMT3_OP_TCC (0x3a << ASM_FMT3_OP3_SHIFT) + #define ASM_FMT3_RD_SHIFT 25 #define ASM_FMT3_RD_MASK (0x1f << ASM_FMT3_RD_SHIFT) #define ASM_FMT3_RD(val) \ @@ -203,6 +204,7 @@ #define ASM_OP_BGE (ASM_OP0 | ASM_FMT2_OP2_BCC | ASM_FMT2_COND_BGE) #define ASM_OP_BAPCC (ASM_OP0 | ASM_FMT2_OP2_BPCC | ASM_FMT2_COND_BA) #define ASM_OP_RD (ASM_OP2 | (0x28 << ASM_FMT3_OP3_SHIFT)) +#define ASM_OP_TA (ASM_OP2 | ASM_FMT3_OP_TCC | ASM_FMT2_COND_BA) #define ASM_ORLO(rs, val, rd) \ (ASM_OP_OR | ((rs) << ASM_FMT3_RS1_SHIFT) | \ @@ -243,6 +245,8 @@ #define ASM_CALL(orig, dest) (ASM_OP_CALL | ASM_DISP30(orig, dest)) +#define ASM_TA(lvl) (ASM_OP_TA | ASM_FMT3_IMM | ((lvl) & 0xff)) + #define ASM_RET \ (ASM_OP_JMPL | (ASM_REG_I7 << ASM_FMT3_RS1_SHIFT) | \ (ASM_REG_G0 << ASM_FMT3_RD_SHIFT) | ASM_FMT3_IMM | \ @@ -253,6 +257,7 @@ (ASM_REG_G0 << ASM_FMT3_RD_SHIFT) | ASM_FMT3_IMM | \ (sizeof (asm_instr_t) << 1)) +#define ASM_MINFRAME (22 * 8) #define ASM_SAVEIMM(rd, val, rs1) \ (ASM_OP_SAVE | ((rs1) << ASM_FMT3_RS1_SHIFT) | \ ((rd) << ASM_FMT3_RD_SHIFT) | ASM_FMT3_IMM | ((val) & ASM_SIMM13_MASK)) diff --git a/dtrace/include/x86_64/dtrace/fbt_arch.h b/dtrace/include/x86_64/dtrace/fbt_arch.h index 0b9a269d77b49..afda80b523894 100644 --- a/dtrace/include/x86_64/dtrace/fbt_arch.h +++ b/dtrace/include/x86_64/dtrace/fbt_arch.h @@ -28,10 +28,24 @@ * * CDDL HEADER END * - * Copyright 2017 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. */ #define FBT_AFRAMES 9 +typedef struct fbt_probe { + char *fbp_name; /* name of probe */ + dtrace_id_t fbp_id; /* probe ID */ + struct module *fbp_module; /* defining module */ + int fbp_loadcnt; /* load count for module */ + int fbp_primary; /* non-zero if primary mod */ + asm_instr_t *fbp_patchpoint;/* patch point */ + asm_instr_t fbp_patchval; /* instruction to patch */ + asm_instr_t fbp_savedval; /* saved instruction value */ + uintptr_t fbp_roffset; /* relative offset */ + int fbp_rval; + struct fbt_probe *fbp_next; /* next probe */ + struct fbt_probe *fbp_hashnext; /* next on hash */ +} fbt_probe_t; + #endif /* _X86_64_FBT_ARCH_H */ diff --git a/dtrace/profile_mod.c b/dtrace/profile_mod.c index e3cc2b16b449f..a2d9f10519ce8 100644 --- a/dtrace/profile_mod.c +++ b/dtrace/profile_mod.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010, 2011, 2012 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -47,16 +46,17 @@ static const dtrace_pattr_t profile_attr = { DT_PROVIDER_POPS(profile) static dtrace_pops_t profile_pops = { - profile_provide, - NULL, - profile_enable, - profile_disable, - NULL, - NULL, - NULL, - NULL, - profile_usermode, - profile_destroy, + .dtps_provide = profile_provide, + .dtps_provide_module = NULL, + .dtps_destroy_module = NULL, + .dtps_enable = profile_enable, + .dtps_disable = profile_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = NULL, + .dtps_getargval = NULL, + .dtps_usermode = profile_usermode, + .dtps_destroy = profile_destroy, }; DT_PROVIDER_MODULE(profile, DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER) diff --git a/dtrace/sdt_dev.c b/dtrace/sdt_dev.c index a672fb57561a2..01a5e16c75571 100644 --- a/dtrace/sdt_dev.c +++ b/dtrace/sdt_dev.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010-2016 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -279,6 +278,12 @@ void sdt_provide_module(void *arg, struct module *mp) if (mp->sdt_probec == 0) return; + /* + * Nothing if arch specific module setup fails. + */ + if (!sdt_provide_module_arch(NULL, mp)) + return; + /* * Do not provide any probes unless all SDT providers have been created * for this meta-provider. diff --git a/dtrace/sdt_impl.h b/dtrace/sdt_impl.h index 785dc4fd02af3..70cd704612cd9 100644 --- a/dtrace/sdt_impl.h +++ b/dtrace/sdt_impl.h @@ -53,7 +53,7 @@ extern void sdt_enable_arch(sdt_probe_t *, dtrace_id_t, void *); extern void sdt_disable_arch(sdt_probe_t *, dtrace_id_t, void *); extern void sdt_provide_module(void *, struct module *); -extern void sdt_cleanup_module(void *, struct module *); +extern void sdt_destroy_module(void *, struct module *); extern int _sdt_enable(void *, dtrace_id_t, void *); extern void _sdt_disable(void *, dtrace_id_t, void *); extern void sdt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); diff --git a/dtrace/sdt_mod.c b/dtrace/sdt_mod.c index 8f24a7ec86d22..aa45e0c0c72e5 100644 --- a/dtrace/sdt_mod.c +++ b/dtrace/sdt_mod.c @@ -22,8 +22,7 @@ * * CDDL HEADER END * - * Copyright 2010-2016 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -120,20 +119,21 @@ static dtrace_pattr_t perf_attr = { DT_PROVIDER_POPS(sdt) static dtrace_pops_t sdt_pops = { - NULL, - sdt_provide_module, - sdt_enable, - sdt_disable, - NULL, - NULL, - sdt_getargdesc, + .dtps_provide = NULL, + .dtps_provide_module = sdt_provide_module, + .dtps_destroy_module = sdt_destroy_module, + .dtps_enable = sdt_enable, + .dtps_disable = sdt_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = sdt_getargdesc, #ifdef CONFIG_X86_64 - sdt_getarg, + .dtps_getargval = sdt_getarg, #else - NULL, + .dtps_getargval = NULL, #endif - NULL, - sdt_destroy, + .dtps_usermode = NULL, + .dtps_destroy = sdt_destroy, }; dtrace_mprovider_t sdt_providers[] = { diff --git a/dtrace/sdt_sparc64.c b/dtrace/sdt_sparc64.c index 1c744ba11bdc9..3a40518bbee0e 100644 --- a/dtrace/sdt_sparc64.c +++ b/dtrace/sdt_sparc64.c @@ -21,14 +21,14 @@ * * CDDL HEADER END * - * Copyright 2010-2016 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include #include #include #include +#include #include #include "dtrace.h" @@ -64,11 +64,7 @@ * For is-enabled probes, we just drop an "or %g0, 1, %o0" * directly into the delay slot. */ -#ifndef SDT_TRAMP_SIZE -# error The kernel must define SDT_TRAMP_SIZE! -#elif SDT_TRAMP_SIZE < 11 -# error SDT_TRAMP_SIZE must be at least 11 instructions! -#endif +#define SDT_TRAMP_SIZE 11 #define SA(x) ((long)ALIGN((x), 4)) #define MINFRAME STACKFRAME_SZ @@ -166,32 +162,43 @@ void sdt_provide_probe_arch(sdt_probe_t *sdp, struct module *mp, int idx) sdp->sdp_savedval = *sdp->sdp_patchpoint; } +/* + * Allocates SDT trampoline that is executable. + */ int sdt_provide_module_arch(void *arg, struct module *mp) { - /* - * The vmlinux pseudo-module (core Linux kernel) is a special case... - */ - if (mp == dtrace_kmod && PDATA(mp)->sdt_tab == NULL) { - PDATA(mp)->sdt_tab = (asm_instr_t *)ALIGN( - (uintptr_t)PDATA(mp) + sizeof(dtrace_module_t), 8); - return 1; + if (mp->sdt_probec > 0 && PDATA(mp)->sdt_tab == NULL) { + asm_instr_t *tramp = dtrace_alloc_text(mp, mp->sdt_probec * + SDT_TRAMP_SIZE * + sizeof (asm_instr_t)); + + if (tramp == NULL) + return 0; + + PDATA(mp)->sdt_tab = tramp; } return 1; } -void sdt_cleanup_module(void *dmy, struct module *mp) +void sdt_destroy_module(void *arg, struct module *mp) { + if (PDATA(mp)->sdt_tab != NULL) { + dtrace_free_text(PDATA(mp)->sdt_tab); + PDATA(mp)->sdt_tab = NULL; + } } void sdt_enable_arch(sdt_probe_t *sdp, dtrace_id_t id, void *arg) { *sdp->sdp_patchpoint = sdp->sdp_patchval; + flushi(sdp->sdp_patchpoint); } void sdt_disable_arch(sdt_probe_t *sdp, dtrace_id_t id, void *arg) { *sdp->sdp_patchpoint = sdp->sdp_savedval; + flushi(sdp->sdp_patchpoint); } int sdt_dev_init_arch(void) diff --git a/dtrace/sdt_x86_64.c b/dtrace/sdt_x86_64.c index 546cf30933aca..c91019eba522f 100644 --- a/dtrace/sdt_x86_64.c +++ b/dtrace/sdt_x86_64.c @@ -67,6 +67,15 @@ void sdt_provide_probe_arch(sdt_probe_t *sdp, struct module *mp, int idx) sdp->sdp_savedval = *sdp->sdp_patchpoint; } +int sdt_provide_module_arch(void *arg, struct module *mp) +{ + return 1; +} + +void sdt_destroy_module(void *arg, struct module *mp) +{ +} + void sdt_enable_arch(sdt_probe_t *sdp, dtrace_id_t id, void *arg) { dtrace_invop_enable(sdp->sdp_patchpoint, sdp->sdp_patchval); diff --git a/dtrace/systrace_mod.c b/dtrace/systrace_mod.c index 3f89f8ea2f59b..3d1f258f74555 100644 --- a/dtrace/systrace_mod.c +++ b/dtrace/systrace_mod.c @@ -21,8 +21,7 @@ * * CDDL HEADER END * - * Copyright 2010, 2011 Oracle, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -47,16 +46,17 @@ static const dtrace_pattr_t syscall_attr = { DT_PROVIDER_POPS(systrace) static dtrace_pops_t syscall_pops = { - systrace_provide, - NULL, - systrace_enable, - systrace_disable, - NULL, - NULL, - NULL, - NULL, - NULL, - systrace_destroy + .dtps_provide = systrace_provide, + .dtps_provide_module = NULL, + .dtps_destroy_module = NULL, + .dtps_enable = systrace_enable, + .dtps_disable = systrace_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = NULL, + .dtps_getargval = NULL, + .dtps_usermode = NULL, + .dtps_destroy = systrace_destroy }; DT_PROVIDER_MODULE(syscall, DTRACE_PRIV_USER)