From: Tomas Jedlicka Date: Fri, 7 Apr 2017 20:51:53 +0000 (-0400) Subject: dtrace: FBT module support and SPARCs return probes X-Git-Tag: v4.1.12-106.0.20170720_1900~42^2 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=d781a97e9071a105bdac06cb75ab675b993abb51;p=users%2Fjedix%2Flinux-maple.git dtrace: FBT module support and SPARCs return probes This fix adds two features to FBT provider: 1) support for modules 2) support for return probes on SPARC The module support of x86 was almost ready as it does not rely on trampolines and uses hashtables of tracepoints. This works well if we know amount of probes in advance so can reserve correct amount of memory during module load time. Unfortunately that is not possible on SPARC and we need to allocate a trampoline dynamically. Major part of this code is about removing all static assumptions about FBT from kernel code and moving the responsibility to dtrace modules. Trampolines for SPARC are now allocated dynamically (including kernel's pseudo module). This applies to SDT trampolines too. Second change adds scan for return probes on SPARC with small heuristics to quickly skip over cases that are not interesting for DTrace. At the same time this patch allocates new SPARC Trap for FBT. Support for .init section is not available on any platform. The .init section is freed after a module is fully loaded and it is not possible to remove its probes without further chagnes in DTrace framework (modules). This is deffered for later work. Orabug: 25960276 Orabug: 26384199 Signed-off-by: Tomas Jedlicka Reviewed-by: Kris Van Hees Reviewed-by: Rob Gardner --- diff --git a/arch/sparc/include/asm/dtrace_arch.h b/arch/sparc/include/asm/dtrace_arch.h index 80ee42b8b524..ff21a2aa0c14 100644 --- a/arch/sparc/include/asm/dtrace_arch.h +++ b/arch/sparc/include/asm/dtrace_arch.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2013,2014 Oracle, Inc. */ +/* Copyright (c) 2013, 2017, Oracle and/or its affiliates. All rights reserved. */ #ifndef _SPARC_DTRACE_ARCH_H #define _SPARC_DTRACE_ARCH_H @@ -7,53 +7,6 @@ typedef uint32_t asm_instr_t; -/* - * Maximum size (in instruction count) of SDT and FBT trampolines. - */ -#define SDT_TRAMP_SIZE 11 -#define FBT_TRAMP_SIZE 13 - -/* - * Maximum number of SDT and FBT probes. The actual number available to DTRACE - * may be lower due to runtime filtering of troublesome functions. - */ -#define DTRACE_SDT_MAX(mp) (mp->sdt_probec) -#define DTRACE_FBT_MAX(mp) (mp->num_ftrace_callsites) - -/* - * The following macros are used to partition the PDATA memory block. The SDT - * trampolines are stored first, followed by the FBT trampolines. - * - * DTRACE_PD_SDT_OFF: - * Offset (in the PDATA memory block) for space to store SDT trampolines. - * DTRACE_PD_FBT_OFF: - * Offset (in the PDATA memory block) for space to store FBT trampolines. - * DTRACE_PD_MAXSIZE: - * Maximum size of the PDATA memory block (if no SDT or FBT probes get - * filtered out). - * DTRACE_PD_MAXSIZE: - * Maximum size of the PDATA memory bock for the kernel pseudo-module. - * There is a separate macro for this because (at boot time) the maximum - * number of SDT and FBT probes is stored in global constants. Wehn the - * kernel pseudo-module is initialized, the value of those constants is - * assigned to the appropriate module struct members so that the macros - * above (DTRACE_SDT_MAX and DTRACE_FBT_MAX) can be used after that point. - */ -#define DTRACE_PD_SDT_OFF_(sc, fc) 0 -#define DTRACE_PD_SDT_OFF(mp) DTRACE_PD_SDT_OFF_(DTRACE_SDT_MAX(mp), \ - DTRACE_FBT_MAX(mp)) -#define DTRACE_PD_FBT_OFF_(sc, fc) (DTRACE_PD_SDT_OFF_((sc), (fc)) + \ - (sc) * SDT_TRAMP_SIZE * \ - sizeof(asm_instr_t)) -#define DTRACE_PD_FBT_OFF(mp) DTRACE_PD_FBT_OFF_(DTRACE_SDT_MAX(mp), \ - DTRACE_FBT_MAX(mp)) -#define DTRACE_PD_MAXSIZE_(sc, fc) (DTRACE_PD_FBT_OFF_((sc), (fc)) + \ - (fc) * FBT_TRAMP_SIZE * \ - sizeof(asm_instr_t)) -#define DTRACE_PD_MAXSIZE(mp) DTRACE_PD_MAXSIZE_(DTRACE_SDT_MAX(mp), \ - DTRACE_FBT_MAX(mp)) - -#define DTRACE_PD_MAXSIZE_KERNEL DTRACE_PD_MAXSIZE_(dtrace_sdt_nprobes, \ - dtrace_fbt_nfuncs) +asmlinkage void dtrace_fbt_trap(unsigned long, struct pt_regs *); #endif /* _SPARC_DTRACE_ARCH_H */ diff --git a/arch/sparc/include/asm/dtrace_util.h b/arch/sparc/include/asm/dtrace_util.h index 32f2158dde07..d930b9789ed9 100644 --- a/arch/sparc/include/asm/dtrace_util.h +++ b/arch/sparc/include/asm/dtrace_util.h @@ -1,8 +1,12 @@ -/* Copyright (C) 2013,2014 Oracle, Inc. */ +/* Copyright (C) 2013, 2017, Oracle and/or its affiliates. All rights reserved. */ #ifndef _SPARC_DTRACE_UTIL_H #define _SPARC_DTRACE_UTIL_H +#include + extern int dtrace_user_addr_is_exec(uintptr_t); +extern int dtrace_fbt_set_handler(void (*func)(struct pt_regs *)); + #endif /* _SPARC_DTRACE_UTIL_H */ diff --git a/arch/sparc/include/asm/ttable.h b/arch/sparc/include/asm/ttable.h index 71b5a67522ab..ebb700268f61 100644 --- a/arch/sparc/include/asm/ttable.h +++ b/arch/sparc/include/asm/ttable.h @@ -192,6 +192,12 @@ #define KGDB_TRAP(lvl) TRAP_ARG(bad_trap, lvl) #endif +#ifdef CONFIG_DTRACE +#define DTRACE_FBT_TRAP(lvl) TRAP_IRQ(dtrace_fbt_trap, lvl) +#else +#define DTRACE_FBT_TRAP(lvl) TRAP_ARG(bad_trap, lvl) +#endif + #define SUN4V_ITSB_MISS \ ldxa [%g0] ASI_SCRATCHPAD, %g2; \ ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4; \ diff --git a/arch/sparc/kernel/dtrace_fbt.c b/arch/sparc/kernel/dtrace_fbt.c index e1c07b24b189..3d3707a76c43 100644 --- a/arch/sparc/kernel/dtrace_fbt.c +++ b/arch/sparc/kernel/dtrace_fbt.c @@ -2,17 +2,20 @@ * FILE: dtrace_fbt.c * DESCRIPTION: Dynamic Tracing: FBT registration code (arch-specific) * - * Copyright (C) 2010-2016 Oracle Corporation + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include #include +#include #include #include +#include #include #include #include #include +#include #include #include @@ -51,6 +54,7 @@ #define ASM_IMM22_SHIFT 10 #define ASM_OP0 (((uint32_t)0) << ASM_OP_SHIFT) +#define ASM_OP1 (((uint32_t)1) << ASM_OP_SHIFT) #define ASM_OP2 (((uint32_t)2) << ASM_OP_SHIFT) #define ASM_FMT3_OP3_SHIFT 19 @@ -98,7 +102,10 @@ #define ASM_FMT2_COND_BGE (0xb << ASM_FMT2_COND_SHIFT) #define ASM_OP_SAVE (ASM_OP2 | (0x3c << ASM_FMT3_OP3_SHIFT)) +#define ASM_OP_JMPL (ASM_OP2 | (0x38 << ASM_FMT3_OP3_SHIFT)) +#define ASM_OP_RETURN (ASM_OP2 | (0x39 << ASM_FMT3_OP3_SHIFT)) #define ASM_OP_SETHI (ASM_OP0 | ASM_FMT2_OP2_SETHI) +#define ASM_OP_RD (ASM_OP2 | (0x28 << ASM_FMT3_OP3_SHIFT)) #define ASM_SETHI(val, reg) \ (ASM_OP_SETHI | (reg << ASM_FMT2_RD_SHIFT) | \ @@ -117,6 +124,22 @@ ASM_FMT3_RS1(instr) == ASM_REG_O6 && \ !(ASM_FMT3_ISIMM(instr) && ASM_FMT3_SIMM13(instr) == 0)) +#define ASM_IS_RDPC(instr) ((ASM_FMT3_OP(instr) == ASM_OP_RD) && \ + (ASM_FMT3_RD(instr) == ASM_REG_PC)) + +#define ASM_IS_PCRELATIVE(instr) \ + ((((instr) & ASM_OP_MASK) == ASM_OP0 && \ + ((instr) & ASM_FMT2_OP2_MASK) != ASM_FMT2_OP2_SETHI) || \ + ((instr) & ASM_OP_MASK) == ASM_OP1 || \ + ASM_IS_RDPC(instr)) + +#define ASM_IS_CTI(instr) \ + ((((instr) & ASM_OP_MASK) == ASM_OP0 && \ + ((instr) & ASM_FMT2_OP2_MASK) != ASM_FMT2_OP2_SETHI) || \ + ((instr) & ASM_OP_MASK) == ASM_OP1 || \ + (ASM_FMT3_OP(instr) == ASM_OP_JMPL) || \ + (ASM_FMT3_OP(instr) == ASM_OP_RETURN)) + #define ASM_IS_NOP(instr) ((instr) == ASM_NOP) #define ASM_MOD_INPUTS(instr) (ASM_OP(instr) == ASM_OP2 && \ @@ -140,11 +163,11 @@ dtrace_fbt_populate_bl(void) #undef BL_SENTRY }; -void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe) +void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe, struct module *mp, + void *arg) { loff_t pos; struct kallsym_iter sym; - size_t blpos = 0; asm_instr_t *paddr = NULL; dt_fbt_bl_entry_t *blent = NULL; @@ -158,7 +181,8 @@ void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe) pos = 0; kallsyms_iter_reset(&sym, 0); while (kallsyms_iter_update(&sym, pos++)) { - asm_instr_t *addr, *end; + asm_instr_t *addr, *end, *ins; + void *fbtp = NULL; /* * There is no point considering non-function symbols for FBT, @@ -178,9 +202,18 @@ void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe) continue; /* - * Only core kernel symbols are of interest here. + * Handle only symbols that belong to the module we have been + * asked for. + */ + if (mp == dtrace_kmod && !core_kernel_text(sym.value)) + continue; + + /* + * Ensure we have not been given .init symbol from kallsyms + * interface. This could lead to memory corruption once DTrace + * tries to enable probe in already freed memory. */ - if (!core_kernel_text(sym.value)) + if (mp != dtrace_kmod && !within_module_core(sym.value, mp)) continue; /* @@ -224,13 +257,12 @@ void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe) paddr = addr; if (ASM_IS_SAVE(*addr)) { - asm_instr_t *ins = addr; - /* * If there are other saves, this function has multiple * entry points or some other complex construct - we'll * skip it. */ + ins = addr; while (++ins < end) { if (ASM_IS_SAVE(*ins)) break; @@ -262,10 +294,69 @@ void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe) ASM_MOD_OUTPUTS(*(addr + 2)))) continue; - fbt_add_probe(dtrace_kmod, sym.name, FBT_ENTRY, 32, - addr + 1, 0, NULL); + fbt_add_probe(mp, sym.name, FBT_ENTRY, 32, + addr + 1, 0, NULL, arg); } else continue; + + /* Scan function for possible return probes. */ + for (ins = addr; ins + 1 < end; ins++) { + + /* Only CTIs may become return probe. */ + if (!ASM_IS_CTI(*ins)) + continue; + + /* + * Check the delay slot for incompatible instructions: + * - DCTI + * - PC relative instruction + * + * More detailed analysis is performed in the fbt module. + */ + if (ASM_IS_CTI(*(ins + 1))) + continue; + + if (ASM_IS_PCRELATIVE(*(ins + 1))) + continue; + + /* Create or update the return probe. */ + fbtp = fbt_add_probe(mp, sym.name, FBT_RETURN, 32, ins, + (uintptr_t)ins - (uintptr_t)addr, + fbtp, arg); + } } } EXPORT_SYMBOL(dtrace_fbt_init); + +static void (*fbt_handler)(struct pt_regs *) = NULL; + +int dtrace_fbt_set_handler(void (*func)(struct pt_regs *)) +{ + fbt_handler = func; + return 0; +} +EXPORT_SYMBOL(dtrace_fbt_set_handler); + +asmlinkage void dtrace_fbt_trap(unsigned long traplevel, struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + + if (user_mode(regs)) { + local_irq_enable(); + bad_trap(regs, traplevel); + goto out; + } + + /* + * If we take this trap and fbt_handler is not set we are out of luck. + * Since we don't know why the trap fired (it should never happen in + * DTrace code unless fbt_handler is set), there is no way of knowing + * whether it is safe to just do nothing. + */ + BUG_ON(fbt_handler == NULL); + + fbt_handler(regs); + +out: + exception_exit(prev_state); +} diff --git a/arch/sparc/kernel/dtrace_util.c b/arch/sparc/kernel/dtrace_util.c index 47e19175cd7b..77d68597bef9 100644 --- a/arch/sparc/kernel/dtrace_util.c +++ b/arch/sparc/kernel/dtrace_util.c @@ -2,7 +2,7 @@ * FILE: dtrace_util.c * DESCRIPTION: Dynamic Tracing: Architecture utility functions * - * Copyright (C) 2010-2014 Oracle Corporation + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include diff --git a/arch/sparc/kernel/fbt_blacklist.h b/arch/sparc/kernel/fbt_blacklist.h index fc7b2d42f4f0..ee9e341b6768 100644 --- a/arch/sparc/kernel/fbt_blacklist.h +++ b/arch/sparc/kernel/fbt_blacklist.h @@ -3,7 +3,7 @@ BL_SENTRY(typeof(__atomic_notifier_call_chain), __atomic_notifier_call_chain) BL_SENTRY(typeof(atomic_notifier_call_chain), atomic_notifier_call_chain) BL_SENTRY(typeof(__raw_notifier_call_chain), __raw_notifier_call_chain) BL_SENTRY(typeof(raw_notifier_call_chain), raw_notifier_call_chain) -BL_SENTRY(void *, notify_die) +BL_DENTRY(void *, notify_die) BL_SENTRY(void *, rcu_nmi_exit) BL_SENTRY(void *, rcu_nmi_enter) BL_SENTRY(typeof(ktime_get_raw_fast_ns), ktime_get_raw_fast_ns) diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c index 54d55e6e9adc..c22ae6d3773b 100644 --- a/arch/sparc/kernel/module.c +++ b/arch/sparc/kernel/module.c @@ -226,20 +226,10 @@ int module_finalize(const Elf_Ehdr *hdr, } # ifdef CONFIG_DTRACE - if (DTRACE_SDT_MAX(me) + DTRACE_FBT_MAX(me) > 0) - me->pdata = module_alloc(DTRACE_PD_MAXSIZE(me)); - else - me->pdata = NULL; + me->pdata = NULL; # endif return 0; } -# ifdef CONFIG_DTRACE -void module_arch_cleanup(struct module *me) -{ - if (me->pdata) - module_memfree(me->pdata); -} -#endif #endif /* CONFIG_SPARC64 */ diff --git a/arch/sparc/kernel/ttable_64.S b/arch/sparc/kernel/ttable_64.S index c6dfdaa29e20..73345bfd6c8a 100644 --- a/arch/sparc/kernel/ttable_64.S +++ b/arch/sparc/kernel/ttable_64.S @@ -165,7 +165,7 @@ tl0_resv169: BTRAP(0x169) BTRAP(0x16a) BTRAP(0x16b) BTRAP(0x16c) tl0_linux64: LINUX_64BIT_SYSCALL_TRAP tl0_gsctx: TRAP(sparc64_get_context) TRAP(sparc64_set_context) tl0_resv170: KPROBES_TRAP(0x170) KPROBES_TRAP(0x171) KGDB_TRAP(0x172) -tl0_resv173: BTRAP(0x173) BTRAP(0x174) BTRAP(0x175) BTRAP(0x176) BTRAP(0x177) +tl0_resv173: UPROBES_TRAP(0x173) UPROBES_TRAP(0x174) DTRACE_FBT_TRAP(0x175) BTRAP(0x176) BTRAP(0x177) tl0_resv178: BTRAP(0x178) BTRAP(0x179) BTRAP(0x17a) BTRAP(0x17b) BTRAP(0x17c) tl0_resv17d: BTRAP(0x17d) BTRAP(0x17e) BTRAP(0x17f) #define BTRAPS(x) BTRAP(x) BTRAP(x+1) BTRAP(x+2) BTRAP(x+3) BTRAP(x+4) BTRAP(x+5) BTRAP(x+6) BTRAP(x+7) diff --git a/arch/x86/include/asm/dtrace_arch.h b/arch/x86/include/asm/dtrace_arch.h index 4c2bd305a99c..8dfa953382f3 100644 --- a/arch/x86/include/asm/dtrace_arch.h +++ b/arch/x86/include/asm/dtrace_arch.h @@ -1,17 +1,10 @@ -/* Copyright (C) 2013-2016 Oracle, Inc. */ +/* Copyright (c) 2013, 2017, Oracle and/or its affiliates. All rights reserved. */ #ifndef _X86_DTRACE_ARCH_H #define _X86_DTRACE_ARCH_H typedef uint8_t asm_instr_t; -/* - * No additional memory needs to be allocated for the PDATA section on x86. - */ -#define DTRACE_PD_MAXSIZE(mp) (0) - -#define DTRACE_PD_MAXSIZE_KERNEL (0) - #define ASM_CALL_SIZE 5 #endif /* _X86_DTRACE_ARCH_H */ diff --git a/arch/x86/kernel/dtrace_fbt.c b/arch/x86/kernel/dtrace_fbt.c index 6786a7dbcac0..549af864ec25 100644 --- a/arch/x86/kernel/dtrace_fbt.c +++ b/arch/x86/kernel/dtrace_fbt.c @@ -2,7 +2,7 @@ * FILE: dtrace_fbt.c * DESCRIPTION: Dynamic Tracing: FBT registration code (arch-specific) * - * Copyright (C) 2010-2014 Oracle Corporation + * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. */ #include @@ -39,7 +39,8 @@ dtrace_fbt_populate_bl(void) #undef BL_DENTRY } -void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe) +void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe, struct module *mp, + void *arg) { loff_t pos; struct kallsym_iter sym; @@ -72,12 +73,19 @@ void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe) continue; /* - * Only core kernel symbols are of interest here. + * Handle only symbols that belong to the module we have been + * asked for. */ - if (!core_kernel_text(sym.value)) + if (mp == dtrace_kmod && !core_kernel_text(sym.value)) continue; - /* TODO: Jumplabel blacklist ? */ + /* + * Ensure we have not been given .init symbol from kallsyms + * interface. This could lead to memory corruption once DTrace + * tries to enable probe in already freed memory. + */ + if (mp != dtrace_kmod && !within_module_core(sym.value, mp)) + continue; /* * See if the symbol is on the FBT's blacklist. Since both @@ -134,9 +142,9 @@ void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe) case 0: /* start of function */ if (*addr == FBT_PUSHL_EBP) { fbt_add_probe( - dtrace_kmod, sym.name, + mp, sym.name, FBT_ENTRY, *addr, addr, 0, - NULL); + NULL, arg); state = 1; } else if (insc > 2) state = 2; @@ -147,9 +155,9 @@ void dtrace_fbt_init(fbt_add_probe_fn fbt_add_probe) off = addr - (asm_instr_t *)sym.value; fbtp = fbt_add_probe( - dtrace_kmod, sym.name, + mp, sym.name, FBT_RETURN, *addr, addr, off, - fbtp); + fbtp, arg); } break; } diff --git a/include/linux/dtrace_fbt.h b/include/linux/dtrace_fbt.h index 75c1e8745f2c..da18a4ed06cf 100644 --- a/include/linux/dtrace_fbt.h +++ b/include/linux/dtrace_fbt.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015, 2017, Oracle and/or its affiliates. All rights reserved. */ +/* Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. */ #ifndef _LINUX_DTRACE_FBT_H #define _LINUX_DTRACE_FBT_H @@ -18,7 +18,9 @@ extern unsigned long dtrace_fbt_nfuncs __attribute__((weak)); * - probe type (FBT_ENTRY or FBT_RETURN) * - probe subtype (arch-specific) * - address (location of the probe) + * - offset from the function start * - return value from previous callback invocation + * - cookie passed to dtrace_fbt_init * Returns: * - generic pointer (only to be used to pass back in) */ @@ -26,8 +28,8 @@ extern unsigned long dtrace_fbt_nfuncs __attribute__((weak)); #define FBT_RETURN 1 typedef void *(*fbt_add_probe_fn)(struct module *, char *, int, int, - asm_instr_t *, uintptr_t, void *); -extern void dtrace_fbt_init(fbt_add_probe_fn); + asm_instr_t *, uintptr_t, void *, void *); +extern void dtrace_fbt_init(fbt_add_probe_fn, struct module *, void *); /* * Dynamic blacklist routines. diff --git a/include/linux/dtrace_os.h b/include/linux/dtrace_os.h index e3200c2d6c84..dffe8848a6f9 100644 --- a/include/linux/dtrace_os.h +++ b/include/linux/dtrace_os.h @@ -26,6 +26,9 @@ extern struct module *dtrace_kmod; extern void dtrace_os_init(void); extern void dtrace_os_exit(void); +extern void *dtrace_alloc_text(struct module *, unsigned long); +extern void dtrace_free_text(void *); + extern void dtrace_enable(void); extern void dtrace_disable(void); diff --git a/kernel/dtrace/dtrace_os.c b/kernel/dtrace/dtrace_os.c index 95afcd4151ab..547c6ad8c1b8 100644 --- a/kernel/dtrace/dtrace_os.c +++ b/kernel/dtrace/dtrace_os.c @@ -54,19 +54,9 @@ void dtrace_os_init(void) * only data we're interested in is the name, the SDT probe points data * (to be filled in by dtrace_sdt_register()), and the probe data. * DTrace uses an architecture-specific structure (hidden from us here) - * to hold some data, and since we do not know the layout or the size, - * we ensure that we allocate enough memory to accomodate the largest - * of those structures. On some architectures there may not be a need - * for additional data. In that case, pdata will be NULL. - * - * So, the memory we allocate will hold: - * - the dtrace_kmod module structure - * - a block of memory (aligned at a structure boundary) to be - * used for pdata and other related data [optional] - * The memory is allocated from the modules space. + * to hold some data. */ - module_size = ALIGN(sizeof(struct module), 8) + - DTRACE_PD_MAXSIZE_KERNEL; + module_size = ALIGN(sizeof(struct module), 8); dtrace_kmod = module_alloc(module_size); if (dtrace_kmod == NULL) { pr_warning("%s: cannot allocate kernel pseudo-module\n", @@ -77,17 +67,21 @@ void dtrace_os_init(void) memset(dtrace_kmod, 0, module_size); strlcpy(dtrace_kmod->name, "vmlinux", MODULE_NAME_LEN); - if (DTRACE_PD_MAXSIZE_KERNEL > 0) - dtrace_kmod->pdata = (char *)dtrace_kmod + - ALIGN(sizeof(struct module), 8); - else - dtrace_kmod->pdata = NULL; + /* + * Some sizing info is required for kernel module. We are going to use + * modules VA range for trampoline anyway so lets pretend a kernel has + * no init section and VA range (0, MODULES_VADDR) is occupied by kernel itself + */ + dtrace_kmod->module_core = NULL; + dtrace_kmod->core_size = MODULES_VADDR; - dtrace_kmod->core_size = DTRACE_PD_MAXSIZE_KERNEL; dtrace_kmod->num_ftrace_callsites = dtrace_fbt_nfuncs; dtrace_kmod->state = MODULE_STATE_LIVE; atomic_inc(&dtrace_kmod->refcnt); + INIT_LIST_HEAD(&dtrace_kmod->source_list); + INIT_LIST_HEAD(&dtrace_kmod->target_list); + psinfo_cachep = kmem_cache_create("psinfo_cache", sizeof(dtrace_psinfo_t), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, @@ -103,6 +97,45 @@ void dtrace_os_init(void) } EXPORT_SYMBOL(dtrace_os_init); +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define TRAMP_RANGE 0x80000000 + +void *dtrace_alloc_text(struct module *mp, unsigned long size) +{ + unsigned long mp_start, mp_end; + unsigned long va_start, va_end; + void *trampoline; + + /* module range */ + mp_start = (unsigned long) mp->module_core; + mp_end = mp_start + mp->core_size; + + if (mp->module_init != NULL) { + mp_start = MIN(mp_start, (unsigned long)mp->module_init); + mp_end = MAX(mp_end, (unsigned long)mp->module_init + + mp->init_size); + } + + /* get trampoline range */ + va_end = MIN(mp_start + TRAMP_RANGE, MODULES_END); + va_start = (mp_end < TRAMP_RANGE) ? 0 : mp_end - TRAMP_RANGE; + va_start = MAX(va_start, MODULES_VADDR); + + trampoline = __vmalloc_node_range(size, 1, va_start, va_end, + GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, + __builtin_return_address(0)); + + return trampoline; +} +EXPORT_SYMBOL(dtrace_alloc_text); + +void dtrace_free_text(void *ptr) +{ + return vfree(ptr); +} +EXPORT_SYMBOL(dtrace_free_text); + /*---------------------------------------------------------------------------*\ (* TASK PSINFO SUPPORT *) \*---------------------------------------------------------------------------*/