dtrace: change the DTrace startup handling (at boot time) for SDT

The DTrace OS level handling was initialized at DTrace module load, which
caused major indigestion on the side of the scheduler when SDT probe points at
crucial locations in the scheduler were being patched by one CPU while another
was trying to get some real work done.  Even a nice stop_machine() based
approach turned out not to be possible, because that *cough* depends on the
scheduler also.

Instead, the DTrace OS support is initialized from the Linux boot sequence,
before SMP is enabled, which removes the complications altogether (and it is a
lot cleaner and faster).  We also call CPU-specific initialization for DTrace
during the boot sequence, albeit *after* the CPUs have been identified for SMP,
to ensure that we get accurate information.

Renamed sdt_register.c to be dtrace_sdt.c (for consistency).  And implemented
a better patching of SDT probe points.

Added a 'nosdt' kernel command line option to allow system wide diabling of
SDT probe points (at the kernel level).  This can be used when the patching of
SDT probe points somehow causes a problem.

Signed-off-by: Kris Van Hees <kris.van.hees@oracle.com>

diff --git a/include/linux/sdt.h b/include/linux/sdt.h
index 4542171c6b7f3b86a1d9efab7ecd224869572da8..0399602191f329cc93490e2c4239ec614704096b 100644 (file)
--- a/include/linux/sdt.h
+++ b/include/linux/sdt.h
@@ -125,9 +125,6 @@ typedef uint8_t     sdt_instr_t;
 extern unsigned long dtrace_sdt_nprobes __attribute__((weak));
 extern void *dtrace_sdt_probes __attribute__((weak));
 
-extern void sdt_probe_enable(sdt_instr_t *);
-extern void sdt_probe_disable(sdt_instr_t *);
-
 typedef struct dtrace_sdt_probeinfo {
        unsigned long addr;
        unsigned long name_len;
@@ -135,7 +132,7 @@ typedef struct dtrace_sdt_probeinfo {
        char name[0];
 } __aligned(sizeof(unsigned long)) dtrace_sdt_probeinfo_t;
 
-void dtrace_register_builtins(void);
+void dtrace_sdt_register(struct module *);
 
 #endif /* __KERNEL__ */
 

diff --git a/init/main.c b/init/main.c
index bdb67ba13600497d4ad65bf5a449244eb01cd0ae..5bf30895da18e6b76c4b010c739a30f242f35438 100644 (file)
--- a/init/main.c
+++ b/init/main.c
@@ -82,6 +82,8 @@
 #include <linux/proc_ns.h>
 #include <linux/io.h>
 #include <linux/sdt.h>
+#include <linux/dtrace_os.h>
+#include <linux/dtrace_cpu.h>
 
 #include <asm/io.h>
 #include <asm/bugs.h>
@@ -675,7 +677,7 @@ asmlinkage __visible void __init start_kernel(void)
        ftrace_init();
 
 #ifdef CONFIG_DTRACE
-       dtrace_register_builtins();                                             
+       dtrace_os_init();
 #endif                                                                        
 
        /* Do the rest non-__init'ed, we're now alive */
@@ -997,6 +999,10 @@ static noinline void __init kernel_init_freeable(void)
 
        smp_prepare_cpus(setup_max_cpus);
 
+#ifdef CONFIG_DTRACE
+       dtrace_cpu_init();
+#endif
+
        do_pre_smp_initcalls();
        lockup_detector_init();
 

diff --git a/kernel/dtrace/Makefile b/kernel/dtrace/Makefile
index d284c7eab6a9b86d7306101963621b973f8ccb5f..0e431211d88e0d08f95a1dd90862a5d0cd0bddb6 100644 (file)
--- a/kernel/dtrace/Makefile
+++ b/kernel/dtrace/Makefile
@@ -6,6 +6,6 @@ GCOV_PROFILE := y
 
 ifdef CONFIG_DT_CORE
 obj-y                          += dtrace_os.o dtrace_cpu.o \
-                                  dtrace_stubs_x86_64.o sdt_register.o
+                                  dtrace_stubs_x86_64.o dtrace_sdt.o
 obj-m                          += dtrace_ctf.o
 endif

diff --git a/kernel/dtrace/sdt_register.c b/kernel/dtrace/dtrace_sdt.c
similarity index 56%
rename from kernel/dtrace/sdt_register.c
rename to kernel/dtrace/dtrace_sdt.c
index 6298c6b4d45b125f6017aaa654d5ee306d78f272..a99d5c318e920081b50ad213b20b51d23ab87ada 100644 (file)
--- a/kernel/dtrace/sdt_register.c
+++ b/kernel/dtrace/dtrace_sdt.c
@@ -12,6 +12,7 @@
 #include <asm-generic/bitsperlong.h>
 #include <asm-generic/sections.h>
 #include <asm/alternative.h>
+#include <asm/nmi.h>
 #include <asm/nops.h>
 
 #define        SDT_TRAP_INSTR  0xf0
@@ -19,20 +20,33 @@
 
 const char             *sdt_prefix = "__dtrace_probe_";
 
-void sdt_probe_enable(sdt_instr_t *addr)
+/* This code is based on apply_alternatives and text_poke_early.  It needs to
+ * run before SMP is initialized in order to avoid SMP problems with patching
+ * code that might be accessed on another CPU.
+ */
+static void __init_or_module text_poke_batch(struct text_poke_param *reqs,
+                                            int cnt)
 {
-       text_poke(addr, ((unsigned char []){SDT_TRAP_INSTR}), 1);
-}
-EXPORT_SYMBOL(sdt_probe_enable);
+       int                     i;
+       unsigned long           flags;
+       struct text_poke_param  *tpp;
 
-void sdt_probe_disable(sdt_instr_t *addr)
-{
-       text_poke((void *)addr, ideal_nops[1], 1);
+       stop_nmi();
+       local_irq_save(flags);
+
+       for (i = 0; i < cnt; i++) {
+               tpp = &reqs[i];
+               memcpy(tpp->addr, tpp->opcode, tpp->len);
+       }
+
+       sync_core();
+       local_irq_restore(flags);
+       restart_nmi();
 }
-EXPORT_SYMBOL(sdt_probe_disable);
 
 static int sdt_probe_add(struct module *mp, char *name, char *func,
-                        uintptr_t addr, void *nops)
+                        uintptr_t addr, struct text_poke_param *tpp,
+                        void *nops)
 {
        sdt_probedesc_t *sdp;
        uint8_t *instr;
@@ -59,23 +73,24 @@ static int sdt_probe_add(struct module *mp, char *name, char *func,
        sdp->sdpd_next = mp->sdt_probes;
        mp->sdt_probes = sdp;
 
-       mutex_lock(&text_mutex);
-       text_poke(instr, nops, SDT_NOP_SIZE);
-       mutex_unlock(&text_mutex);
+       tpp->addr = instr;
+       tpp->opcode = nops;
+       tpp->len = SDT_NOP_SIZE;
 
        return 0;
 }
 
-void dtrace_register_builtins(void)
+void dtrace_sdt_register(struct module *mod)
 {
-       unsigned long           cnt;
+       int                     i, cnt;
        dtrace_sdt_probeinfo_t  *pi =
                                (dtrace_sdt_probeinfo_t *)&dtrace_sdt_probes;
        void                    *nextpi;
        uint8_t                 nops[SDT_NOP_SIZE];
+       struct text_poke_param  *reqs;
 
-       if (dtrace_kmod == NULL) {
-               pr_warning("%s: no kernel pseudo-module allocated\n",
+       if (mod == NULL) {
+               pr_warning("%s: no module provided - nothing registered\n",
                           __func__);
                return;
        }
@@ -94,17 +109,44 @@ void dtrace_register_builtins(void)
        add_nops(nops, 1);
        add_nops(nops + 1, SDT_NOP_SIZE - 1);
 
-       for (cnt = 0; cnt < dtrace_sdt_nprobes; cnt++) {
+       /*
+        * Set up a batch of text_poke requests that will handle replacing all
+        * calls at SDT probe locations with the NOP sequence.  Allocate the
+        * requests array, and then fill it in.
+        */
+       reqs = (struct text_poke_param *)
+                       vmalloc(dtrace_sdt_nprobes *
+                               sizeof(struct text_poke_param));
+       if (reqs == NULL) {
+               pr_warning("%s: failed to allocate text_poke_param array\n",
+                          __func__);
+               return;
+       }
+
+       for (i = cnt = 0; cnt < dtrace_sdt_nprobes; i++) {
                char    *func = pi->name + pi->name_len + 1;
 
-               if (sdt_probe_add(dtrace_kmod, pi->name, func, pi->addr, nops))
+               if (sdt_probe_add(dtrace_kmod, pi->name, func, pi->addr,
+                                 &reqs[cnt], nops))
                        pr_warning("%s: failed to add SDT probe %s\n",
                                   __func__, pi->name);
+               else
+                       cnt++;
 
                nextpi = (void *)pi + sizeof(dtrace_sdt_probeinfo_t)
                        + roundup(pi->name_len + 1 +
                                  pi->func_len + 1, BITS_PER_LONG / 8);
                pi = nextpi;
        }
+
+       text_poke_batch(reqs, cnt);
+}
+
+static int __init nosdt(char *str)
+{
+        dtrace_sdt_nprobes = 0;
+
+        return 0;
 }
-EXPORT_SYMBOL(dtrace_register_builtins);
+
+early_param("nosdt", nosdt);