Support for page fault and general protection fault detection for DTrace, to

ensure that DTrace memory access faults are non-fatal.

Signed-off-by: Kris Van Hees <kris.van.hees@oracle.com>

diff --git a/dtrace/dtrace.h b/dtrace/dtrace.h
index 76a6e8da68a0c436572726c31238439701fa224e..9e7f05124b4b07943e5215397dd0d778ea934f4e 100644 (file)
--- a/dtrace/dtrace.h
+++ b/dtrace/dtrace.h
@@ -2224,53 +2224,7 @@ typedef struct dtrace_mprovider {
 
 typedef unsigned long  dtrace_icookie_t;
 
-#define DTRACE_CPUFLAG_ISSET(flag) \
-       (cpu_core[smp_processor_id()].cpuc_dtrace_flags & (flag))
-
-#define DTRACE_CPUFLAG_SET(flag) \
-       (cpu_core[smp_processor_id()].cpuc_dtrace_flags |= (flag))
-
-#define DTRACE_CPUFLAG_CLEAR(flag) \
-       (cpu_core[smp_processor_id()].cpuc_dtrace_flags &= ~(flag))
-
-#define CPU_DTRACE_NOFAULT     0x0001
-#define CPU_DTRACE_DROP                0x0002
-#define CPU_DTRACE_BADADDR     0x0004
-#define CPU_DTRACE_BADALIGN    0x0008
-#define CPU_DTRACE_DIVZERO     0x0010
-#define CPU_DTRACE_ILLOP       0x0020
-#define CPU_DTRACE_NOSCRATCH   0x0040
-#define CPU_DTRACE_KPRIV       0x0080
-#define CPU_DTRACE_UPRIV       0x0100
-#define CPU_DTRACE_TUPOFLOW    0x0200
-#define CPU_DTRACE_ENTRY       0x0800
-#define CPU_DTRACE_BADSTACK    0x1000
-
-#define CPU_DTRACE_FAULT       (CPU_DTRACE_BADADDR | CPU_DTRACE_BADALIGN | \
-                                CPU_DTRACE_DIVZERO | CPU_DTRACE_ILLOP | \
-                                CPU_DTRACE_NOSCRATCH | CPU_DTRACE_KPRIV | \
-                                CPU_DTRACE_UPRIV | CPU_DTRACE_TUPOFLOW | \
-                                CPU_DTRACE_BADSTACK)
-#define CPU_DTRACE_ERROR       (CPU_DTRACE_FAULT | CPU_DTRACE_DROP)
-
-#define CPUC_SIZE      (sizeof (uint16_t) + sizeof(uint8_t) + \
-                        sizeof(uintptr_t) + sizeof(struct mutex))
-#define CPUC_PADSIZE   (192 - CPUC_SIZE)
-
-typedef struct cpu_core {
-       uint16_t cpuc_dtrace_flags;
-       uint8_t cpuc_dcpc_intr_state;
-       uint8_t cpuc_pad[CPUC_PADSIZE];
-       uintptr_t cpuc_dtrace_illval;
-       struct mutex cpuc_pid_lock;
-
-       uintptr_t cpu_dtrace_caller;
-       ktime_t cpu_dtrace_chillmark;
-       ktime_t cpu_dtrace_chilled;
-} cpu_core_t;
-
-extern cpu_core_t              cpu_core[];
-extern struct mutex            cpu_lock;
+extern struct mutex    cpu_lock;
 
 extern void dtrace_sync(void);
 extern void dtrace_toxic_ranges(void (*)(uintptr_t, uintptr_t));

diff --git a/dtrace/dtrace_dev.c b/dtrace/dtrace_dev.c
index 62d17ab061eef631633cc07cc774b14d00c661b8..ab1aafa100fe9292bd08441f29bb168db52b207e 100644 (file)
--- a/dtrace/dtrace_dev.c
+++ b/dtrace/dtrace_dev.c
@@ -26,6 +26,7 @@
  */
 
 #include <linux/delay.h>
+#include <linux/dtrace_cpu.h>
 #include <linux/dtrace_ioctl.h>
 #include <linux/fs.h>
 #include <linux/jiffies.h>
@@ -1152,13 +1153,13 @@ int dtrace_istoxic(uintptr_t kaddr, size_t size)
 
                if (kaddr - taddr < tsize) {
                        DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
-                       cpu_core[smp_processor_id()].cpuc_dtrace_illval = kaddr;
+                       this_cpu_core->cpuc_dtrace_illval = kaddr;
                        return 1;
                }
 
                if (taddr - kaddr < size) {
                        DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
-                       cpu_core[smp_processor_id()].cpuc_dtrace_illval = taddr;
+                       this_cpu_core->cpuc_dtrace_illval = kaddr;
                        return 1;
                }
        }

diff --git a/dtrace/dtrace_dif.c b/dtrace/dtrace_dif.c
index 5fe142d54c4ad3f545a50ec2df8d632f3189ab04..3a59d16f8dd53feb7faca464ad62a9805e6ff71d 100644 (file)
--- a/dtrace/dtrace_dif.c
+++ b/dtrace/dtrace_dif.c
@@ -25,6 +25,7 @@
  * Use is subject to license terms.
  */
 
+#include <linux/dtrace_cpu.h>
 #include <linux/hardirq.h>
 #include <linux/in6.h>
 #include <linux/inet.h>
@@ -909,9 +910,8 @@ void dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
                size_t                  size = bits / NBBY;             \
                uint##bits##_t          rval;                           \
                int                     i;                              \
-               int                     cpu = smp_processor_id();       \
                volatile uint16_t       *flags = (volatile uint16_t *)  \
-                           &cpu_core[cpu].cpuc_dtrace_flags;           \
+                           &this_cpu_core->cpuc_dtrace_flags;          \
                                                                        \
                /*                                                      \
                 * Deviation from the OpenSolaris code...  Protect      \
@@ -920,7 +920,7 @@ void dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
                 */                                                     \
                if (addr == 0) {                                        \
                        *flags |= CPU_DTRACE_BADADDR;                   \
-                       cpu_core[cpu].cpuc_dtrace_illval = addr;        \
+                       this_cpu_core->cpuc_dtrace_illval = addr;       \
                        return 0;                                       \
                }                                                       \
                                                                        \
@@ -932,12 +932,12 @@ void dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
                                                                        \
                        if (addr + size <= dtrace_toxrange[i].dtt_base) \
                                continue;                               \
-                                                               \
+                                                                       \
                        /*                                              \
                         * This address falls within a toxic region.    \
                         */                                             \
                        *flags |= CPU_DTRACE_BADADDR;                   \
-                       cpu_core[cpu].cpuc_dtrace_illval = addr;        \
+                       this_cpu_core->cpuc_dtrace_illval = addr;       \
                        return 0;                                       \
                }                                                       \
                                                                        \
@@ -1089,8 +1089,7 @@ static int
 dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
     dtrace_vstate_t *vstate)
 {
-       int                     cpu = smp_processor_id();
-       volatile uintptr_t      *illval = &cpu_core[cpu].cpuc_dtrace_illval;
+       volatile uintptr_t      *illval = &this_cpu_core->cpuc_dtrace_illval;
 
        /*
         * If we hold the privilege to read from kernel memory, then
@@ -1243,8 +1242,7 @@ static int dtrace_bcmp(const void *s1, const void *s2, size_t len)
 {
        volatile uint16_t       *flags;
 
-       flags = (volatile uint16_t *)&cpu_core[
-                                       smp_processor_id()].cpuc_dtrace_flags;
+       flags = (volatile uint16_t *)&this_cpu_core->cpuc_dtrace_flags;
 
        if (s1 == s2)
                return 0;
@@ -2173,12 +2171,8 @@ static void dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
                            dtrace_key_t *tupregs, int nargs,
                            dtrace_mstate_t *mstate, dtrace_state_t *state)
 {
-       volatile uint16_t       *flags = &cpu_core[
-                                               smp_processor_id()
-                                         ].cpuc_dtrace_flags;
-       volatile uintptr_t      *illval = &cpu_core[
-                                               smp_processor_id()
-                                          ].cpuc_dtrace_illval;
+       volatile uint16_t       *flags = &this_cpu_core->cpuc_dtrace_flags;
+       volatile uintptr_t      *illval = &this_cpu_core->cpuc_dtrace_illval;
        dtrace_vstate_t         *vstate = &state->dts_vstate;
 
        union {
@@ -2476,6 +2470,9 @@ static void dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
                                rval = 1;
                                break;
                        }
+
+                       if (p == p->real_parent)
+                               break;
                }
 
                DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
@@ -3667,13 +3664,12 @@ uint64_t dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
        const char              *strtab = difo->dtdo_strtab;
        const uint64_t          *inttab = difo->dtdo_inttab;
 
-       int                     cpu = smp_processor_id();
        uint64_t                rval = 0;
        dtrace_statvar_t        *svar;
        dtrace_dstate_t         *dstate = &vstate->dtvs_dynvars;
        dtrace_difv_t           *v;
-       volatile uint16_t       *flags = &cpu_core[cpu].cpuc_dtrace_flags;
-       volatile uintptr_t      *illval = &cpu_core[cpu].cpuc_dtrace_illval;
+       volatile uint16_t       *flags = &this_cpu_core->cpuc_dtrace_flags;
+       volatile uintptr_t      *illval = &this_cpu_core->cpuc_dtrace_illval;
 
        dtrace_key_t            tupregs[DIF_DTR_NREGS + 2];
                                                /* +2 for thread and id */

diff --git a/dtrace/dtrace_helper.c b/dtrace/dtrace_helper.c
index 9634c45a5a0fadae8772c36cd77750ad8761eeca..589fd4add1d10c084a88b4a5a6ad71a25e062041 100644 (file)
--- a/dtrace/dtrace_helper.c
+++ b/dtrace/dtrace_helper.c
@@ -25,6 +25,8 @@
  * Use is subject to license terms.
  */
 
+#include <linux/dtrace_cpu.h>
+
 #include "dtrace.h"
 
 static uint32_t        dtrace_helptrace_next = 0;
@@ -44,9 +46,7 @@ static void dtrace_helper_trace(dtrace_helper_action_t *helper,
 {
        uint32_t                size, next, nnext, i;
        dtrace_helptrace_t      *ent;
-       uint16_t                flags = cpu_core[
-                                           smp_processor_id()
-                                       ].cpuc_dtrace_flags;
+       uint16_t                flags = this_cpu_core->cpuc_dtrace_flags;
 
        if (!dtrace_helptrace_enabled)
                return;
@@ -87,7 +87,7 @@ static void dtrace_helper_trace(dtrace_helper_action_t *helper,
                                ?  mstate->dtms_fltoffs
                                : -1;
        ent->dtht_fault = DTRACE_FLAGS2FLT(flags);
-       ent->dtht_illval = cpu_core[smp_processor_id()].cpuc_dtrace_illval;
+       ent->dtht_illval = this_cpu_core->cpuc_dtrace_illval;
 
        for (i = 0; i < vstate->dtvs_nlocals; i++) {
                dtrace_statvar_t        *svar;
@@ -105,9 +105,7 @@ static void dtrace_helper_trace(dtrace_helper_action_t *helper,
 uint64_t dtrace_helper(int which, dtrace_mstate_t *mstate,
                       dtrace_state_t *state, uint64_t arg0, uint64_t arg1)
 {
-       uint16_t                *flags = &cpu_core[
-                                               smp_processor_id()
-                                         ].cpuc_dtrace_flags;
+       uint16_t                *flags = &this_cpu_core->cpuc_dtrace_flags;
        uint64_t                sarg0 = mstate->dtms_arg[0];
        uint64_t                sarg1 = mstate->dtms_arg[1];
        uint64_t                rval = 0;

diff --git a/dtrace/dtrace_isa.c b/dtrace/dtrace_isa.c
index 3628ad2098bb268a685d3ddf757358fe75f8a475..1020009110c3a53c4395ac27a04f08b5ce25e27b 100644 (file)
--- a/dtrace/dtrace_isa.c
+++ b/dtrace/dtrace_isa.c
@@ -25,6 +25,7 @@
  * Use is subject to license terms.
  */
 
+#include <linux/dtrace_cpu.h>
 #include <linux/hardirq.h>
 #include <linux/smp.h>
 #include <asm/stacktrace.h>
@@ -35,8 +36,6 @@
 uintptr_t _userlimit = 0x00007fffffffffffLL;
 uintptr_t kernelbase = 0xffff880000000000LL;
 
-cpu_core_t     cpu_core[NR_CPUS];
-EXPORT_SYMBOL(cpu_core);
 EXPORT_SYMBOL(dtrace_getfp);
 
 DEFINE_MUTEX(cpu_lock);
@@ -53,14 +52,14 @@ static int dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size)
 #else
        if (kaddr < kernelbase || kaddr + size < kaddr) {
                DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
-               cpu_core[smp_processor_id()].cpuc_dtrace_illval = kaddr;
+               this_cpu_core->cpuc_dtrace_illval = kaddr;
                return 0;
        }
 #endif
 
        if (uaddr + size >= kernelbase || uaddr + size < uaddr) {
                DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
-               cpu_core[smp_processor_id()].cpuc_dtrace_illval = uaddr;
+               this_cpu_core->cpuc_dtrace_illval = uaddr;
                return 0;
        }
 
@@ -102,8 +101,7 @@ void dtrace_copyoutstr(uintptr_t uaddr, uintptr_t kaddr, size_t size,
                                                                              \
                if ((uintptr_t)uaddr > _userlimit) {                          \
                        DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);               \
-                       cpu_core[smp_processor_id()].cpuc_dtrace_illval =     \
-                                                       (uintptr_t)uaddr;     \
+                       this_cpu_core->cpuc_dtrace_illval = (uintptr_t)uaddr; \
                        return 0;                                             \
                }                                                             \
                                                                              \

diff --git a/dtrace/dtrace_priv.c b/dtrace/dtrace_priv.c
index 88cf4b26f51d22857f33d18d0847df6487f03be7..a19b08e2fa9520cee5bb78fcf75d1eff1ab8db64 100644 (file)
--- a/dtrace/dtrace_priv.c
+++ b/dtrace/dtrace_priv.c
@@ -25,6 +25,8 @@
  * Use is subject to license terms.
  */
 
+#include <linux/dtrace_cpu.h>
+
 #include "dtrace.h"
 
 /*

diff --git a/dtrace/dtrace_probe.c b/dtrace/dtrace_probe.c
index 0cd23e9d83c030ccd20449d16533ce8922597f45..053316c7582428f7c96408d11bbe86f6a3df6729 100644 (file)
--- a/dtrace/dtrace_probe.c
+++ b/dtrace/dtrace_probe.c
@@ -25,6 +25,7 @@
  * Use is subject to license terms.
  */
 
+#include <linux/dtrace_cpu.h>
 #include <linux/hardirq.h>
 #include <linux/idr.h>
 #include <linux/module.h>
@@ -375,7 +376,7 @@ static void dtrace_action_chill(dtrace_mstate_t *mstate, ktime_t val)
 {
        ktime_t                 now;
        volatile uint16_t       *flags;
-       cpu_core_t              *cpu = &cpu_core[smp_processor_id()];
+       cpu_core_t              *cpu = this_cpu_core;
 
        if (dtrace_destructive_disallow)
                return;
@@ -428,9 +429,7 @@ static void dtrace_action_ustack(dtrace_mstate_t *mstate,
        char            *str = (char *)&pcs[nframes];
        int             size, offs = 0, i, j;
        uintptr_t       old = mstate->dtms_scratch_ptr, saved;
-       uint16_t        *flags = &cpu_core[
-                                       smp_processor_id()
-                                 ].cpuc_dtrace_flags;
+       uint16_t        *flags = &this_cpu_core->cpuc_dtrace_flags;
        char            *sym;
 
        /*
@@ -599,7 +598,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
        mstate.dtms_arg[3] = arg3;
        mstate.dtms_arg[4] = arg4;
 
-       flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags;
+       flags = (volatile uint16_t *)&this_cpu_core->cpuc_dtrace_flags;
 
        for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
                dtrace_predicate_t      *pred = ecb->dte_predicate;
@@ -1139,7 +1138,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
                                        ?  mstate.dtms_fltoffs
                                        : -1,
                                DTRACE_FLAGS2FLT(*flags),
-                               cpu_core[cpuid].cpuc_dtrace_illval);
+                               this_cpu_core->cpuc_dtrace_illval);
 
                        continue;
                }

diff --git a/dtrace/dtrace_probe_ctx.c b/dtrace/dtrace_probe_ctx.c
index 2650a259e4b5584d4608f71d873e98f1d30d7d28..e25531107a12d6b960d2e87a7bfa9194540cd3de 100644 (file)
--- a/dtrace/dtrace_probe_ctx.c
+++ b/dtrace/dtrace_probe_ctx.c
@@ -25,6 +25,8 @@
  * Use is subject to license terms.
  */
 
+#include <linux/dtrace_cpu.h>
+
 #include "dtrace.h"
 
 void dtrace_panic(const char *fmt, ...)

diff --git a/dtrace/dtrace_spec.c b/dtrace/dtrace_spec.c
index d02224d3361e0998622458e619a611c8c885a8be..6748c8e06c5030e11f3099643be179a013bb4c64 100644 (file)
--- a/dtrace/dtrace_spec.c
+++ b/dtrace/dtrace_spec.c
@@ -25,6 +25,7 @@
  * Use is subject to license terms.
  */
 
+#include <linux/dtrace_cpu.h>
 #include <linux/smp.h>
 #include <asm/cmpxchg.h>
 
@@ -95,7 +96,7 @@ void dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,
                return;
 
        if (which > state->dts_nspeculations) {
-               cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
+               per_cpu_core(cpu)->cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
                return;
        }
 
@@ -238,7 +239,7 @@ void dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,
                return;
 
        if (which > state->dts_nspeculations) {
-               cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
+               per_cpu_core(cpu)->cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
                return;
        }
 
@@ -379,7 +380,7 @@ void dtrace_speculation_clean(dtrace_state_t *state)
  * atomically transitioned into the ACTIVEMANY state.
  */
 dtrace_buffer_t *dtrace_speculation_buffer(dtrace_state_t *state,
-                                          processorid_t cpuid,
+                                          processorid_t cpu,
                                           dtrace_specid_t which)
 {
        dtrace_speculation_t            *spec;
@@ -390,12 +391,12 @@ dtrace_buffer_t *dtrace_speculation_buffer(dtrace_state_t *state,
                return NULL;
 
        if (which > state->dts_nspeculations) {
-               cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
+               per_cpu_core(cpu)->cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
                return NULL;
        }
 
        spec = &state->dts_speculations[which - 1];
-       buf = &spec->dtsp_buffer[cpuid];
+       buf = &spec->dtsp_buffer[cpu];
 
        do {
                curr = spec->dtsp_state;

diff --git a/dtrace/dtrace_util.c b/dtrace/dtrace_util.c
index 9b4abf7c92d340b0f62bd190049993cede472e5c..a4f086c359f466d2c299582d776c1067f11f42da 100644 (file)
--- a/dtrace/dtrace_util.c
+++ b/dtrace/dtrace_util.c
@@ -25,6 +25,7 @@
  * Use is subject to license terms.
  */
 
+#include <linux/dtrace_cpu.h>
 #include <linux/vmalloc.h>
 
 #include "dtrace.h"
@@ -71,8 +72,7 @@ int dtrace_strncmp(char *s1, char *s2, size_t limit)
        if (s1 == s2 || limit == 0)
                return 0;
 
-       flags = (volatile uint16_t *)
-               &cpu_core[smp_processor_id()].cpuc_dtrace_flags;
+       flags = (volatile uint16_t *)&this_cpu_core->cpuc_dtrace_flags;
 
        do {
                if (s1 == NULL)