From e1f7ace792c3a4df6cb31f722be7639b7ddbf534 Mon Sep 17 00:00:00 2001 From: Kris Van Hees Date: Wed, 14 Dec 2011 00:09:14 -0500 Subject: [PATCH] Support for page fault and general protection fault detection for DTrace, to ensure that DTrace memory access faults are non-fatal. Signed-off-by: Kris Van Hees --- dtrace/dtrace.h | 48 +-------------------------------------- dtrace/dtrace_dev.c | 5 ++-- dtrace/dtrace_dif.c | 32 ++++++++++++-------------- dtrace/dtrace_helper.c | 12 ++++------ dtrace/dtrace_isa.c | 10 ++++---- dtrace/dtrace_priv.c | 2 ++ dtrace/dtrace_probe.c | 11 ++++----- dtrace/dtrace_probe_ctx.c | 2 ++ dtrace/dtrace_spec.c | 11 +++++---- dtrace/dtrace_util.c | 4 ++-- 10 files changed, 44 insertions(+), 93 deletions(-) diff --git a/dtrace/dtrace.h b/dtrace/dtrace.h index 76a6e8da68a0..9e7f05124b4b 100644 --- a/dtrace/dtrace.h +++ b/dtrace/dtrace.h @@ -2224,53 +2224,7 @@ typedef struct dtrace_mprovider { typedef unsigned long dtrace_icookie_t; -#define DTRACE_CPUFLAG_ISSET(flag) \ - (cpu_core[smp_processor_id()].cpuc_dtrace_flags & (flag)) - -#define DTRACE_CPUFLAG_SET(flag) \ - (cpu_core[smp_processor_id()].cpuc_dtrace_flags |= (flag)) - -#define DTRACE_CPUFLAG_CLEAR(flag) \ - (cpu_core[smp_processor_id()].cpuc_dtrace_flags &= ~(flag)) - -#define CPU_DTRACE_NOFAULT 0x0001 -#define CPU_DTRACE_DROP 0x0002 -#define CPU_DTRACE_BADADDR 0x0004 -#define CPU_DTRACE_BADALIGN 0x0008 -#define CPU_DTRACE_DIVZERO 0x0010 -#define CPU_DTRACE_ILLOP 0x0020 -#define CPU_DTRACE_NOSCRATCH 0x0040 -#define CPU_DTRACE_KPRIV 0x0080 -#define CPU_DTRACE_UPRIV 0x0100 -#define CPU_DTRACE_TUPOFLOW 0x0200 -#define CPU_DTRACE_ENTRY 0x0800 -#define CPU_DTRACE_BADSTACK 0x1000 - -#define CPU_DTRACE_FAULT (CPU_DTRACE_BADADDR | CPU_DTRACE_BADALIGN | \ - CPU_DTRACE_DIVZERO | CPU_DTRACE_ILLOP | \ - CPU_DTRACE_NOSCRATCH | CPU_DTRACE_KPRIV | \ - CPU_DTRACE_UPRIV | CPU_DTRACE_TUPOFLOW | \ - CPU_DTRACE_BADSTACK) -#define CPU_DTRACE_ERROR (CPU_DTRACE_FAULT | CPU_DTRACE_DROP) - -#define CPUC_SIZE (sizeof (uint16_t) + sizeof(uint8_t) + \ - sizeof(uintptr_t) + sizeof(struct mutex)) -#define CPUC_PADSIZE (192 - CPUC_SIZE) - -typedef struct cpu_core { - uint16_t cpuc_dtrace_flags; - uint8_t cpuc_dcpc_intr_state; - uint8_t cpuc_pad[CPUC_PADSIZE]; - uintptr_t cpuc_dtrace_illval; - struct mutex cpuc_pid_lock; - - uintptr_t cpu_dtrace_caller; - ktime_t cpu_dtrace_chillmark; - ktime_t cpu_dtrace_chilled; -} cpu_core_t; - -extern cpu_core_t cpu_core[]; -extern struct mutex cpu_lock; +extern struct mutex cpu_lock; extern void dtrace_sync(void); extern void dtrace_toxic_ranges(void (*)(uintptr_t, uintptr_t)); diff --git a/dtrace/dtrace_dev.c b/dtrace/dtrace_dev.c index 62d17ab061ee..ab1aafa100fe 100644 --- a/dtrace/dtrace_dev.c +++ b/dtrace/dtrace_dev.c @@ -26,6 +26,7 @@ */ #include +#include #include #include #include @@ -1152,13 +1153,13 @@ int dtrace_istoxic(uintptr_t kaddr, size_t size) if (kaddr - taddr < tsize) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); - cpu_core[smp_processor_id()].cpuc_dtrace_illval = kaddr; + this_cpu_core->cpuc_dtrace_illval = kaddr; return 1; } if (taddr - kaddr < size) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); - cpu_core[smp_processor_id()].cpuc_dtrace_illval = taddr; + this_cpu_core->cpuc_dtrace_illval = kaddr; return 1; } } diff --git a/dtrace/dtrace_dif.c b/dtrace/dtrace_dif.c index 5fe142d54c4a..3a59d16f8dd5 100644 --- a/dtrace/dtrace_dif.c +++ b/dtrace/dtrace_dif.c @@ -25,6 +25,7 @@ * Use is subject to license terms. */ +#include #include #include #include @@ -909,9 +910,8 @@ void dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate) size_t size = bits / NBBY; \ uint##bits##_t rval; \ int i; \ - int cpu = smp_processor_id(); \ volatile uint16_t *flags = (volatile uint16_t *) \ - &cpu_core[cpu].cpuc_dtrace_flags; \ + &this_cpu_core->cpuc_dtrace_flags; \ \ /* \ * Deviation from the OpenSolaris code... Protect \ @@ -920,7 +920,7 @@ void dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate) */ \ if (addr == 0) { \ *flags |= CPU_DTRACE_BADADDR; \ - cpu_core[cpu].cpuc_dtrace_illval = addr; \ + this_cpu_core->cpuc_dtrace_illval = addr; \ return 0; \ } \ \ @@ -932,12 +932,12 @@ void dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate) \ if (addr + size <= dtrace_toxrange[i].dtt_base) \ continue; \ - \ + \ /* \ * This address falls within a toxic region. \ */ \ *flags |= CPU_DTRACE_BADADDR; \ - cpu_core[cpu].cpuc_dtrace_illval = addr; \ + this_cpu_core->cpuc_dtrace_illval = addr; \ return 0; \ } \ \ @@ -1089,8 +1089,7 @@ static int dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) { - int cpu = smp_processor_id(); - volatile uintptr_t *illval = &cpu_core[cpu].cpuc_dtrace_illval; + volatile uintptr_t *illval = &this_cpu_core->cpuc_dtrace_illval; /* * If we hold the privilege to read from kernel memory, then @@ -1243,8 +1242,7 @@ static int dtrace_bcmp(const void *s1, const void *s2, size_t len) { volatile uint16_t *flags; - flags = (volatile uint16_t *)&cpu_core[ - smp_processor_id()].cpuc_dtrace_flags; + flags = (volatile uint16_t *)&this_cpu_core->cpuc_dtrace_flags; if (s1 == s2) return 0; @@ -2173,12 +2171,8 @@ static void dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, dtrace_key_t *tupregs, int nargs, dtrace_mstate_t *mstate, dtrace_state_t *state) { - volatile uint16_t *flags = &cpu_core[ - smp_processor_id() - ].cpuc_dtrace_flags; - volatile uintptr_t *illval = &cpu_core[ - smp_processor_id() - ].cpuc_dtrace_illval; + volatile uint16_t *flags = &this_cpu_core->cpuc_dtrace_flags; + volatile uintptr_t *illval = &this_cpu_core->cpuc_dtrace_illval; dtrace_vstate_t *vstate = &state->dts_vstate; union { @@ -2476,6 +2470,9 @@ static void dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, rval = 1; break; } + + if (p == p->real_parent) + break; } DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); @@ -3667,13 +3664,12 @@ uint64_t dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate, const char *strtab = difo->dtdo_strtab; const uint64_t *inttab = difo->dtdo_inttab; - int cpu = smp_processor_id(); uint64_t rval = 0; dtrace_statvar_t *svar; dtrace_dstate_t *dstate = &vstate->dtvs_dynvars; dtrace_difv_t *v; - volatile uint16_t *flags = &cpu_core[cpu].cpuc_dtrace_flags; - volatile uintptr_t *illval = &cpu_core[cpu].cpuc_dtrace_illval; + volatile uint16_t *flags = &this_cpu_core->cpuc_dtrace_flags; + volatile uintptr_t *illval = &this_cpu_core->cpuc_dtrace_illval; dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */ diff --git a/dtrace/dtrace_helper.c b/dtrace/dtrace_helper.c index 9634c45a5a0f..589fd4add1d1 100644 --- a/dtrace/dtrace_helper.c +++ b/dtrace/dtrace_helper.c @@ -25,6 +25,8 @@ * Use is subject to license terms. */ +#include + #include "dtrace.h" static uint32_t dtrace_helptrace_next = 0; @@ -44,9 +46,7 @@ static void dtrace_helper_trace(dtrace_helper_action_t *helper, { uint32_t size, next, nnext, i; dtrace_helptrace_t *ent; - uint16_t flags = cpu_core[ - smp_processor_id() - ].cpuc_dtrace_flags; + uint16_t flags = this_cpu_core->cpuc_dtrace_flags; if (!dtrace_helptrace_enabled) return; @@ -87,7 +87,7 @@ static void dtrace_helper_trace(dtrace_helper_action_t *helper, ? mstate->dtms_fltoffs : -1; ent->dtht_fault = DTRACE_FLAGS2FLT(flags); - ent->dtht_illval = cpu_core[smp_processor_id()].cpuc_dtrace_illval; + ent->dtht_illval = this_cpu_core->cpuc_dtrace_illval; for (i = 0; i < vstate->dtvs_nlocals; i++) { dtrace_statvar_t *svar; @@ -105,9 +105,7 @@ static void dtrace_helper_trace(dtrace_helper_action_t *helper, uint64_t dtrace_helper(int which, dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t arg0, uint64_t arg1) { - uint16_t *flags = &cpu_core[ - smp_processor_id() - ].cpuc_dtrace_flags; + uint16_t *flags = &this_cpu_core->cpuc_dtrace_flags; uint64_t sarg0 = mstate->dtms_arg[0]; uint64_t sarg1 = mstate->dtms_arg[1]; uint64_t rval = 0; diff --git a/dtrace/dtrace_isa.c b/dtrace/dtrace_isa.c index 3628ad2098bb..1020009110c3 100644 --- a/dtrace/dtrace_isa.c +++ b/dtrace/dtrace_isa.c @@ -25,6 +25,7 @@ * Use is subject to license terms. */ +#include #include #include #include @@ -35,8 +36,6 @@ uintptr_t _userlimit = 0x00007fffffffffffLL; uintptr_t kernelbase = 0xffff880000000000LL; -cpu_core_t cpu_core[NR_CPUS]; -EXPORT_SYMBOL(cpu_core); EXPORT_SYMBOL(dtrace_getfp); DEFINE_MUTEX(cpu_lock); @@ -53,14 +52,14 @@ static int dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size) #else if (kaddr < kernelbase || kaddr + size < kaddr) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); - cpu_core[smp_processor_id()].cpuc_dtrace_illval = kaddr; + this_cpu_core->cpuc_dtrace_illval = kaddr; return 0; } #endif if (uaddr + size >= kernelbase || uaddr + size < uaddr) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); - cpu_core[smp_processor_id()].cpuc_dtrace_illval = uaddr; + this_cpu_core->cpuc_dtrace_illval = uaddr; return 0; } @@ -102,8 +101,7 @@ void dtrace_copyoutstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, \ if ((uintptr_t)uaddr > _userlimit) { \ DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); \ - cpu_core[smp_processor_id()].cpuc_dtrace_illval = \ - (uintptr_t)uaddr; \ + this_cpu_core->cpuc_dtrace_illval = (uintptr_t)uaddr; \ return 0; \ } \ \ diff --git a/dtrace/dtrace_priv.c b/dtrace/dtrace_priv.c index 88cf4b26f51d..a19b08e2fa95 100644 --- a/dtrace/dtrace_priv.c +++ b/dtrace/dtrace_priv.c @@ -25,6 +25,8 @@ * Use is subject to license terms. */ +#include + #include "dtrace.h" /* diff --git a/dtrace/dtrace_probe.c b/dtrace/dtrace_probe.c index 0cd23e9d83c0..053316c75824 100644 --- a/dtrace/dtrace_probe.c +++ b/dtrace/dtrace_probe.c @@ -25,6 +25,7 @@ * Use is subject to license terms. */ +#include #include #include #include @@ -375,7 +376,7 @@ static void dtrace_action_chill(dtrace_mstate_t *mstate, ktime_t val) { ktime_t now; volatile uint16_t *flags; - cpu_core_t *cpu = &cpu_core[smp_processor_id()]; + cpu_core_t *cpu = this_cpu_core; if (dtrace_destructive_disallow) return; @@ -428,9 +429,7 @@ static void dtrace_action_ustack(dtrace_mstate_t *mstate, char *str = (char *)&pcs[nframes]; int size, offs = 0, i, j; uintptr_t old = mstate->dtms_scratch_ptr, saved; - uint16_t *flags = &cpu_core[ - smp_processor_id() - ].cpuc_dtrace_flags; + uint16_t *flags = &this_cpu_core->cpuc_dtrace_flags; char *sym; /* @@ -599,7 +598,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1, mstate.dtms_arg[3] = arg3; mstate.dtms_arg[4] = arg4; - flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags; + flags = (volatile uint16_t *)&this_cpu_core->cpuc_dtrace_flags; for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) { dtrace_predicate_t *pred = ecb->dte_predicate; @@ -1139,7 +1138,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1, ? mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags), - cpu_core[cpuid].cpuc_dtrace_illval); + this_cpu_core->cpuc_dtrace_illval); continue; } diff --git a/dtrace/dtrace_probe_ctx.c b/dtrace/dtrace_probe_ctx.c index 2650a259e4b5..e25531107a12 100644 --- a/dtrace/dtrace_probe_ctx.c +++ b/dtrace/dtrace_probe_ctx.c @@ -25,6 +25,8 @@ * Use is subject to license terms. */ +#include + #include "dtrace.h" void dtrace_panic(const char *fmt, ...) diff --git a/dtrace/dtrace_spec.c b/dtrace/dtrace_spec.c index d02224d3361e..6748c8e06c50 100644 --- a/dtrace/dtrace_spec.c +++ b/dtrace/dtrace_spec.c @@ -25,6 +25,7 @@ * Use is subject to license terms. */ +#include #include #include @@ -95,7 +96,7 @@ void dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu, return; if (which > state->dts_nspeculations) { - cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; + per_cpu_core(cpu)->cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; return; } @@ -238,7 +239,7 @@ void dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu, return; if (which > state->dts_nspeculations) { - cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; + per_cpu_core(cpu)->cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; return; } @@ -379,7 +380,7 @@ void dtrace_speculation_clean(dtrace_state_t *state) * atomically transitioned into the ACTIVEMANY state. */ dtrace_buffer_t *dtrace_speculation_buffer(dtrace_state_t *state, - processorid_t cpuid, + processorid_t cpu, dtrace_specid_t which) { dtrace_speculation_t *spec; @@ -390,12 +391,12 @@ dtrace_buffer_t *dtrace_speculation_buffer(dtrace_state_t *state, return NULL; if (which > state->dts_nspeculations) { - cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; + per_cpu_core(cpu)->cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; return NULL; } spec = &state->dts_speculations[which - 1]; - buf = &spec->dtsp_buffer[cpuid]; + buf = &spec->dtsp_buffer[cpu]; do { curr = spec->dtsp_state; diff --git a/dtrace/dtrace_util.c b/dtrace/dtrace_util.c index 9b4abf7c92d3..a4f086c359f4 100644 --- a/dtrace/dtrace_util.c +++ b/dtrace/dtrace_util.c @@ -25,6 +25,7 @@ * Use is subject to license terms. */ +#include #include #include "dtrace.h" @@ -71,8 +72,7 @@ int dtrace_strncmp(char *s1, char *s2, size_t limit) if (s1 == s2 || limit == 0) return 0; - flags = (volatile uint16_t *) - &cpu_core[smp_processor_id()].cpuc_dtrace_flags; + flags = (volatile uint16_t *)&this_cpu_core->cpuc_dtrace_flags; do { if (s1 == NULL) -- 2.50.1