From: Alan Maguire Date: Wed, 4 Oct 2017 21:18:40 +0000 (+0200) Subject: dtrace lockstat provider probes X-Git-Tag: v4.1.12-117.0_27200813_3~314 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=b2fa5f7c5991f941454aa9d56d7b9a2970f62cc6;p=users%2Fjedix%2Flinux-maple.git dtrace lockstat provider probes This patch adds DTrace probes for locking events covering mutexes, read-write locks and spinlocks and is similar to the lockstat provider for Solaris. However it differs from the Solaris lockstat provider in one way - on Linux, rwlocks are implemented via spinlocks so there is no "rw-block" probe; rather a "rw-spin" probe. Additionally, rwlocks cannot be upgraded or downgraded, so the "rw-upgrade" and "rw-downgrade" probes are not present. Probes: lockstat:::adaptive-acquire lockstat:::adaptive-acquire-error lockstat:::adaptive-block lockstat:::adaptive-spin lockstat:::adaptive-release lockstat:::rw-acquire lockstat:::rw-release lockstat:::rw-spin lockstat:::spin-acquire lockstat:::spin-release lockstat:::spin-spin The "-acquire" probes fire when the lock is acquired. The "-spin" probes fire on contention events when then lock needed to spin. The probe fires just prior to acquisition of locks where contention occurred and arg1 contains the total time spent spinning. The "adaptive-block" probe fires on contention events where the thread blocked waiting on lock acquisition. The probe fires just prior to lock acquisition and arg1 contains the total sleep time incurred. The "-error" probe fires when an error occurs when trying to acquire an adpative lock. The "-release" probes fire when the lock is released. Arguments: arg0: the lock itself (a struct mutex *, spinlock_t *, or rwlock_t *) arg1: for rw-acquire/rw-release probes only, arg1 is RW_READER for acquire/release as reader, RW_WRITER for acquire/release as a writer. for *-spin or *-block probes, arg1 is the total time in nanoseconds spent spinning or blocking. arg2: for rw-spin only, arg2 is RW_READER when spinning on a rwlock as a reader, RW_WRITER when spinning on a rwlock as a writer. Orabug: 26149674 Orabug: 26149956 Signed-off-by: Alan Maguire Acked-by: Nick Alcock Reviewed-by: Steve Sistare Reviewed-by: Kris Van Hees --- diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h index 64b611782ef0..bb234dc3a9ad 100644 --- a/arch/x86/include/asm/spinlock.h +++ b/arch/x86/include/asm/spinlock.h @@ -8,6 +8,7 @@ #include #include #include +#include /* * Your basic SMP spinlocks, allowing only a single CPU anywhere @@ -102,11 +103,15 @@ static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) static __always_inline void arch_spin_lock(arch_spinlock_t *lock) { register struct __raw_tickets inc = { .tail = TICKET_LOCK_INC }; + u64 spinstart = 0, spinend, spintime; inc = xadd(&lock->tickets, inc); if (likely(inc.head == inc.tail)) goto out; + if (DTRACE_LOCKSTAT_ENABLED(spin__spin)) + spinstart = dtrace_gethrtime_ns(); + for (;;) { unsigned count = SPIN_THRESHOLD; @@ -122,6 +127,12 @@ clear_slowpath: __ticket_check_and_clear_slowpath(lock, inc.head); out: barrier(); /* make sure nothing creeps before the lock is taken */ + if (DTRACE_LOCKSTAT_ENABLED(spin__spin) && spinstart) { + spinend = dtrace_gethrtime_ns(); + spintime = spinend > spinstart ? spinend - spinstart : 0; + DTRACE_LOCKSTAT(spin__spin, spinlock_t *, lock, + uint64_t, spintime); + } } static __always_inline int arch_spin_trylock(arch_spinlock_t *lock) diff --git a/dtrace/sdt_mod.c b/dtrace/sdt_mod.c index 8203878b3b25..055c6fc11ec9 100644 --- a/dtrace/sdt_mod.c +++ b/dtrace/sdt_mod.c @@ -135,6 +135,7 @@ dtrace_mprovider_t sdt_providers[] = { { "proc", "__proc_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 }, { "io", "__io_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 }, { "ip", "__ip_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 }, + { "lockstat", "__lockstat_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 }, { "tcp", "__tcp_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 }, { "udp", "__udp_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 }, { "mib", "__mib_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 }, diff --git a/include/linux/dtrace_os.h b/include/linux/dtrace_os.h index 86b1223a2bff..145e6783784f 100644 --- a/include/linux/dtrace_os.h +++ b/include/linux/dtrace_os.h @@ -54,6 +54,7 @@ extern void dtrace_vtime_switch(struct task_struct *, struct task_struct *); extern void dtrace_vtime_suspend(void); extern void dtrace_vtime_resume(void); extern void dtrace_chill(ktime_t, ktime_t, ktime_t); +extern ktime_t dtrace_gethrtime(void); extern void dtrace_skip_instruction(struct pt_regs *); diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h index 5b9b84b20407..c985120f5867 100644 --- a/include/linux/rwlock_api_smp.h +++ b/include/linux/rwlock_api_smp.h @@ -5,6 +5,8 @@ # error "please don't include this file directly" #endif +#include + /* * include/linux/rwlock_api_smp.h * @@ -119,6 +121,8 @@ static inline int __raw_read_trylock(rwlock_t *lock) preempt_disable(); if (do_raw_read_trylock(lock)) { rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_); + DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_READER); return 1; } preempt_enable(); @@ -130,6 +134,8 @@ static inline int __raw_write_trylock(rwlock_t *lock) preempt_disable(); if (do_raw_write_trylock(lock)) { rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_); + DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_WRITER); return 1; } preempt_enable(); @@ -148,6 +154,8 @@ static inline void __raw_read_lock(rwlock_t *lock) preempt_disable(); rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_read_trylock, do_raw_read_lock); + DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_READER); } static inline unsigned long __raw_read_lock_irqsave(rwlock_t *lock) @@ -159,6 +167,8 @@ static inline unsigned long __raw_read_lock_irqsave(rwlock_t *lock) rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED_FLAGS(lock, do_raw_read_trylock, do_raw_read_lock, do_raw_read_lock_flags, &flags); + DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_READER); return flags; } @@ -168,6 +178,8 @@ static inline void __raw_read_lock_irq(rwlock_t *lock) preempt_disable(); rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_read_trylock, do_raw_read_lock); + DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_READER); } static inline void __raw_read_lock_bh(rwlock_t *lock) @@ -175,6 +187,8 @@ static inline void __raw_read_lock_bh(rwlock_t *lock) __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_read_trylock, do_raw_read_lock); + DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_READER); } static inline unsigned long __raw_write_lock_irqsave(rwlock_t *lock) @@ -186,6 +200,8 @@ static inline unsigned long __raw_write_lock_irqsave(rwlock_t *lock) rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED_FLAGS(lock, do_raw_write_trylock, do_raw_write_lock, do_raw_write_lock_flags, &flags); + DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_WRITER); return flags; } @@ -195,6 +211,8 @@ static inline void __raw_write_lock_irq(rwlock_t *lock) preempt_disable(); rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_write_trylock, do_raw_write_lock); + DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_WRITER); } static inline void __raw_write_lock_bh(rwlock_t *lock) @@ -202,6 +220,8 @@ static inline void __raw_write_lock_bh(rwlock_t *lock) __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_write_trylock, do_raw_write_lock); + DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_WRITER); } static inline void __raw_write_lock(rwlock_t *lock) @@ -209,6 +229,8 @@ static inline void __raw_write_lock(rwlock_t *lock) preempt_disable(); rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_write_trylock, do_raw_write_lock); + DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_WRITER); } #endif /* CONFIG_PREEMPT */ @@ -217,6 +239,8 @@ static inline void __raw_write_unlock(rwlock_t *lock) { rwlock_release(&lock->dep_map, 1, _RET_IP_); do_raw_write_unlock(lock); + DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_WRITER); preempt_enable(); } @@ -224,6 +248,8 @@ static inline void __raw_read_unlock(rwlock_t *lock) { rwlock_release(&lock->dep_map, 1, _RET_IP_); do_raw_read_unlock(lock); + DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_READER); preempt_enable(); } @@ -232,6 +258,8 @@ __raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) { rwlock_release(&lock->dep_map, 1, _RET_IP_); do_raw_read_unlock(lock); + DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_READER); local_irq_restore(flags); preempt_enable(); } @@ -240,6 +268,8 @@ static inline void __raw_read_unlock_irq(rwlock_t *lock) { rwlock_release(&lock->dep_map, 1, _RET_IP_); do_raw_read_unlock(lock); + DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_READER); local_irq_enable(); preempt_enable(); } @@ -248,6 +278,8 @@ static inline void __raw_read_unlock_bh(rwlock_t *lock) { rwlock_release(&lock->dep_map, 1, _RET_IP_); do_raw_read_unlock(lock); + DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_READER); __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); } @@ -256,6 +288,8 @@ static inline void __raw_write_unlock_irqrestore(rwlock_t *lock, { rwlock_release(&lock->dep_map, 1, _RET_IP_); do_raw_write_unlock(lock); + DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_WRITER); local_irq_restore(flags); preempt_enable(); } @@ -264,6 +298,8 @@ static inline void __raw_write_unlock_irq(rwlock_t *lock) { rwlock_release(&lock->dep_map, 1, _RET_IP_); do_raw_write_unlock(lock); + DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_WRITER); local_irq_enable(); preempt_enable(); } @@ -272,6 +308,8 @@ static inline void __raw_write_unlock_bh(rwlock_t *lock) { rwlock_release(&lock->dep_map, 1, _RET_IP_); do_raw_write_unlock(lock); + DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int, + DTRACE_LOCKSTAT_RW_WRITER); __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); } diff --git a/include/linux/sdt.h b/include/linux/sdt.h index 371449843c04..a9034946425f 100644 --- a/include/linux/sdt.h +++ b/include/linux/sdt.h @@ -169,4 +169,13 @@ typedef struct sdt_probedesc { #define DTRACE_SRP(name, ...) \ DTRACE_PROBE(__srp_##name, ## __VA_ARGS__); +#define DTRACE_LOCKSTAT_ENABLED(name) \ + DTRACE_PROBE_ENABLED(__lockstat_##name) + +#define DTRACE_LOCKSTAT(name, ...) \ + DTRACE_PROBE(__lockstat_##name, ## __VA_ARGS__) + +#define DTRACE_LOCKSTAT_RW_WRITER 0 +#define DTRACE_LOCKSTAT_RW_READER 1 + #endif /* _LINUX_SDT_H_ */ diff --git a/include/linux/sdt_internal.h b/include/linux/sdt_internal.h index ca0edbe5da2a..21dbe5c5b52b 100644 --- a/include/linux/sdt_internal.h +++ b/include/linux/sdt_internal.h @@ -7,6 +7,8 @@ #ifndef _LINUX_SDT_INTERNAL_H_ #define _LINUX_SDT_INTERNAL_H_ +#include + /* * This counts the number of args. */ @@ -145,4 +147,7 @@ #define __DTRACE_APPLY_17(m, def, x1, x2, x3, x4, x5, x6, x7, x8, ...) m(x1), m(x2), m(x3), m(x4), m(x5), m(x6), m(x7), m(x8) #define __DTRACE_APPLY_18(m, def, x1, x2, x3, x4, x5, x6, x7, x8, ...) m(x1), m(x2), m(x3), m(x4), m(x5), m(x6), m(x7), m(x8) +/* Needed for lockstat probes where we cannot include ktime.h */ +extern u64 dtrace_gethrtime_ns(void); + #endif /* _LINUX_SDT_INTERNAL_H */ diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index 5344268e6e62..e807fb3d94ba 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -5,6 +5,8 @@ # error "please don't include this file directly" #endif +#include + /* * include/linux/spinlock_api_smp.h * @@ -90,6 +92,7 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) preempt_disable(); if (do_raw_spin_trylock(lock)) { spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); + DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock); return 1; } preempt_enable(); @@ -120,6 +123,7 @@ static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock) #else do_raw_spin_lock_flags(lock, &flags); #endif + DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock); return flags; } @@ -129,6 +133,7 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock) preempt_disable(); spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); + DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock); } static inline void __raw_spin_lock_bh(raw_spinlock_t *lock) @@ -136,6 +141,7 @@ static inline void __raw_spin_lock_bh(raw_spinlock_t *lock) __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); + DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock); } static inline void __raw_spin_lock(raw_spinlock_t *lock) @@ -143,6 +149,7 @@ static inline void __raw_spin_lock(raw_spinlock_t *lock) preempt_disable(); spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); + DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock); } #endif /* !CONFIG_GENERIC_LOCKBREAK || CONFIG_DEBUG_LOCK_ALLOC */ @@ -151,6 +158,7 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock) { spin_release(&lock->dep_map, 1, _RET_IP_); do_raw_spin_unlock(lock); + DTRACE_LOCKSTAT(spin__release, spinlock_t *, lock); preempt_enable(); } @@ -159,6 +167,7 @@ static inline void __raw_spin_unlock_irqrestore(raw_spinlock_t *lock, { spin_release(&lock->dep_map, 1, _RET_IP_); do_raw_spin_unlock(lock); + DTRACE_LOCKSTAT(spin__release, spinlock_t *, lock); local_irq_restore(flags); preempt_enable(); } @@ -167,6 +176,7 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock) { spin_release(&lock->dep_map, 1, _RET_IP_); do_raw_spin_unlock(lock); + DTRACE_LOCKSTAT(spin__release, spinlock_t *, lock); local_irq_enable(); preempt_enable(); } @@ -175,6 +185,7 @@ static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock) { spin_release(&lock->dep_map, 1, _RET_IP_); do_raw_spin_unlock(lock); + DTRACE_LOCKSTAT(spin__release, spinlock_t *, lock); __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); } @@ -183,6 +194,7 @@ static inline int __raw_spin_trylock_bh(raw_spinlock_t *lock) __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); if (do_raw_spin_trylock(lock)) { spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); + DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock); return 1; } __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); diff --git a/kernel/dtrace/dtrace_os.c b/kernel/dtrace/dtrace_os.c index 047fd5611fc7..199595a05ce5 100644 --- a/kernel/dtrace/dtrace_os.c +++ b/kernel/dtrace/dtrace_os.c @@ -548,6 +548,19 @@ void dtrace_chill(ktime_t val, ktime_t interval, ktime_t int_max) } EXPORT_SYMBOL(dtrace_chill); +/* Needed for lockstat probes where we cannot include ktime.h */ +u64 dtrace_gethrtime_ns(void) +{ + return ktime_get_raw_fast_ns(); +} +EXPORT_SYMBOL(dtrace_gethrtime_ns); + +ktime_t dtrace_gethrtime(void) +{ + return ns_to_ktime(dtrace_gethrtime_ns()); +} +EXPORT_SYMBOL(dtrace_gethrtime); + void dtrace_stacktrace(stacktrace_state_t *st) { struct stack_trace trace; diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 4cccea6b8934..005c5520b0d2 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -26,6 +26,7 @@ #include #include #include +#include /* * In the DEBUG case we are using the "NULL fastpath" for mutexes, @@ -101,6 +102,7 @@ void __sched mutex_lock(struct mutex *lock) */ __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath); mutex_set_owner(lock); + DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock); } EXPORT_SYMBOL(mutex_lock); @@ -435,6 +437,7 @@ void __sched mutex_unlock(struct mutex *lock) mutex_clear_owner(lock); #endif __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath); + DTRACE_LOCKSTAT(adaptive__release, struct mutex *, lock); } EXPORT_SYMBOL(mutex_unlock); @@ -509,6 +512,8 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, struct lockdep_map *nest_lock, unsigned long ip, struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx) { + u64 spinstart = 0, spinend, spintotal = 0; + u64 waitstart, waitend, waittotal = 0; struct task_struct *task = current; struct mutex_waiter waiter; unsigned long flags; @@ -519,6 +524,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, if (mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) { /* got the lock, yay! */ + DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock); preempt_enable(); return 0; } @@ -541,6 +547,9 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, lock_contended(&lock->dep_map, ip); + if (DTRACE_LOCKSTAT_ENABLED(adaptive__spin)) + spinstart = dtrace_gethrtime_ns(); + for (;;) { /* * Lets try to take the lock again - this is needed even if @@ -575,7 +584,16 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, /* didn't get the lock, go to sleep: */ spin_unlock_mutex(&lock->wait_lock, flags); - schedule_preempt_disabled(); + + if (DTRACE_LOCKSTAT_ENABLED(adaptive__block)) { + waitstart = dtrace_gethrtime_ns(); + schedule_preempt_disabled(); + waitend = dtrace_gethrtime_ns(); + if (waitend > waitstart) + waittotal += waitend - waitstart; + } else + schedule_preempt_disabled(); + spin_lock_mutex(&lock->wait_lock, flags); } __set_task_state(task, TASK_RUNNING); @@ -597,6 +615,19 @@ skip_wait: } spin_unlock_mutex(&lock->wait_lock, flags); + + if (DTRACE_LOCKSTAT_ENABLED(adaptive__spin) && spinstart) { + spinend = dtrace_gethrtime_ns(); + spintotal = (spinend > spinstart) ? (spinend - spinstart) : 0; + spintotal = (spintotal > waittotal) ? + (spintotal - waittotal) : 0; + DTRACE_LOCKSTAT(adaptive__spin, struct mutex *, lock, + uint64_t, spintotal); + } + if (DTRACE_LOCKSTAT_ENABLED(adaptive__block) && waittotal) + DTRACE_LOCKSTAT(adaptive__block, struct mutex *, lock, + uint64_t, waittotal); + DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock); preempt_enable(); return 0; @@ -605,6 +636,8 @@ err: spin_unlock_mutex(&lock->wait_lock, flags); debug_mutex_free_waiter(&waiter); mutex_release(&lock->dep_map, 1, ip); + DTRACE_LOCKSTAT(adaptive__acquire__error, struct mutex *, lock, + int, ret); preempt_enable(); return ret; } @@ -790,6 +823,7 @@ int __sched mutex_lock_interruptible(struct mutex *lock) ret = __mutex_fastpath_lock_retval(&lock->count); if (likely(!ret)) { mutex_set_owner(lock); + DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock); return 0; } else return __mutex_lock_interruptible_slowpath(lock); @@ -805,6 +839,7 @@ int __sched mutex_lock_killable(struct mutex *lock) ret = __mutex_fastpath_lock_retval(&lock->count); if (likely(!ret)) { mutex_set_owner(lock); + DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock); return 0; } else return __mutex_lock_killable_slowpath(lock); @@ -901,8 +936,10 @@ int __sched mutex_trylock(struct mutex *lock) int ret; ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath); - if (ret) + if (ret) { mutex_set_owner(lock); + DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock); + } return ret; } diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c index ddf799918520..20e5df4959e8 100644 --- a/kernel/locking/qrwlock.c +++ b/kernel/locking/qrwlock.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -67,9 +68,13 @@ rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts) */ void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts) { + u64 spinstart = 0, spinend, spintime; + /* * Readers come here when they cannot get the lock without waiting */ + if (DTRACE_LOCKSTAT_ENABLED(rw__spin)) + spinstart = dtrace_gethrtime_ns(); if (unlikely(in_interrupt())) { /* * Readers in interrupt context will get the lock immediately @@ -79,7 +84,7 @@ void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts) * is available without waiting in the queue. */ rspin_until_writer_unlock(lock, cnts); - return; + goto done; } atomic_sub(_QR_BIAS, &lock->cnts); @@ -101,6 +106,14 @@ void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts) * Signal the next one in queue to become queue head */ arch_spin_unlock(&lock->lock); +done: + if (DTRACE_LOCKSTAT_ENABLED(rw__spin) && spinstart) { + spinend = dtrace_gethrtime_ns(); + spintime = spinend > spinstart ? spinend - spinstart : 0; + DTRACE_LOCKSTAT(rw__spin, rwlock_t *, lock, uint64_t, spintime, + int, DTRACE_LOCKSTAT_RW_READER); + } + } EXPORT_SYMBOL(queued_read_lock_slowpath); @@ -111,8 +124,11 @@ EXPORT_SYMBOL(queued_read_lock_slowpath); void queued_write_lock_slowpath(struct qrwlock *lock) { u32 cnts; + u64 spinstart = 0, spinend, spintime; /* Put the writer into the wait queue */ + if (DTRACE_LOCKSTAT_ENABLED(rw__spin)) + spinstart = dtrace_gethrtime_ns(); arch_spin_lock(&lock->lock); /* Try to acquire the lock directly if no reader is present */ @@ -146,5 +162,11 @@ void queued_write_lock_slowpath(struct qrwlock *lock) } unlock: arch_spin_unlock(&lock->lock); + if (DTRACE_LOCKSTAT_ENABLED(rw__spin) && spinstart) { + spinend = dtrace_gethrtime_ns(); + spintime = spinend > spinstart ? spinend - spinstart : 0; + DTRACE_LOCKSTAT(rw__spin, rwlock_t *, lock, uint64_t, spintime, + int, DTRACE_LOCKSTAT_RW_WRITER); + } } EXPORT_SYMBOL(queued_write_lock_slowpath);