dtrace lockstat provider probes

This patch adds DTrace probes for locking events covering mutexes,
read-write locks and spinlocks and is similar to the lockstat provider
for Solaris.  However it differs from the Solaris lockstat provider
in one way - on Linux, rwlocks are implemented via spinlocks so there
is no "rw-block" probe; rather a "rw-spin" probe.  Additionally,
rwlocks cannot be upgraded or downgraded, so the "rw-upgrade" and
"rw-downgrade" probes are not present.

Probes:

lockstat:::adaptive-acquire
lockstat:::adaptive-acquire-error
lockstat:::adaptive-block
lockstat:::adaptive-spin
lockstat:::adaptive-release

lockstat:::rw-acquire
lockstat:::rw-release
lockstat:::rw-spin

lockstat:::spin-acquire
lockstat:::spin-release
lockstat:::spin-spin

The "-acquire" probes fire when the lock is acquired.
The "-spin" probes fire on contention events when then lock needed
to spin.  The probe fires just prior to acquisition of locks where
contention occurred and arg1 contains the total time spent spinning.
The "adaptive-block" probe fires on contention events where the thread
blocked waiting on lock acquisition.  The probe fires just prior to
lock acquisition and arg1 contains the total sleep time incurred.
The "-error" probe fires when an error occurs when trying to
acquire an adpative lock.
The "-release" probes fire when the lock is released.

Arguments:

arg0: the lock itself (a struct mutex *, spinlock_t *, or rwlock_t *)

arg1:

for rw-acquire/rw-release probes only, arg1 is RW_READER for
acquire/release as reader, RW_WRITER for acquire/release as a writer.

for *-spin or *-block probes, arg1 is the total time in nanoseconds
spent spinning or blocking.

arg2:

for rw-spin only, arg2 is RW_READER when spinning on a rwlock as a reader,
RW_WRITER when spinning on a rwlock as a writer.

Orabug: 26149674
Orabug: 26149956

Signed-off-by: Alan Maguire <alan.maguire@oracle.com>
Acked-by: Nick Alcock <nick.alcock@oracle.com>
Reviewed-by: Steve Sistare <steven.sistare@oracle.com>
Reviewed-by: Kris Van Hees <kris.van.hees@oracle.com>

diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 64b611782ef0856f1744611936f76d6e8de1bb57..bb234dc3a9add250f0741d131dde7e682d79987a 100644 (file)
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -8,6 +8,7 @@
 #include <linux/compiler.h>
 #include <asm/paravirt.h>
 #include <asm/bitops.h>
+#include <linux/sdt.h>
 
 /*
  * Your basic SMP spinlocks, allowing only a single CPU anywhere
@@ -102,11 +103,15 @@ static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
 static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
 {
        register struct __raw_tickets inc = { .tail = TICKET_LOCK_INC };
+       u64 spinstart = 0, spinend, spintime;
 
        inc = xadd(&lock->tickets, inc);
        if (likely(inc.head == inc.tail))
                goto out;
 
+       if (DTRACE_LOCKSTAT_ENABLED(spin__spin))
+               spinstart = dtrace_gethrtime_ns();
+
        for (;;) {
                unsigned count = SPIN_THRESHOLD;
 
@@ -122,6 +127,12 @@ clear_slowpath:
        __ticket_check_and_clear_slowpath(lock, inc.head);
 out:
        barrier();      /* make sure nothing creeps before the lock is taken */
+       if (DTRACE_LOCKSTAT_ENABLED(spin__spin) && spinstart) {
+               spinend = dtrace_gethrtime_ns();
+               spintime = spinend > spinstart ? spinend - spinstart : 0;
+               DTRACE_LOCKSTAT(spin__spin, spinlock_t *, lock,
+                               uint64_t, spintime);
+       }
 }
 
 static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)

diff --git a/dtrace/sdt_mod.c b/dtrace/sdt_mod.c
index 8203878b3b2523fdd5d78769f448d49c173c3f6c..055c6fc11ec9becd7b428ed7487b649284d190cb 100644 (file)
--- a/dtrace/sdt_mod.c
+++ b/dtrace/sdt_mod.c
@@ -135,6 +135,7 @@ dtrace_mprovider_t sdt_providers[] = {
   { "proc", "__proc_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 },
   { "io", "__io_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 },
   { "ip", "__ip_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 },
+  { "lockstat", "__lockstat_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 },
   { "tcp", "__tcp_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 },
   { "udp", "__udp_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 },
   { "mib", "__mib_", &stab_attr, DTRACE_PRIV_KERNEL, &sdt_pops, 0 },

diff --git a/include/linux/dtrace_os.h b/include/linux/dtrace_os.h
index 86b1223a2bff16f5f842193b33a2f7e929995dd9..145e6783784f6b8ea25fbd2060f2c43534a9729f 100644 (file)
--- a/include/linux/dtrace_os.h
+++ b/include/linux/dtrace_os.h
@@ -54,6 +54,7 @@ extern void dtrace_vtime_switch(struct task_struct *, struct task_struct *);
 extern void dtrace_vtime_suspend(void);
 extern void dtrace_vtime_resume(void);
 extern void dtrace_chill(ktime_t, ktime_t, ktime_t);
+extern ktime_t dtrace_gethrtime(void);
 
 extern void dtrace_skip_instruction(struct pt_regs *);
 

diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h
index 5b9b84b204070f8e6780e3836ea3c9efdc4a1e61..c985120f5867498e74c5dd18ff1f1aa2a2bacd34 100644 (file)
--- a/include/linux/rwlock_api_smp.h
+++ b/include/linux/rwlock_api_smp.h
@@ -5,6 +5,8 @@
 # error "please don't include this file directly"
 #endif
 
+#include <linux/sdt.h>
+
 /*
  * include/linux/rwlock_api_smp.h
  *
@@ -119,6 +121,8 @@ static inline int __raw_read_trylock(rwlock_t *lock)
        preempt_disable();
        if (do_raw_read_trylock(lock)) {
                rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
+               DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int,
+                               DTRACE_LOCKSTAT_RW_READER);
                return 1;
        }
        preempt_enable();
@@ -130,6 +134,8 @@ static inline int __raw_write_trylock(rwlock_t *lock)
        preempt_disable();
        if (do_raw_write_trylock(lock)) {
                rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
+               DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int,
+                               DTRACE_LOCKSTAT_RW_WRITER);
                return 1;
        }
        preempt_enable();
@@ -148,6 +154,8 @@ static inline void __raw_read_lock(rwlock_t *lock)
        preempt_disable();
        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED(lock, do_raw_read_trylock, do_raw_read_lock);
+       DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_READER);
 }
 
 static inline unsigned long __raw_read_lock_irqsave(rwlock_t *lock)
@@ -159,6 +167,8 @@ static inline unsigned long __raw_read_lock_irqsave(rwlock_t *lock)
        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED_FLAGS(lock, do_raw_read_trylock, do_raw_read_lock,
                             do_raw_read_lock_flags, &flags);
+       DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_READER);
        return flags;
 }
 
@@ -168,6 +178,8 @@ static inline void __raw_read_lock_irq(rwlock_t *lock)
        preempt_disable();
        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED(lock, do_raw_read_trylock, do_raw_read_lock);
+       DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_READER);
 }
 
 static inline void __raw_read_lock_bh(rwlock_t *lock)
@@ -175,6 +187,8 @@ static inline void __raw_read_lock_bh(rwlock_t *lock)
        __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
        rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED(lock, do_raw_read_trylock, do_raw_read_lock);
+       DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_READER);
 }
 
 static inline unsigned long __raw_write_lock_irqsave(rwlock_t *lock)
@@ -186,6 +200,8 @@ static inline unsigned long __raw_write_lock_irqsave(rwlock_t *lock)
        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED_FLAGS(lock, do_raw_write_trylock, do_raw_write_lock,
                             do_raw_write_lock_flags, &flags);
+       DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_WRITER);
        return flags;
 }
 
@@ -195,6 +211,8 @@ static inline void __raw_write_lock_irq(rwlock_t *lock)
        preempt_disable();
        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED(lock, do_raw_write_trylock, do_raw_write_lock);
+       DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_WRITER);
 }
 
 static inline void __raw_write_lock_bh(rwlock_t *lock)
@@ -202,6 +220,8 @@ static inline void __raw_write_lock_bh(rwlock_t *lock)
        __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED(lock, do_raw_write_trylock, do_raw_write_lock);
+       DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_WRITER);
 }
 
 static inline void __raw_write_lock(rwlock_t *lock)
@@ -209,6 +229,8 @@ static inline void __raw_write_lock(rwlock_t *lock)
        preempt_disable();
        rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED(lock, do_raw_write_trylock, do_raw_write_lock);
+       DTRACE_LOCKSTAT(rw__acquire, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_WRITER);
 }
 
 #endif /* CONFIG_PREEMPT */
@@ -217,6 +239,8 @@ static inline void __raw_write_unlock(rwlock_t *lock)
 {
        rwlock_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_write_unlock(lock);
+       DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_WRITER);
        preempt_enable();
 }
 
@@ -224,6 +248,8 @@ static inline void __raw_read_unlock(rwlock_t *lock)
 {
        rwlock_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_read_unlock(lock);
+       DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_READER);
        preempt_enable();
 }
 
@@ -232,6 +258,8 @@ __raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 {
        rwlock_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_read_unlock(lock);
+       DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_READER);
        local_irq_restore(flags);
        preempt_enable();
 }
@@ -240,6 +268,8 @@ static inline void __raw_read_unlock_irq(rwlock_t *lock)
 {
        rwlock_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_read_unlock(lock);
+       DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_READER);
        local_irq_enable();
        preempt_enable();
 }
@@ -248,6 +278,8 @@ static inline void __raw_read_unlock_bh(rwlock_t *lock)
 {
        rwlock_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_read_unlock(lock);
+       DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_READER);
        __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
 }
 
@@ -256,6 +288,8 @@ static inline void __raw_write_unlock_irqrestore(rwlock_t *lock,
 {
        rwlock_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_write_unlock(lock);
+       DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_WRITER);
        local_irq_restore(flags);
        preempt_enable();
 }
@@ -264,6 +298,8 @@ static inline void __raw_write_unlock_irq(rwlock_t *lock)
 {
        rwlock_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_write_unlock(lock);
+       DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_WRITER);
        local_irq_enable();
        preempt_enable();
 }
@@ -272,6 +308,8 @@ static inline void __raw_write_unlock_bh(rwlock_t *lock)
 {
        rwlock_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_write_unlock(lock);
+       DTRACE_LOCKSTAT(rw__release, struct rwlock *, lock, int,
+                       DTRACE_LOCKSTAT_RW_WRITER);
        __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
 }
 

diff --git a/include/linux/sdt.h b/include/linux/sdt.h
index 371449843c04871ba1d2ff2ffa93bba7c94b54bb..a9034946425f49079998a5dd37e525f5a7749fab 100644 (file)
--- a/include/linux/sdt.h
+++ b/include/linux/sdt.h
@@ -169,4 +169,13 @@ typedef struct sdt_probedesc {
 #define        DTRACE_SRP(name, ...)                                           \
        DTRACE_PROBE(__srp_##name, ## __VA_ARGS__);
 
+#define        DTRACE_LOCKSTAT_ENABLED(name)                                   \
+       DTRACE_PROBE_ENABLED(__lockstat_##name)
+
+#define        DTRACE_LOCKSTAT(name, ...)                                      \
+       DTRACE_PROBE(__lockstat_##name, ## __VA_ARGS__)
+
+#define        DTRACE_LOCKSTAT_RW_WRITER       0
+#define        DTRACE_LOCKSTAT_RW_READER       1
+
 #endif /* _LINUX_SDT_H_ */

diff --git a/include/linux/sdt_internal.h b/include/linux/sdt_internal.h
index ca0edbe5da2a49b22b183ac67c527e716d3276aa..21dbe5c5b52b840693f38fa4cea786d5dbc1bd26 100644 (file)
--- a/include/linux/sdt_internal.h
+++ b/include/linux/sdt_internal.h
@@ -7,6 +7,8 @@
 #ifndef _LINUX_SDT_INTERNAL_H_
 #define _LINUX_SDT_INTERNAL_H_
 
+#include <linux/types.h>
+
 /*
  * This counts the number of args.
  */
@@ -145,4 +147,7 @@
 #define __DTRACE_APPLY_17(m, def, x1, x2, x3, x4, x5, x6, x7, x8, ...) m(x1), m(x2), m(x3), m(x4), m(x5), m(x6), m(x7), m(x8)
 #define __DTRACE_APPLY_18(m, def, x1, x2, x3, x4, x5, x6, x7, x8, ...) m(x1), m(x2), m(x3), m(x4), m(x5), m(x6), m(x7), m(x8)
 
+/* Needed for lockstat probes where we cannot include ktime.h */
+extern u64 dtrace_gethrtime_ns(void);
+
 #endif /* _LINUX_SDT_INTERNAL_H */

diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h
index 5344268e6e62fe7db66e2e305829309e968c4d2f..e807fb3d94badbc5ba58848e40ab33e08d1dd11a 100644 (file)
--- a/include/linux/spinlock_api_smp.h
+++ b/include/linux/spinlock_api_smp.h
@@ -5,6 +5,8 @@
 # error "please don't include this file directly"
 #endif
 
+#include <linux/sdt.h>
+
 /*
  * include/linux/spinlock_api_smp.h
  *
@@ -90,6 +92,7 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock)
        preempt_disable();
        if (do_raw_spin_trylock(lock)) {
                spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
+               DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock);
                return 1;
        }
        preempt_enable();
@@ -120,6 +123,7 @@ static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock)
 #else
        do_raw_spin_lock_flags(lock, &flags);
 #endif
+       DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock);
        return flags;
 }
 
@@ -129,6 +133,7 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock)
        preempt_disable();
        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
+       DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock);
 }
 
 static inline void __raw_spin_lock_bh(raw_spinlock_t *lock)
@@ -136,6 +141,7 @@ static inline void __raw_spin_lock_bh(raw_spinlock_t *lock)
        __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
+       DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock);
 }
 
 static inline void __raw_spin_lock(raw_spinlock_t *lock)
@@ -143,6 +149,7 @@ static inline void __raw_spin_lock(raw_spinlock_t *lock)
        preempt_disable();
        spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
        LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
+       DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock);
 }
 
 #endif /* !CONFIG_GENERIC_LOCKBREAK || CONFIG_DEBUG_LOCK_ALLOC */
@@ -151,6 +158,7 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock)
 {
        spin_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_spin_unlock(lock);
+       DTRACE_LOCKSTAT(spin__release, spinlock_t *, lock);
        preempt_enable();
 }
 
@@ -159,6 +167,7 @@ static inline void __raw_spin_unlock_irqrestore(raw_spinlock_t *lock,
 {
        spin_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_spin_unlock(lock);
+       DTRACE_LOCKSTAT(spin__release, spinlock_t *, lock);
        local_irq_restore(flags);
        preempt_enable();
 }
@@ -167,6 +176,7 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock)
 {
        spin_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_spin_unlock(lock);
+       DTRACE_LOCKSTAT(spin__release, spinlock_t *, lock);
        local_irq_enable();
        preempt_enable();
 }
@@ -175,6 +185,7 @@ static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock)
 {
        spin_release(&lock->dep_map, 1, _RET_IP_);
        do_raw_spin_unlock(lock);
+       DTRACE_LOCKSTAT(spin__release, spinlock_t *, lock);
        __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
 }
 
@@ -183,6 +194,7 @@ static inline int __raw_spin_trylock_bh(raw_spinlock_t *lock)
        __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
        if (do_raw_spin_trylock(lock)) {
                spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
+               DTRACE_LOCKSTAT(spin__acquire, spinlock_t *, lock);
                return 1;
        }
        __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);

diff --git a/kernel/dtrace/dtrace_os.c b/kernel/dtrace/dtrace_os.c
index 047fd5611fc713b5eed070322c309cbe0fee9b26..199595a05ce5eadfe76b5846d3b4d3a58490aaf5 100644 (file)
--- a/kernel/dtrace/dtrace_os.c
+++ b/kernel/dtrace/dtrace_os.c
@@ -548,6 +548,19 @@ void dtrace_chill(ktime_t val, ktime_t interval, ktime_t int_max)
 }
 EXPORT_SYMBOL(dtrace_chill);
 
+/* Needed for lockstat probes where we cannot include ktime.h */
+u64 dtrace_gethrtime_ns(void)
+{
+       return ktime_get_raw_fast_ns();
+}
+EXPORT_SYMBOL(dtrace_gethrtime_ns);
+
+ktime_t dtrace_gethrtime(void)
+{
+       return ns_to_ktime(dtrace_gethrtime_ns());
+}
+EXPORT_SYMBOL(dtrace_gethrtime);
+
 void dtrace_stacktrace(stacktrace_state_t *st)
 {
        struct stack_trace      trace;

diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 4cccea6b8934f5697fa3dfa609ae4bd10db78b6f..005c5520b0d2b6b5c47789a8cb0ace48ee982e99 100644 (file)
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -26,6 +26,7 @@
 #include <linux/interrupt.h>
 #include <linux/debug_locks.h>
 #include <linux/osq_lock.h>
+#include <linux/sdt.h>
 
 /*
  * In the DEBUG case we are using the "NULL fastpath" for mutexes,
@@ -101,6 +102,7 @@ void __sched mutex_lock(struct mutex *lock)
         */
        __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);
        mutex_set_owner(lock);
+       DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock);
 }
 
 EXPORT_SYMBOL(mutex_lock);
@@ -435,6 +437,7 @@ void __sched mutex_unlock(struct mutex *lock)
        mutex_clear_owner(lock);
 #endif
        __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);
+       DTRACE_LOCKSTAT(adaptive__release, struct mutex *, lock);
 }
 
 EXPORT_SYMBOL(mutex_unlock);
@@ -509,6 +512,8 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                    struct lockdep_map *nest_lock, unsigned long ip,
                    struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
 {
+       u64 spinstart = 0, spinend, spintotal = 0;
+       u64 waitstart, waitend, waittotal = 0;
        struct task_struct *task = current;
        struct mutex_waiter waiter;
        unsigned long flags;
@@ -519,6 +524,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
 
        if (mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) {
                /* got the lock, yay! */
+               DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock);
                preempt_enable();
                return 0;
        }
@@ -541,6 +547,9 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
 
        lock_contended(&lock->dep_map, ip);
 
+       if (DTRACE_LOCKSTAT_ENABLED(adaptive__spin))
+               spinstart = dtrace_gethrtime_ns();
+
        for (;;) {
                /*
                 * Lets try to take the lock again - this is needed even if
@@ -575,7 +584,16 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
 
                /* didn't get the lock, go to sleep: */
                spin_unlock_mutex(&lock->wait_lock, flags);
-               schedule_preempt_disabled();
+
+               if (DTRACE_LOCKSTAT_ENABLED(adaptive__block)) {
+                       waitstart = dtrace_gethrtime_ns();
+                       schedule_preempt_disabled();
+                       waitend = dtrace_gethrtime_ns();
+                       if (waitend > waitstart)
+                               waittotal += waitend - waitstart;
+               } else
+                       schedule_preempt_disabled();
+
                spin_lock_mutex(&lock->wait_lock, flags);
        }
        __set_task_state(task, TASK_RUNNING);
@@ -597,6 +615,19 @@ skip_wait:
        }
 
        spin_unlock_mutex(&lock->wait_lock, flags);
+
+       if (DTRACE_LOCKSTAT_ENABLED(adaptive__spin) && spinstart) {
+               spinend = dtrace_gethrtime_ns();
+               spintotal = (spinend > spinstart) ? (spinend - spinstart) : 0;
+               spintotal = (spintotal > waittotal) ?
+                       (spintotal - waittotal) : 0;
+               DTRACE_LOCKSTAT(adaptive__spin, struct mutex *, lock,
+                               uint64_t, spintotal);
+       }
+       if (DTRACE_LOCKSTAT_ENABLED(adaptive__block) && waittotal)
+               DTRACE_LOCKSTAT(adaptive__block, struct mutex *, lock,
+                               uint64_t, waittotal);
+       DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock);
        preempt_enable();
        return 0;
 
@@ -605,6 +636,8 @@ err:
        spin_unlock_mutex(&lock->wait_lock, flags);
        debug_mutex_free_waiter(&waiter);
        mutex_release(&lock->dep_map, 1, ip);
+       DTRACE_LOCKSTAT(adaptive__acquire__error, struct mutex *, lock,
+                       int, ret);
        preempt_enable();
        return ret;
 }
@@ -790,6 +823,7 @@ int __sched mutex_lock_interruptible(struct mutex *lock)
        ret =  __mutex_fastpath_lock_retval(&lock->count);
        if (likely(!ret)) {
                mutex_set_owner(lock);
+               DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock);
                return 0;
        } else
                return __mutex_lock_interruptible_slowpath(lock);
@@ -805,6 +839,7 @@ int __sched mutex_lock_killable(struct mutex *lock)
        ret = __mutex_fastpath_lock_retval(&lock->count);
        if (likely(!ret)) {
                mutex_set_owner(lock);
+               DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock);
                return 0;
        } else
                return __mutex_lock_killable_slowpath(lock);
@@ -901,8 +936,10 @@ int __sched mutex_trylock(struct mutex *lock)
        int ret;
 
        ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath);
-       if (ret)
+       if (ret) {
                mutex_set_owner(lock);
+               DTRACE_LOCKSTAT(adaptive__acquire, struct mutex *, lock);
+       }
 
        return ret;
 }

diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c
index ddf799918520db765b0d06cde9ac115532946d5f..20e5df4959e8c23f5109cc7a7e13c488e2644cfa 100644 (file)
--- a/kernel/locking/qrwlock.c
+++ b/kernel/locking/qrwlock.c
@@ -20,6 +20,7 @@
 #include <linux/cpumask.h>
 #include <linux/percpu.h>
 #include <linux/hardirq.h>
+#include <linux/sdt.h>
 #include <linux/spinlock.h>
 #include <asm/qrwlock.h>
 
@@ -67,9 +68,13 @@ rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts)
  */
 void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts)
 {
+       u64 spinstart = 0, spinend, spintime;
+
        /*
         * Readers come here when they cannot get the lock without waiting
         */
+       if (DTRACE_LOCKSTAT_ENABLED(rw__spin))
+               spinstart = dtrace_gethrtime_ns();
        if (unlikely(in_interrupt())) {
                /*
                 * Readers in interrupt context will get the lock immediately
@@ -79,7 +84,7 @@ void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts)
                 * is available without waiting in the queue.
                 */
                rspin_until_writer_unlock(lock, cnts);
-               return;
+               goto done;
        }
        atomic_sub(_QR_BIAS, &lock->cnts);
 
@@ -101,6 +106,14 @@ void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts)
         * Signal the next one in queue to become queue head
         */
        arch_spin_unlock(&lock->lock);
+done:
+       if (DTRACE_LOCKSTAT_ENABLED(rw__spin) && spinstart) {
+               spinend = dtrace_gethrtime_ns();
+               spintime = spinend > spinstart ? spinend - spinstart : 0;
+               DTRACE_LOCKSTAT(rw__spin, rwlock_t *, lock, uint64_t, spintime,
+                               int, DTRACE_LOCKSTAT_RW_READER);
+       }
+
 }
 EXPORT_SYMBOL(queued_read_lock_slowpath);
 
@@ -111,8 +124,11 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);
 void queued_write_lock_slowpath(struct qrwlock *lock)
 {
        u32 cnts;
+       u64 spinstart = 0, spinend, spintime;
 
        /* Put the writer into the wait queue */
+       if (DTRACE_LOCKSTAT_ENABLED(rw__spin))
+               spinstart = dtrace_gethrtime_ns();
        arch_spin_lock(&lock->lock);
 
        /* Try to acquire the lock directly if no reader is present */
@@ -146,5 +162,11 @@ void queued_write_lock_slowpath(struct qrwlock *lock)
        }
 unlock:
        arch_spin_unlock(&lock->lock);
+       if (DTRACE_LOCKSTAT_ENABLED(rw__spin) && spinstart) {
+               spinend = dtrace_gethrtime_ns();
+               spintime = spinend > spinstart ? spinend - spinstart : 0;
+               DTRACE_LOCKSTAT(rw__spin, rwlock_t *, lock, uint64_t, spintime,
+                               int, DTRACE_LOCKSTAT_RW_WRITER);
+       }
 }
 EXPORT_SYMBOL(queued_write_lock_slowpath);