rcu-tasks: Count trylocks to estimate call_rcu_tasks() contention

This commit converts the unconditional raw_spin_lock_rcu_node() lock
acquisition in call_rcu_tasks_generic() to a trylock followed by an
unconditional acquisition if the trylock fails.  If the trylock fails,
the failure is counted, but the count is reset to zero on each new jiffy.

This statistic will be used to determine when to move from a single
callback queue to per-CPU callback queues.

Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index 9d3eddaecfdee4c74e3d50017a2cb415af89c1be..231a949cdb81d22482ae05092401dcdb32db416f 100644 (file)
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -24,6 +24,8 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp);
  * struct rcu_tasks_percpu - Per-CPU component of definition for a Tasks-RCU-like mechanism.
  * @cblist: Callback list.
  * @lock: Lock protecting per-CPU callback list.
+ * @rtp_jiffies: Jiffies counter value for statistics.
+ * @rtp_n_lock_retries: Rough lock-contention statistic.
  * @rtp_work: Work queue for invoking callbacks.
  * @barrier_q_head: RCU callback for barrier operation.
  * @cpu: CPU number corresponding to this entry.
@@ -32,6 +34,8 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp);
 struct rcu_tasks_percpu {
        struct rcu_segcblist cblist;
        raw_spinlock_t __private lock;
+       unsigned long rtp_jiffies;
+       unsigned long rtp_n_lock_retries;
        struct work_struct rtp_work;
        struct rcu_head barrier_q_head;
        int cpu;
@@ -231,6 +235,7 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func,
                                   struct rcu_tasks *rtp)
 {
        unsigned long flags;
+       unsigned long j;
        bool needwake;
        struct rcu_tasks_percpu *rtpcp;
 
@@ -239,7 +244,15 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func,
        local_irq_save(flags);
        rtpcp = per_cpu_ptr(rtp->rtpcpu,
                            smp_processor_id() >> READ_ONCE(rtp->percpu_enqueue_shift));
-       raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.
+       if (!raw_spin_trylock_rcu_node(rtpcp)) { // irqs already disabled.
+               raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.
+               j = jiffies;
+               if (rtpcp->rtp_jiffies != j) {
+                       rtpcp->rtp_jiffies = j;
+                       rtpcp->rtp_n_lock_retries = 0;
+               }
+               rtpcp->rtp_n_lock_retries++;
+       }
        if (!rcu_segcblist_is_enabled(&rtpcp->cblist)) {
                raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled.
                cblist_init_generic(rtp);