rcu/nocb: Locally accelerate callbacks as long as offloading isn't complete

The local callbacks processing checks if any callbacks need acceleration.
This commit carries out this checking under nocb lock protection in
the middle of toggle operations, during which time rcu_core() executes
concurrently with GP/CB kthreads.

Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Lai Jiangshan <jiangshanlai@gmail.com>
Cc: Joel Fernandes <joel@joelfernandes.org>
Cc: Neeraj Upadhyay <neeraju@codeaurora.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Inspired-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Boqun Feng <boqun.feng@gmail.com>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index ec14c017c0e3b961f0cea418823ecdbdfc6d0b43..03810a58fdd0d3ebe0f866910cad7c937b54a5c6 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2699,7 +2699,6 @@ static __latent_entropy void rcu_core(void)
        unsigned long flags;
        struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
        struct rcu_node *rnp = rdp->mynode;
-       const bool offloaded = rcu_segcblist_is_offloaded(&rdp->cblist);
        const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist);
 
        if (cpu_is_offline(smp_processor_id()))
@@ -2720,11 +2719,11 @@ static __latent_entropy void rcu_core(void)
 
        /* No grace period and unregistered callbacks? */
        if (!rcu_gp_in_progress() &&
-           rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) {
-               local_irq_save(flags);
+           rcu_segcblist_is_enabled(&rdp->cblist) && do_batch) {
+               rcu_nocb_lock_irqsave(rdp, flags);
                if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
                        rcu_accelerate_cbs_unlocked(rnp, rdp);
-               local_irq_restore(flags);
+               rcu_nocb_unlock_irqrestore(rdp, flags);
        }
 
        rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());