rcu: Make rcu_state.n_online_cpus updates atomic

To support onlining multiple CPUs concurrently,
change rcu_state.n_online_cpus updates to be atomic.
Note, it's ok for rcu_blocking_is_gp() to do a
atomic_read(&rcu_state.n_online_cpus), as the
value of .n_online_cpus switches from 1->2, in
rcutree_prepare_cpu(), which runs before the new
CPU comes online. Similarly 2->1 transition happens
from rcutree_dead_cpu(), which executes after the
CPU is offlined, and runs on the last online CPU.

Signed-off-by: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index a2fff37efcf0a7840e4a4ec42c0c82b6a95c9916..cccd138fcf66cb3791aff8db801bb50079724ca4 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2454,7 +2454,7 @@ int rcutree_dead_cpu(unsigned int cpu)
        if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
                return 0;
 
-       WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus - 1);
+       atomic_dec(&rcu_state.n_online_cpus);
        /* Adjust any no-longer-needed kthreads. */
        rcu_boost_kthread_setaffinity(rnp, -1);
        // Stop-machine done, so allow nohz_full to disable tick.
@@ -3734,7 +3734,7 @@ static int rcu_blocking_is_gp(void)
         * in the code, without the need for additional memory barriers.
         * Those memory barriers are provided by CPU-hotplug code.
         */
-       ret = READ_ONCE(rcu_state.n_online_cpus) <= 1;
+       ret = atomic_read(&rcu_state.n_online_cpus) <= 1;
        preempt_enable();
        return ret;
 }
@@ -4202,7 +4202,7 @@ int rcutree_prepare_cpu(unsigned int cpu)
        raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
        rcu_spawn_one_boost_kthread(rnp);
        rcu_spawn_cpu_nocb_kthread(cpu);
-       WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus + 1);
+       atomic_inc(&rcu_state.n_online_cpus);
 
        return 0;
 }

diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 9815b7844e585d9eed463a9397f2ef7858666240..18a96c1e4920d11bdf95485653a94ab219bab977 100644 (file)
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -301,7 +301,7 @@ struct rcu_state {
                                                /* Hierarchy levels (+1 to */
                                                /*  shut bogus gcc warning) */
        int ncpus;                              /* # CPUs seen so far. */
-       int n_online_cpus;                      /* # CPUs online for RCU. */
+       atomic_t n_online_cpus;                 /* # CPUs online for RCU. */
 
        /* The following fields are guarded by the root rcu_node's lock. */