rcu: Make CPU_DYING_IDLE an explicit call

Make the RCU CPU_DYING_IDLE callback an explicit function call, so it gets
invoked at the proper place.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: Rik van Riel <riel@redhat.com>
Cc: Rafael Wysocki <rafael.j.wysocki@intel.com>
Cc: "Srivatsa S. Bhat" <srivatsa@mit.edu>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Sebastian Siewior <bigeasy@linutronix.de>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Paul McKenney <paulmck@linux.vnet.ibm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: http://lkml.kernel.org/r/20160226182341.870167933@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/include/linux/cpu.h b/include/linux/cpu.h
index 91a48d1b4ca01346e89efcd9737487f34f0c1c7a..f9b1fab4388a2052a3cf229eaf64d2249e8ecf8a 100644 (file)
--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -101,9 +101,7 @@ enum {
                                        * Called on the new cpu, just before
                                        * enabling interrupts. Must not sleep,
                                        * must not fail */
-#define CPU_DYING_IDLE         0x000B /* CPU (unsigned)v dying, reached
-                                       * idle loop. */
-#define CPU_BROKEN             0x000C /* CPU (unsigned)v did not die properly,
+#define CPU_BROKEN             0x000B /* CPU (unsigned)v did not die properly,
                                        * perhaps due to preemption. */
 
 /* Used for CPU hotplug events occurring while tasks are frozen due to a suspend

diff --git a/include/linux/notifier.h b/include/linux/notifier.h
index d14a4c362465712857d1ea8fe39b974a5c05c4ce..4149868de4e6c82e8248544ff01ddfe801258be0 100644 (file)
--- a/include/linux/notifier.h
+++ b/include/linux/notifier.h
@@ -47,6 +47,8 @@
  * runtime initialization.
  */
 
+struct notifier_block;
+
 typedef        int (*notifier_fn_t)(struct notifier_block *nb,
                        unsigned long action, void *data);
 

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 14e6f47ee16fe49774894f7e9107d4357b2ce0a7..fc46fe3ea259934e7f35ef6df33ea23fd0ca5a5e 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -332,9 +332,7 @@ void rcu_init(void);
 void rcu_sched_qs(void);
 void rcu_bh_qs(void);
 void rcu_check_callbacks(int user);
-struct notifier_block;
-int rcu_cpu_notify(struct notifier_block *self,
-                  unsigned long action, void *hcpu);
+void rcu_report_dead(unsigned int cpu);
 
 #ifndef CONFIG_TINY_RCU
 void rcu_end_inkernel_boot(void);

diff --git a/kernel/cpu.c b/kernel/cpu.c
index 0e8c07f2566e8de4bb3814dccc6b1413f4b0b95a..ff8059b76a853a28e1514ad3d45b8ded23590156 100644 (file)
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -762,6 +762,7 @@ void cpuhp_report_idle_dead(void)
        BUG_ON(st->state != CPUHP_AP_OFFLINE);
        st->state = CPUHP_AP_IDLE_DEAD;
        complete(&st->done);
+       rcu_report_dead(smp_processor_id());
 }
 
 #else

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index e41dd4131f7a141976e771653e3169f2955f6f33..85b41341272e6ac0f455c5102da5bbae406e7b23 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2606,28 +2606,6 @@ static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
        }
 }
 
-/*
- * The CPU is exiting the idle loop into the arch_cpu_idle_dead()
- * function.  We now remove it from the rcu_node tree's ->qsmaskinit
- * bit masks.
- */
-static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
-{
-       unsigned long flags;
-       unsigned long mask;
-       struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
-       struct rcu_node *rnp = rdp->mynode;  /* Outgoing CPU's rdp & rnp. */
-
-       if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
-               return;
-
-       /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
-       mask = rdp->grpmask;
-       raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
-       rnp->qsmaskinitnext &= ~mask;
-       raw_spin_unlock_irqrestore(&rnp->lock, flags);
-}
-
 /*
  * The CPU has been completely removed, and some other CPU is reporting
  * this fact from process context.  Do the remainder of the cleanup,
@@ -4247,6 +4225,43 @@ static void rcu_prepare_cpu(int cpu)
                rcu_init_percpu_data(cpu, rsp);
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+/*
+ * The CPU is exiting the idle loop into the arch_cpu_idle_dead()
+ * function.  We now remove it from the rcu_node tree's ->qsmaskinit
+ * bit masks.
+ */
+static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
+{
+       unsigned long flags;
+       unsigned long mask;
+       struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
+       struct rcu_node *rnp = rdp->mynode;  /* Outgoing CPU's rdp & rnp. */
+
+       if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
+               return;
+
+       /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
+       mask = rdp->grpmask;
+       raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
+       rnp->qsmaskinitnext &= ~mask;
+       raw_spin_unlock_irqrestore(&rnp->lock, flags);
+}
+
+void rcu_report_dead(unsigned int cpu)
+{
+       struct rcu_state *rsp;
+
+       /* QS for any half-done expedited RCU-sched GP. */
+       preempt_disable();
+       rcu_report_exp_rdp(&rcu_sched_state,
+                          this_cpu_ptr(rcu_sched_state.rda), true);
+       preempt_enable();
+       for_each_rcu_flavor(rsp)
+               rcu_cleanup_dying_idle_cpu(cpu, rsp);
+}
+#endif
+
 /*
  * Handle CPU online/offline notification events.
  */
@@ -4278,17 +4293,6 @@ int rcu_cpu_notify(struct notifier_block *self,
                for_each_rcu_flavor(rsp)
                        rcu_cleanup_dying_cpu(rsp);
                break;
-       case CPU_DYING_IDLE:
-               /* QS for any half-done expedited RCU-sched GP. */
-               preempt_disable();
-               rcu_report_exp_rdp(&rcu_sched_state,
-                                  this_cpu_ptr(rcu_sched_state.rda), true);
-               preempt_enable();
-
-               for_each_rcu_flavor(rsp) {
-                       rcu_cleanup_dying_idle_cpu(cpu, rsp);
-               }
-               break;
        case CPU_DEAD:
        case CPU_DEAD_FROZEN:
        case CPU_UP_CANCELED:

diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 8abbe89e91143c417dd986b81be76be379eb8001..bd12c6c714ecea0718565df4539304e3bf9d9e57 100644 (file)
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -220,8 +220,6 @@ static void cpu_idle_loop(void)
                        rmb();
 
                        if (cpu_is_offline(smp_processor_id())) {
-                               rcu_cpu_notify(NULL, CPU_DYING_IDLE,
-                                              (void *)(long)smp_processor_id());
                                cpuhp_report_idle_dead();
                                arch_cpu_idle_dead();
                        }