rcu: Move rcu_nmi_{enter,exit}() to prepare for consolidation

This is a code-motion-only commit that prepares to define rcu_irq_enter()
in terms of rcu_nmi_enter() and rcu_irq_exit() in terms of rcu_irq_exit().

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 49f661bb8ffee8762b473138cd07c78808e0aa7a..419f3c38e1b6cba26cf1ff1fee8ea4cd77724dfe 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -866,6 +866,44 @@ void rcu_user_enter(void)
 }
 #endif /* CONFIG_NO_HZ_FULL */
 
+/**
+ * rcu_nmi_exit - inform RCU of exit from NMI context
+ *
+ * If we are returning from the outermost NMI handler that interrupted an
+ * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
+ * to let the RCU grace-period handling know that the CPU is back to
+ * being RCU-idle.
+ *
+ * If you add or remove a call to rcu_nmi_exit(), be sure to test
+ * with CONFIG_RCU_EQS_DEBUG=y.
+ */
+void rcu_nmi_exit(void)
+{
+       struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+       /*
+        * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
+        * (We are exiting an NMI handler, so RCU better be paying attention
+        * to us!)
+        */
+       WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
+       WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
+
+       /*
+        * If the nesting level is not 1, the CPU wasn't RCU-idle, so
+        * leave it in non-RCU-idle state.
+        */
+       if (rdtp->dynticks_nmi_nesting != 1) {
+               WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */
+                          rdtp->dynticks_nmi_nesting - 2);
+               return;
+       }
+
+       /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
+       WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
+       rcu_dynticks_eqs_enter();
+}
+
 /**
  * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
  *
@@ -1012,6 +1050,43 @@ void rcu_user_exit(void)
 }
 #endif /* CONFIG_NO_HZ_FULL */
 
+/**
+ * rcu_nmi_enter - inform RCU of entry to NMI context
+ *
+ * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
+ * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
+ * that the CPU is active.  This implementation permits nested NMIs, as
+ * long as the nesting level does not overflow an int.  (You will probably
+ * run out of stack space first.)
+ *
+ * If you add or remove a call to rcu_nmi_enter(), be sure to test
+ * with CONFIG_RCU_EQS_DEBUG=y.
+ */
+void rcu_nmi_enter(void)
+{
+       struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+       int incby = 2;
+
+       /* Complain about underflow. */
+       WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0);
+
+       /*
+        * If idle from RCU viewpoint, atomically increment ->dynticks
+        * to mark non-idle and increment ->dynticks_nmi_nesting by one.
+        * Otherwise, increment ->dynticks_nmi_nesting by two.  This means
+        * if ->dynticks_nmi_nesting is equal to one, we are guaranteed
+        * to be in the outermost NMI handler that interrupted an RCU-idle
+        * period (observation due to Andy Lutomirski).
+        */
+       if (rcu_dynticks_curr_cpu_in_eqs()) {
+               rcu_dynticks_eqs_exit();
+               incby = 1;
+       }
+       WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */
+                  rdtp->dynticks_nmi_nesting + incby);
+       barrier();
+}
+
 /**
  * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
  *
@@ -1070,81 +1145,6 @@ void rcu_irq_enter_irqson(void)
        local_irq_restore(flags);
 }
 
-/**
- * rcu_nmi_enter - inform RCU of entry to NMI context
- *
- * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
- * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
- * that the CPU is active.  This implementation permits nested NMIs, as
- * long as the nesting level does not overflow an int.  (You will probably
- * run out of stack space first.)
- *
- * If you add or remove a call to rcu_nmi_enter(), be sure to test
- * with CONFIG_RCU_EQS_DEBUG=y.
- */
-void rcu_nmi_enter(void)
-{
-       struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
-       int incby = 2;
-
-       /* Complain about underflow. */
-       WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0);
-
-       /*
-        * If idle from RCU viewpoint, atomically increment ->dynticks
-        * to mark non-idle and increment ->dynticks_nmi_nesting by one.
-        * Otherwise, increment ->dynticks_nmi_nesting by two.  This means
-        * if ->dynticks_nmi_nesting is equal to one, we are guaranteed
-        * to be in the outermost NMI handler that interrupted an RCU-idle
-        * period (observation due to Andy Lutomirski).
-        */
-       if (rcu_dynticks_curr_cpu_in_eqs()) {
-               rcu_dynticks_eqs_exit();
-               incby = 1;
-       }
-       WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */
-                  rdtp->dynticks_nmi_nesting + incby);
-       barrier();
-}
-
-/**
- * rcu_nmi_exit - inform RCU of exit from NMI context
- *
- * If we are returning from the outermost NMI handler that interrupted an
- * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
- * to let the RCU grace-period handling know that the CPU is back to
- * being RCU-idle.
- *
- * If you add or remove a call to rcu_nmi_exit(), be sure to test
- * with CONFIG_RCU_EQS_DEBUG=y.
- */
-void rcu_nmi_exit(void)
-{
-       struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
-
-       /*
-        * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
-        * (We are exiting an NMI handler, so RCU better be paying attention
-        * to us!)
-        */
-       WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
-       WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
-
-       /*
-        * If the nesting level is not 1, the CPU wasn't RCU-idle, so
-        * leave it in non-RCU-idle state.
-        */
-       if (rdtp->dynticks_nmi_nesting != 1) {
-               WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */
-                          rdtp->dynticks_nmi_nesting - 2);
-               return;
-       }
-
-       /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
-       WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
-       rcu_dynticks_eqs_enter();
-}
-
 /**
  * rcu_is_watching - see if RCU thinks that the current CPU is idle
  *