rcu: Make ->dynticks_nesting be a simple counter

Now that ->dynticks_nesting counts only process-level dyntick-idle
entry and exit, there is no need for the elaborate segmented counter
with its guard fields and overflow checking.  This commit therefore
makes ->dynticks_nesting be a simple counter.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index f4a411964c417ffa2fbe68e38744b9d182018831..afe0559d18678cfbaba8b18e9fe360bb0e7ad608 100644 (file)
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -30,32 +30,7 @@
 #define RCU_TRACE(stmt)
 #endif /* #else #ifdef CONFIG_RCU_TRACE */
 
-/*
- * Process-level increment to ->dynticks_nesting field.  This allows for
- * architectures that use half-interrupts and half-exceptions from
- * process context.
- *
- * DYNTICK_TASK_NEST_MASK defines a field of width DYNTICK_TASK_NEST_WIDTH
- * that counts the number of process-based reasons why RCU cannot
- * consider the corresponding CPU to be idle, and DYNTICK_TASK_NEST_VALUE
- * is the value used to increment or decrement this field.
- *
- * The rest of the bits could in principle be used to count interrupts,
- * but this would mean that a negative-one value in the interrupt
- * field could incorrectly zero out the DYNTICK_TASK_NEST_MASK field.
- * We therefore provide a two-bit guard field defined by DYNTICK_TASK_MASK
- * that is set to DYNTICK_TASK_FLAG upon initial exit from idle.
- * The DYNTICK_TASK_EXIT_IDLE value is thus the combined value used upon
- * initial exit from idle.
- */
-#define DYNTICK_TASK_NEST_WIDTH 7
-#define DYNTICK_TASK_NEST_VALUE ((LLONG_MAX >> DYNTICK_TASK_NEST_WIDTH) + 1)
-#define DYNTICK_TASK_NEST_MASK  (LLONG_MAX - DYNTICK_TASK_NEST_VALUE + 1)
-#define DYNTICK_TASK_FLAG         ((DYNTICK_TASK_NEST_VALUE / 8) * 2)
-#define DYNTICK_TASK_MASK         ((DYNTICK_TASK_NEST_VALUE / 8) * 3)
-#define DYNTICK_TASK_EXIT_IDLE    (DYNTICK_TASK_NEST_VALUE + \
-                                   DYNTICK_TASK_FLAG)
-
+/* Offset to allow for unmatched rcu_irq_{enter,exit}(). */
 #define DYNTICK_IRQ_NONIDLE    ((INT_MAX / 2) + 1)
 
 

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index fde0e840563faeb670483eaad1cc066d02c871c7..d123474fe829487a36b3f4d9a29ecd9dc6f62fef 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -265,7 +265,7 @@ void rcu_bh_qs(void)
 #endif
 
 static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
-       .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE,
+       .dynticks_nesting = 1,
        .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
        .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
 };
@@ -813,6 +813,10 @@ static void rcu_eqs_enter_common(bool user)
 /*
  * Enter an RCU extended quiescent state, which can be either the
  * idle loop or adaptive-tickless usermode execution.
+ *
+ * We crowbar the ->dynticks_nmi_nesting field to zero to allow for
+ * the possibility of usermode upcalls having messed up our count
+ * of interrupt nesting level during the prior busy period.
  */
 static void rcu_eqs_enter(bool user)
 {
@@ -821,11 +825,11 @@ static void rcu_eqs_enter(bool user)
        rdtp = this_cpu_ptr(&rcu_dynticks);
        WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0);
        WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
-                    (rdtp->dynticks_nesting & DYNTICK_TASK_NEST_MASK) == 0);
-       if ((rdtp->dynticks_nesting & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE)
+                    rdtp->dynticks_nesting == 0);
+       if (rdtp->dynticks_nesting == 1)
                rcu_eqs_enter_common(user);
        else
-               rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE;
+               rdtp->dynticks_nesting--;
 }
 
 /**
@@ -836,10 +840,6 @@ static void rcu_eqs_enter(bool user)
  * critical sections can occur in irq handlers in idle, a possibility
  * handled by irq_enter() and irq_exit().)
  *
- * We crowbar the ->dynticks_nesting field to zero to allow for
- * the possibility of usermode upcalls having messed up our count
- * of interrupt nesting level during the prior busy period.
- *
  * If you add or remove a call to rcu_idle_enter(), be sure to test with
  * CONFIG_RCU_EQS_DEBUG=y.
  */
@@ -984,6 +984,10 @@ static void rcu_eqs_exit_common(long long newval, int user)
 /*
  * Exit an RCU extended quiescent state, which can be either the
  * idle loop or adaptive-tickless usermode execution.
+ *
+ * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to
+ * allow for the possibility of usermode upcalls messing up our count of
+ * interrupt nesting level during the busy period that is just now starting.
  */
 static void rcu_eqs_exit(bool user)
 {
@@ -994,12 +998,12 @@ static void rcu_eqs_exit(bool user)
        rdtp = this_cpu_ptr(&rcu_dynticks);
        oldval = rdtp->dynticks_nesting;
        WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
-       if (oldval & DYNTICK_TASK_NEST_MASK) {
-               rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE;
+       if (oldval) {
+               rdtp->dynticks_nesting++;
        } else {
                __this_cpu_inc(disable_rcu_irq_enter);
-               rcu_eqs_exit_common(DYNTICK_TASK_EXIT_IDLE, user);
-               rdtp->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
+               rcu_eqs_exit_common(1, user);
+               rdtp->dynticks_nesting = 1;
                __this_cpu_dec(disable_rcu_irq_enter);
                WRITE_ONCE(rdtp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
        }
@@ -1011,11 +1015,6 @@ static void rcu_eqs_exit(bool user)
  * Exit idle mode, in other words, -enter- the mode in which RCU
  * read-side critical sections can occur.
  *
- * We crowbar the ->dynticks_nesting field to DYNTICK_TASK_NEST to
- * allow for the possibility of usermode upcalls messing up our count
- * of interrupt nesting level during the busy period that is just
- * now starting.
- *
  * If you add or remove a call to rcu_idle_exit(), be sure to test with
  * CONFIG_RCU_EQS_DEBUG=y.
  */
@@ -1219,7 +1218,8 @@ EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
  */
 static int rcu_is_cpu_rrupt_from_idle(void)
 {
-       return __this_cpu_read(rcu_dynticks.dynticks_nesting) <= 1;
+       return __this_cpu_read(rcu_dynticks.dynticks_nesting) <= 0 &&
+              __this_cpu_read(rcu_dynticks.dynticks_nmi_nesting) <= 1;
 }
 
 /*
@@ -3709,7 +3709,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
        raw_spin_lock_irqsave_rcu_node(rnp, flags);
        rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
        rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
-       WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
+       WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != 1);
        WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp->dynticks)));
        rdp->cpu = cpu;
        rdp->rsp = rsp;
@@ -3738,7 +3738,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
        if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */
            !init_nocb_callback_list(rdp))
                rcu_segcblist_init(&rdp->cblist);  /* Re-enable callbacks. */
-       rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
+       rdp->dynticks->dynticks_nesting = 1;
        rcu_dynticks_eqs_online();
        raw_spin_unlock_rcu_node(rnp);          /* irqs remain disabled. */
 

diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 46a5d1991450b58d520bb8296c122edb1c8b4f03..dbd7e3753bed41b833309854fe1d92df18e92f7e 100644 (file)
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -39,7 +39,6 @@
  */
 struct rcu_dynticks {
        long long dynticks_nesting; /* Track irq/process nesting level. */
-                                   /* Process level is worth LLONG_MAX/2. */
        int dynticks_nmi_nesting;   /* Track NMI nesting level. */
        atomic_t dynticks;          /* Even value for idle, else odd. */
        bool rcu_need_heavy_qs;     /* GP old, need heavy quiescent state. */