rcu: Create RCU-specific workqueues with rescuers

RCU's expedited grace periods can participate in out-of-memory deadlocks
due to all available system_wq kthreads being blocked and there not being
memory available to create more.  This commit prevents such deadlocks
by allocating an RCU-specific workqueue_struct at early boot time, and
providing it with a rescuer to ensure forward progress.  This uses the
shiny new init_rescuer() function provided by Tejun (but indirectly).

This commit also causes SRCU to use this new RCU-specific
workqueue_struct.  Note that SRCU's use of workqueues never blocks them
waiting for readers, so this should be safe from a forward-progress
viewpoint.  Note that this moves SRCU from system_power_efficient_wq
to a normal workqueue.  In the unlikely event that this results in
measurable degradation, a separate power-efficient workqueue will be
creates for SRCU.

Reported-by: Prateek Sood <prsood@codeaurora.org>
Reported-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Acked-by: Tejun Heo <tj@kernel.org>

diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index 1c868bcfd7057e22276e8274b70b624f26343743..7a693e31184a6448a58c5a7c52a878b72291aefd 100644 (file)
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -485,6 +485,7 @@ void show_rcu_gp_kthreads(void);
 void rcu_force_quiescent_state(void);
 void rcu_bh_force_quiescent_state(void);
 void rcu_sched_force_quiescent_state(void);
+extern struct workqueue_struct *rcu_gp_wq;
 #endif /* #else #ifdef CONFIG_TINY_RCU */
 
 #ifdef CONFIG_RCU_NOCB_CPU

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 045b559b9f222c3ed025c2ca5b3cdaf8814a31f0..743d183792565e81ba827f5bc61a2f5ef562fb44 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -492,8 +492,7 @@ static bool srcu_queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
  */
 static void srcu_schedule_cbs_sdp(struct srcu_data *sdp, unsigned long delay)
 {
-       srcu_queue_delayed_work_on(sdp->cpu, system_power_efficient_wq,
-                                  &sdp->work, delay);
+       srcu_queue_delayed_work_on(sdp->cpu, rcu_gp_wq, &sdp->work, delay);
 }
 
 /*
@@ -691,8 +690,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp,
            rcu_seq_state(sp->srcu_gp_seq) == SRCU_STATE_IDLE) {
                WARN_ON_ONCE(ULONG_CMP_GE(sp->srcu_gp_seq, sp->srcu_gp_seq_needed));
                srcu_gp_start(sp);
-               queue_delayed_work(system_power_efficient_wq, &sp->work,
-                                  srcu_get_delay(sp));
+               queue_delayed_work(rcu_gp_wq, &sp->work, srcu_get_delay(sp));
        }
        spin_unlock_irqrestore_rcu_node(sp, flags);
 }
@@ -1225,7 +1223,7 @@ static void srcu_reschedule(struct srcu_struct *sp, unsigned long delay)
        spin_unlock_irq_rcu_node(sp);
 
        if (pushgp)
-               queue_delayed_work(system_power_efficient_wq, &sp->work, delay);
+               queue_delayed_work(rcu_gp_wq, &sp->work, delay);
 }
 
 /*

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 99d404c6bbbb885f98eb03ce7512ac0638cb14a8..2a734692a5811089df5552c0a3b65aac76d27439 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -4167,6 +4167,8 @@ static void __init rcu_dump_rcu_node_tree(struct rcu_state *rsp)
        pr_cont("\n");
 }
 
+struct workqueue_struct *rcu_gp_wq;
+
 void __init rcu_init(void)
 {
        int cpu;
@@ -4193,6 +4195,10 @@ void __init rcu_init(void)
                rcu_cpu_starting(cpu);
                rcutree_online_cpu(cpu);
        }
+
+       /* Create workqueue for expedited GPs and for Tree SRCU. */
+       rcu_gp_wq = alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM, 0);
+       WARN_ON(!rcu_gp_wq);
 }
 
 #include "tree_exp.h"

diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index 6ad87642f44aefaeea38bfde48c9a5c8f4e4501a..f72eefab854304976c6360a5d9645e5d69bfd130 100644 (file)
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -626,7 +626,7 @@ static void _synchronize_rcu_expedited(struct rcu_state *rsp,
                rew.rew_rsp = rsp;
                rew.rew_s = s;
                INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp);
-               schedule_work(&rew.rew_work);
+               queue_work(rcu_gp_wq, &rew.rew_work);
        }
 
        /* Wait for expedited grace period to complete. */