*/
for_each_possible_cpu(cpu) {
rdp = per_cpu_ptr(&rcu_data, cpu);
-- ---- if (cpu_is_offline(cpu) &&
-- ---- !rcu_rdp_is_offloaded(rdp))
++ ++++retry:
++ ++++ if (smp_load_acquire(&rdp->barrier_seq_snap) == gseq)
+ continue;
-- - -- if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {
-- - -- rcu_barrier_trace(TPS("OnlineQ"), cpu,
-- - -- rcu_state.barrier_sequence);
-- - -- smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
-- - -- } else if (rcu_segcblist_n_cbs(&rdp->cblist) &&
-- - -- cpu_is_offline(cpu)) {
-- - -- rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
-- - -- rcu_state.barrier_sequence);
-- - -- local_irq_disable();
-- - -- rcu_barrier_func((void *)cpu);
-- - -- local_irq_enable();
-- - -- } else if (cpu_is_offline(cpu)) {
-- - -- rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
-- - -- rcu_state.barrier_sequence);
-- - -- } else {
-- - -- rcu_barrier_trace(TPS("OnlineNQ"), cpu,
-- - -- rcu_state.barrier_sequence);
++ ++++ raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags);
++ ++++ if (!rcu_segcblist_n_cbs(&rdp->cblist)) {
++ ++++ WRITE_ONCE(rdp->barrier_seq_snap, gseq);
++ ++++ raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);
++ ++++ rcu_barrier_trace(TPS("NQ"), cpu, rcu_state.barrier_sequence);
++ + ++ continue;
- if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {
- rcu_barrier_trace(TPS("OnlineQ"), cpu,
- rcu_state.barrier_sequence);
- smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
- } else if (rcu_segcblist_n_cbs(&rdp->cblist) &&
- cpu_is_offline(cpu)) {
- rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
- rcu_state.barrier_sequence);
- local_irq_disable();
- rcu_barrier_func((void *)cpu);
- local_irq_enable();
- } else if (cpu_is_offline(cpu)) {
- rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
- rcu_state.barrier_sequence);
- } else {
- rcu_barrier_trace(TPS("OnlineNQ"), cpu,
- rcu_state.barrier_sequence);
++ + ++ }
++ ++++ if (!rcu_rdp_cpu_online(rdp)) {
++ ++++ rcu_barrier_entrain(rdp);
++ ++++ WARN_ON_ONCE(READ_ONCE(rdp->barrier_seq_snap) != gseq);
++ ++++ raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);
++ ++++ rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu, rcu_state.barrier_sequence);
++ ++++ continue;
++ ++++ }
++ ++++ raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);
++ ++++ if (smp_call_function_single(cpu, rcu_barrier_handler, (void *)cpu, 1)) {
++ ++++ schedule_timeout_uninterruptible(1);
++ ++++ goto retry;
+ }
++ ++++ WARN_ON_ONCE(READ_ONCE(rdp->barrier_seq_snap) != gseq);
++ ++++ rcu_barrier_trace(TPS("OnlineQ"), cpu, rcu_state.barrier_sequence);
}
-- ---- cpus_read_unlock();
/*
* Now that we have an rcu_barrier_callback() callback on each