rcu: Make synchronize_rcu_expedited() fast path update .expedited_sequence

[linux-2.6-microblaze.git] / kernel / rcu / tree_exp.h

diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index 1a51f93..54e05d1 100644 (file)
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -906,6 +906,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp)
 void synchronize_rcu_expedited(void)
 {
        bool boottime = (rcu_scheduler_active == RCU_SCHEDULER_INIT);
+       unsigned long flags;
        struct rcu_exp_work rew;
        struct rcu_node *rnp;
        unsigned long s;
@@ -924,6 +925,11 @@ void synchronize_rcu_expedited(void)
                // them, which allows reuse of ->gp_seq_polled_exp_snap.
                rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap);
                rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap);
+
+               local_irq_save(flags);
+               WARN_ON_ONCE(num_online_cpus() > 1);
+               rcu_state.expedited_sequence += (1 << RCU_SEQ_CTR_SHIFT);
+               local_irq_restore(flags);
                return;  // Context allows vacuous grace periods.
        }