rcu/nocb: Leave ->cblist enabled for no-CBs CPUs

[linux-2.6-microblaze.git] / kernel / rcu / rcutorture.c

diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index fce4e7e..b229473 100644 (file)
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -161,6 +161,7 @@ static atomic_long_t n_rcu_torture_timers;
 static long n_barrier_attempts;
 static long n_barrier_successes; /* did rcu_barrier test succeed? */
 static struct list_head rcu_torture_removed;
+static unsigned long shutdown_jiffies;
 
 static int rcu_torture_writer_state;
 #define RTWS_FIXED_DELAY       0
@@ -228,6 +229,15 @@ static u64 notrace rcu_trace_clock_local(void)
 }
 #endif /* #else #ifdef CONFIG_RCU_TRACE */
 
+/*
+ * Stop aggressive CPU-hog tests a bit before the end of the test in order
+ * to avoid interfering with test shutdown.
+ */
+static bool shutdown_time_arrived(void)
+{
+       return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ);
+}
+
 static unsigned long boost_starttime;  /* jiffies of next boost test start. */
 static DEFINE_MUTEX(boost_mutex);      /* protect setting boost_starttime */
                                        /*  and boost task create/destroy. */
@@ -1713,12 +1723,14 @@ static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)
 }
 
 // Give the scheduler a chance, even on nohz_full CPUs.
-static void rcu_torture_fwd_prog_cond_resched(void)
+static void rcu_torture_fwd_prog_cond_resched(unsigned long iter)
 {
        if (IS_ENABLED(CONFIG_PREEMPT) && IS_ENABLED(CONFIG_NO_HZ_FULL)) {
-               if (need_resched())
+               // Real call_rcu() floods hit userspace, so emulate that.
+               if (need_resched() || (iter & 0xfff))
                        schedule();
        } else {
+               // No userspace emulation: CB invocation throttles call_rcu()
                cond_resched();
        }
 }
@@ -1746,7 +1758,7 @@ static unsigned long rcu_torture_fwd_prog_cbfree(void)
                spin_unlock_irqrestore(&rcu_fwd_lock, flags);
                kfree(rfcp);
                freed++;
-               rcu_torture_fwd_prog_cond_resched();
+               rcu_torture_fwd_prog_cond_resched(freed);
        }
        return freed;
 }
@@ -1785,15 +1797,17 @@ static void rcu_torture_fwd_prog_nr(int *tested, int *tested_tries)
        WRITE_ONCE(rcu_fwd_startat, jiffies);
        stopat = rcu_fwd_startat + dur;
        while (time_before(jiffies, stopat) &&
+              !shutdown_time_arrived() &&
               !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
                idx = cur_ops->readlock();
                udelay(10);
                cur_ops->readunlock(idx);
                if (!fwd_progress_need_resched || need_resched())
-                       rcu_torture_fwd_prog_cond_resched();
+                       rcu_torture_fwd_prog_cond_resched(1);
        }
        (*tested_tries)++;
        if (!time_before(jiffies, stopat) &&
+           !shutdown_time_arrived() &&
            !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
                (*tested)++;
                cver = READ_ONCE(rcu_torture_current_version) - cver;
@@ -1852,6 +1866,7 @@ static void rcu_torture_fwd_prog_cr(void)
        gps = cur_ops->get_gp_seq();
        rcu_launder_gp_seq_start = gps;
        while (time_before(jiffies, stopat) &&
+              !shutdown_time_arrived() &&
               !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
                rfcp = READ_ONCE(rcu_fwd_cb_head);
                rfcpn = NULL;
@@ -1875,7 +1890,7 @@ static void rcu_torture_fwd_prog_cr(void)
                        rfcp->rfc_gps = 0;
                }
                cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
-               rcu_torture_fwd_prog_cond_resched();
+               rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs);
        }
        stoppedat = jiffies;
        n_launders_cb_snap = READ_ONCE(n_launders_cb);
@@ -1884,7 +1899,8 @@ static void rcu_torture_fwd_prog_cr(void)
        cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */
        (void)rcu_torture_fwd_prog_cbfree();
 
-       if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop)) {
+       if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) &&
+           !shutdown_time_arrived()) {
                WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED);
                pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n",
                         __func__,
@@ -2465,6 +2481,7 @@ rcu_torture_init(void)
                        goto unwind;
                rcutor_hp = firsterr;
        }
+       shutdown_jiffies = jiffies + shutdown_secs * HZ;
        firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup);
        if (firsterr)
                goto unwind;