2 * Read-Copy Update module-based torture test facility
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, you can access it online at
16 * http://www.gnu.org/licenses/gpl-2.0.html.
18 * Copyright (C) IBM Corporation, 2005, 2006
20 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
21 * Josh Triplett <josh@joshtriplett.org>
23 * See also: Documentation/RCU/torture.txt
25 #include <linux/types.h>
26 #include <linux/kernel.h>
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/kthread.h>
30 #include <linux/err.h>
31 #include <linux/spinlock.h>
32 #include <linux/smp.h>
33 #include <linux/rcupdate.h>
34 #include <linux/interrupt.h>
35 #include <linux/sched/signal.h>
36 #include <uapi/linux/sched/types.h>
37 #include <linux/atomic.h>
38 #include <linux/bitops.h>
39 #include <linux/completion.h>
40 #include <linux/moduleparam.h>
41 #include <linux/percpu.h>
42 #include <linux/notifier.h>
43 #include <linux/reboot.h>
44 #include <linux/freezer.h>
45 #include <linux/cpu.h>
46 #include <linux/delay.h>
47 #include <linux/stat.h>
48 #include <linux/srcu.h>
49 #include <linux/slab.h>
50 #include <linux/trace_clock.h>
51 #include <asm/byteorder.h>
52 #include <linux/torture.h>
53 #include <linux/vmalloc.h>
55 MODULE_LICENSE("GPL");
56 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
59 torture_param(int, cbflood_inter_holdoff, HZ,
60 "Holdoff between floods (jiffies)");
61 torture_param(int, cbflood_intra_holdoff, 1,
62 "Holdoff between bursts (jiffies)");
63 torture_param(int, cbflood_n_burst, 3, "# bursts in flood, zero to disable");
64 torture_param(int, cbflood_n_per_burst, 20000,
65 "# callbacks per burst in flood");
66 torture_param(int, fqs_duration, 0,
67 "Duration of fqs bursts (us), 0 to disable");
68 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)");
69 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)");
70 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");
71 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
72 torture_param(bool, gp_normal, false,
73 "Use normal (non-expedited) GP wait primitives");
74 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");
75 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");
76 torture_param(int, n_barrier_cbs, 0,
77 "# of callbacks/kthreads for barrier testing");
78 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads");
79 torture_param(int, nreaders, -1, "Number of RCU reader threads");
80 torture_param(int, object_debug, 0,
81 "Enable debug-object double call_rcu() testing");
82 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
83 torture_param(int, onoff_interval, 0,
84 "Time between CPU hotplugs (s), 0=disable");
85 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles");
86 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
87 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable.");
88 torture_param(int, stall_cpu_holdoff, 10,
89 "Time to wait before starting stall (s).");
90 torture_param(int, stat_interval, 60,
91 "Number of seconds between stats printk()s");
92 torture_param(int, stutter, 5, "Number of seconds to run/halt test");
93 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
94 torture_param(int, test_boost_duration, 4,
95 "Duration of each boost test, seconds.");
96 torture_param(int, test_boost_interval, 7,
97 "Interval between boost tests, seconds.");
98 torture_param(bool, test_no_idle_hz, true,
99 "Test support for tickless idle CPUs");
100 torture_param(bool, verbose, true,
101 "Enable verbose debugging printk()s");
103 static char *torture_type = "rcu";
104 module_param(torture_type, charp, 0444);
105 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, ...)");
107 static int nrealreaders;
108 static int ncbflooders;
109 static struct task_struct *writer_task;
110 static struct task_struct **fakewriter_tasks;
111 static struct task_struct **reader_tasks;
112 static struct task_struct *stats_task;
113 static struct task_struct **cbflood_task;
114 static struct task_struct *fqs_task;
115 static struct task_struct *boost_tasks[NR_CPUS];
116 static struct task_struct *stall_task;
117 static struct task_struct **barrier_cbs_tasks;
118 static struct task_struct *barrier_task;
120 #define RCU_TORTURE_PIPE_LEN 10
123 struct rcu_head rtort_rcu;
124 int rtort_pipe_count;
125 struct list_head rtort_free;
129 static LIST_HEAD(rcu_torture_freelist);
130 static struct rcu_torture __rcu *rcu_torture_current;
131 static unsigned long rcu_torture_current_version;
132 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
133 static DEFINE_SPINLOCK(rcu_torture_lock);
134 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count);
135 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch);
136 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
137 static atomic_t n_rcu_torture_alloc;
138 static atomic_t n_rcu_torture_alloc_fail;
139 static atomic_t n_rcu_torture_free;
140 static atomic_t n_rcu_torture_mberror;
141 static atomic_t n_rcu_torture_error;
142 static long n_rcu_torture_barrier_error;
143 static long n_rcu_torture_boost_ktrerror;
144 static long n_rcu_torture_boost_rterror;
145 static long n_rcu_torture_boost_failure;
146 static long n_rcu_torture_boosts;
147 static long n_rcu_torture_timers;
148 static long n_barrier_attempts;
149 static long n_barrier_successes;
150 static atomic_long_t n_cbfloods;
151 static struct list_head rcu_torture_removed;
153 static int rcu_torture_writer_state;
154 #define RTWS_FIXED_DELAY 0
156 #define RTWS_REPLACE 2
157 #define RTWS_DEF_FREE 3
158 #define RTWS_EXP_SYNC 4
159 #define RTWS_COND_GET 5
160 #define RTWS_COND_SYNC 6
162 #define RTWS_STUTTER 8
163 #define RTWS_STOPPING 9
164 static const char * const rcu_torture_writer_state_names[] = {
177 static const char *rcu_torture_writer_state_getname(void)
179 unsigned int i = READ_ONCE(rcu_torture_writer_state);
181 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names))
183 return rcu_torture_writer_state_names[i];
186 static int torture_runnable = IS_ENABLED(MODULE);
187 module_param(torture_runnable, int, 0444);
188 MODULE_PARM_DESC(torture_runnable, "Start rcutorture at boot");
190 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
191 #define rcu_can_boost() 1
192 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
193 #define rcu_can_boost() 0
194 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
196 #ifdef CONFIG_RCU_TRACE
197 static u64 notrace rcu_trace_clock_local(void)
199 u64 ts = trace_clock_local();
200 unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC);
203 #else /* #ifdef CONFIG_RCU_TRACE */
204 static u64 notrace rcu_trace_clock_local(void)
208 #endif /* #else #ifdef CONFIG_RCU_TRACE */
210 static unsigned long boost_starttime; /* jiffies of next boost test start. */
211 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */
212 /* and boost task create/destroy. */
213 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */
214 static bool barrier_phase; /* Test phase. */
215 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */
216 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
217 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
220 * Allocate an element from the rcu_tortures pool.
222 static struct rcu_torture *
223 rcu_torture_alloc(void)
227 spin_lock_bh(&rcu_torture_lock);
228 if (list_empty(&rcu_torture_freelist)) {
229 atomic_inc(&n_rcu_torture_alloc_fail);
230 spin_unlock_bh(&rcu_torture_lock);
233 atomic_inc(&n_rcu_torture_alloc);
234 p = rcu_torture_freelist.next;
236 spin_unlock_bh(&rcu_torture_lock);
237 return container_of(p, struct rcu_torture, rtort_free);
241 * Free an element to the rcu_tortures pool.
244 rcu_torture_free(struct rcu_torture *p)
246 atomic_inc(&n_rcu_torture_free);
247 spin_lock_bh(&rcu_torture_lock);
248 list_add_tail(&p->rtort_free, &rcu_torture_freelist);
249 spin_unlock_bh(&rcu_torture_lock);
253 * Operations vector for selecting different types of tests.
256 struct rcu_torture_ops {
259 void (*cleanup)(void);
260 int (*readlock)(void);
261 void (*read_delay)(struct torture_random_state *rrsp);
262 void (*readunlock)(int idx);
263 unsigned long (*started)(void);
264 unsigned long (*completed)(void);
265 void (*deferred_free)(struct rcu_torture *p);
267 void (*exp_sync)(void);
268 unsigned long (*get_state)(void);
269 void (*cond_sync)(unsigned long oldstate);
270 call_rcu_func_t call;
271 void (*cb_barrier)(void);
279 static struct rcu_torture_ops *cur_ops;
282 * Definitions for rcu torture testing.
285 static int rcu_torture_read_lock(void) __acquires(RCU)
291 static void rcu_read_delay(struct torture_random_state *rrsp)
293 unsigned long started;
294 unsigned long completed;
295 const unsigned long shortdelay_us = 200;
296 const unsigned long longdelay_ms = 50;
297 unsigned long long ts;
299 /* We want a short delay sometimes to make a reader delay the grace
300 * period, and we want a long delay occasionally to trigger
301 * force_quiescent_state. */
303 if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
304 started = cur_ops->completed();
305 ts = rcu_trace_clock_local();
306 mdelay(longdelay_ms);
307 completed = cur_ops->completed();
308 do_trace_rcu_torture_read(cur_ops->name, NULL, ts,
311 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
312 udelay(shortdelay_us);
313 #ifdef CONFIG_PREEMPT
314 if (!preempt_count() &&
315 !(torture_random(rrsp) % (nrealreaders * 20000)))
316 preempt_schedule(); /* No QS if preempt_disable() in effect */
320 static void rcu_torture_read_unlock(int idx) __releases(RCU)
326 * Update callback in the pipe. This should be invoked after a grace period.
329 rcu_torture_pipe_update_one(struct rcu_torture *rp)
333 i = rp->rtort_pipe_count;
334 if (i > RCU_TORTURE_PIPE_LEN)
335 i = RCU_TORTURE_PIPE_LEN;
336 atomic_inc(&rcu_torture_wcount[i]);
337 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
338 rp->rtort_mbtest = 0;
345 * Update all callbacks in the pipe. Suitable for synchronous grace-period
349 rcu_torture_pipe_update(struct rcu_torture *old_rp)
351 struct rcu_torture *rp;
352 struct rcu_torture *rp1;
355 list_add(&old_rp->rtort_free, &rcu_torture_removed);
356 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
357 if (rcu_torture_pipe_update_one(rp)) {
358 list_del(&rp->rtort_free);
359 rcu_torture_free(rp);
365 rcu_torture_cb(struct rcu_head *p)
367 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
369 if (torture_must_stop_irq()) {
370 /* Test is ending, just drop callbacks on the floor. */
371 /* The next initialization will pick up the pieces. */
374 if (rcu_torture_pipe_update_one(rp))
375 rcu_torture_free(rp);
377 cur_ops->deferred_free(rp);
380 static unsigned long rcu_no_completed(void)
385 static void rcu_torture_deferred_free(struct rcu_torture *p)
387 call_rcu(&p->rtort_rcu, rcu_torture_cb);
390 static void rcu_sync_torture_init(void)
392 INIT_LIST_HEAD(&rcu_torture_removed);
395 static struct rcu_torture_ops rcu_ops = {
397 .init = rcu_sync_torture_init,
398 .readlock = rcu_torture_read_lock,
399 .read_delay = rcu_read_delay,
400 .readunlock = rcu_torture_read_unlock,
401 .started = rcu_batches_started,
402 .completed = rcu_batches_completed,
403 .deferred_free = rcu_torture_deferred_free,
404 .sync = synchronize_rcu,
405 .exp_sync = synchronize_rcu_expedited,
406 .get_state = get_state_synchronize_rcu,
407 .cond_sync = cond_synchronize_rcu,
409 .cb_barrier = rcu_barrier,
410 .fqs = rcu_force_quiescent_state,
413 .can_boost = rcu_can_boost(),
418 * Definitions for rcu_bh torture testing.
421 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
427 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
429 rcu_read_unlock_bh();
432 static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
434 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
437 static struct rcu_torture_ops rcu_bh_ops = {
438 .ttype = RCU_BH_FLAVOR,
439 .init = rcu_sync_torture_init,
440 .readlock = rcu_bh_torture_read_lock,
441 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
442 .readunlock = rcu_bh_torture_read_unlock,
443 .started = rcu_batches_started_bh,
444 .completed = rcu_batches_completed_bh,
445 .deferred_free = rcu_bh_torture_deferred_free,
446 .sync = synchronize_rcu_bh,
447 .exp_sync = synchronize_rcu_bh_expedited,
449 .cb_barrier = rcu_barrier_bh,
450 .fqs = rcu_bh_force_quiescent_state,
457 * Don't even think about trying any of these in real life!!!
458 * The names includes "busted", and they really means it!
459 * The only purpose of these functions is to provide a buggy RCU
460 * implementation to make sure that rcutorture correctly emits
461 * buggy-RCU error messages.
463 static void rcu_busted_torture_deferred_free(struct rcu_torture *p)
465 /* This is a deliberate bug for testing purposes only! */
466 rcu_torture_cb(&p->rtort_rcu);
469 static void synchronize_rcu_busted(void)
471 /* This is a deliberate bug for testing purposes only! */
475 call_rcu_busted(struct rcu_head *head, rcu_callback_t func)
477 /* This is a deliberate bug for testing purposes only! */
481 static struct rcu_torture_ops rcu_busted_ops = {
482 .ttype = INVALID_RCU_FLAVOR,
483 .init = rcu_sync_torture_init,
484 .readlock = rcu_torture_read_lock,
485 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
486 .readunlock = rcu_torture_read_unlock,
487 .started = rcu_no_completed,
488 .completed = rcu_no_completed,
489 .deferred_free = rcu_busted_torture_deferred_free,
490 .sync = synchronize_rcu_busted,
491 .exp_sync = synchronize_rcu_busted,
492 .call = call_rcu_busted,
501 * Definitions for srcu torture testing.
504 DEFINE_STATIC_SRCU(srcu_ctl);
505 static struct srcu_struct srcu_ctld;
506 static struct srcu_struct *srcu_ctlp = &srcu_ctl;
508 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp)
510 return srcu_read_lock(srcu_ctlp);
513 static void srcu_read_delay(struct torture_random_state *rrsp)
516 const long uspertick = 1000000 / HZ;
517 const long longdelay = 10;
519 /* We want there to be long-running readers, but not all the time. */
521 delay = torture_random(rrsp) %
522 (nrealreaders * 2 * longdelay * uspertick);
524 schedule_timeout_interruptible(longdelay);
526 rcu_read_delay(rrsp);
529 static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp)
531 srcu_read_unlock(srcu_ctlp, idx);
534 static unsigned long srcu_torture_completed(void)
536 return srcu_batches_completed(srcu_ctlp);
539 static void srcu_torture_deferred_free(struct rcu_torture *rp)
541 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb);
544 static void srcu_torture_synchronize(void)
546 synchronize_srcu(srcu_ctlp);
549 static void srcu_torture_call(struct rcu_head *head,
552 call_srcu(srcu_ctlp, head, func);
555 static void srcu_torture_barrier(void)
557 srcu_barrier(srcu_ctlp);
560 static void srcu_torture_stats(void)
562 int __maybe_unused cpu;
565 #if defined(CONFIG_TREE_SRCU) || defined(CONFIG_CLASSIC_SRCU)
566 #ifdef CONFIG_TREE_SRCU
567 idx = srcu_ctlp->srcu_idx & 0x1;
568 #else /* #ifdef CONFIG_TREE_SRCU */
569 idx = srcu_ctlp->completed & 0x1;
570 #endif /* #else #ifdef CONFIG_TREE_SRCU */
571 pr_alert("%s%s Tree SRCU per-CPU(idx=%d):",
572 torture_type, TORTURE_FLAG, idx);
573 for_each_possible_cpu(cpu) {
574 unsigned long l0, l1;
575 unsigned long u0, u1;
577 #ifdef CONFIG_TREE_SRCU
578 struct srcu_data *counts;
580 counts = per_cpu_ptr(srcu_ctlp->sda, cpu);
581 u0 = counts->srcu_unlock_count[!idx];
582 u1 = counts->srcu_unlock_count[idx];
583 #else /* #ifdef CONFIG_TREE_SRCU */
584 struct srcu_array *counts;
586 counts = per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu);
587 u0 = counts->unlock_count[!idx];
588 u1 = counts->unlock_count[idx];
589 #endif /* #else #ifdef CONFIG_TREE_SRCU */
592 * Make sure that a lock is always counted if the corresponding
597 #ifdef CONFIG_TREE_SRCU
598 l0 = counts->srcu_lock_count[!idx];
599 l1 = counts->srcu_lock_count[idx];
600 #else /* #ifdef CONFIG_TREE_SRCU */
601 l0 = counts->lock_count[!idx];
602 l1 = counts->lock_count[idx];
603 #endif /* #else #ifdef CONFIG_TREE_SRCU */
607 pr_cont(" %d(%ld,%ld)", cpu, c0, c1);
610 #elif defined(CONFIG_TINY_SRCU)
611 idx = READ_ONCE(srcu_ctlp->srcu_idx) & 0x1;
612 pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%d,%d)\n",
613 torture_type, TORTURE_FLAG, idx,
614 READ_ONCE(srcu_ctlp->srcu_lock_nesting[!idx]),
615 READ_ONCE(srcu_ctlp->srcu_lock_nesting[idx]));
619 static void srcu_torture_synchronize_expedited(void)
621 synchronize_srcu_expedited(srcu_ctlp);
624 static struct rcu_torture_ops srcu_ops = {
625 .ttype = SRCU_FLAVOR,
626 .init = rcu_sync_torture_init,
627 .readlock = srcu_torture_read_lock,
628 .read_delay = srcu_read_delay,
629 .readunlock = srcu_torture_read_unlock,
631 .completed = srcu_torture_completed,
632 .deferred_free = srcu_torture_deferred_free,
633 .sync = srcu_torture_synchronize,
634 .exp_sync = srcu_torture_synchronize_expedited,
635 .call = srcu_torture_call,
636 .cb_barrier = srcu_torture_barrier,
637 .stats = srcu_torture_stats,
641 static void srcu_torture_init(void)
643 rcu_sync_torture_init();
644 WARN_ON(init_srcu_struct(&srcu_ctld));
645 srcu_ctlp = &srcu_ctld;
648 static void srcu_torture_cleanup(void)
650 cleanup_srcu_struct(&srcu_ctld);
651 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */
654 /* As above, but dynamically allocated. */
655 static struct rcu_torture_ops srcud_ops = {
656 .ttype = SRCU_FLAVOR,
657 .init = srcu_torture_init,
658 .cleanup = srcu_torture_cleanup,
659 .readlock = srcu_torture_read_lock,
660 .read_delay = srcu_read_delay,
661 .readunlock = srcu_torture_read_unlock,
663 .completed = srcu_torture_completed,
664 .deferred_free = srcu_torture_deferred_free,
665 .sync = srcu_torture_synchronize,
666 .exp_sync = srcu_torture_synchronize_expedited,
667 .call = srcu_torture_call,
668 .cb_barrier = srcu_torture_barrier,
669 .stats = srcu_torture_stats,
674 * Definitions for sched torture testing.
677 static int sched_torture_read_lock(void)
683 static void sched_torture_read_unlock(int idx)
688 static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
690 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
693 static struct rcu_torture_ops sched_ops = {
694 .ttype = RCU_SCHED_FLAVOR,
695 .init = rcu_sync_torture_init,
696 .readlock = sched_torture_read_lock,
697 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
698 .readunlock = sched_torture_read_unlock,
699 .started = rcu_batches_started_sched,
700 .completed = rcu_batches_completed_sched,
701 .deferred_free = rcu_sched_torture_deferred_free,
702 .sync = synchronize_sched,
703 .exp_sync = synchronize_sched_expedited,
704 .get_state = get_state_synchronize_sched,
705 .cond_sync = cond_synchronize_sched,
706 .call = call_rcu_sched,
707 .cb_barrier = rcu_barrier_sched,
708 .fqs = rcu_sched_force_quiescent_state,
714 #ifdef CONFIG_TASKS_RCU
717 * Definitions for RCU-tasks torture testing.
720 static int tasks_torture_read_lock(void)
725 static void tasks_torture_read_unlock(int idx)
729 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p)
731 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb);
734 static struct rcu_torture_ops tasks_ops = {
735 .ttype = RCU_TASKS_FLAVOR,
736 .init = rcu_sync_torture_init,
737 .readlock = tasks_torture_read_lock,
738 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
739 .readunlock = tasks_torture_read_unlock,
740 .started = rcu_no_completed,
741 .completed = rcu_no_completed,
742 .deferred_free = rcu_tasks_torture_deferred_free,
743 .sync = synchronize_rcu_tasks,
744 .exp_sync = synchronize_rcu_tasks,
745 .call = call_rcu_tasks,
746 .cb_barrier = rcu_barrier_tasks,
753 #define RCUTORTURE_TASKS_OPS &tasks_ops,
755 static bool __maybe_unused torturing_tasks(void)
757 return cur_ops == &tasks_ops;
760 #else /* #ifdef CONFIG_TASKS_RCU */
762 #define RCUTORTURE_TASKS_OPS
764 static bool __maybe_unused torturing_tasks(void)
769 #endif /* #else #ifdef CONFIG_TASKS_RCU */
772 * RCU torture priority-boost testing. Runs one real-time thread per
773 * CPU for moderate bursts, repeatedly registering RCU callbacks and
774 * spinning waiting for them to be invoked. If a given callback takes
775 * too long to be invoked, we assume that priority inversion has occurred.
778 struct rcu_boost_inflight {
783 static void rcu_torture_boost_cb(struct rcu_head *head)
785 struct rcu_boost_inflight *rbip =
786 container_of(head, struct rcu_boost_inflight, rcu);
788 /* Ensure RCU-core accesses precede clearing ->inflight */
789 smp_store_release(&rbip->inflight, 0);
792 static int rcu_torture_boost(void *arg)
794 unsigned long call_rcu_time;
795 unsigned long endtime;
796 unsigned long oldstarttime;
797 struct rcu_boost_inflight rbi = { .inflight = 0 };
798 struct sched_param sp;
800 VERBOSE_TOROUT_STRING("rcu_torture_boost started");
802 /* Set real-time priority. */
803 sp.sched_priority = 1;
804 if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) {
805 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!");
806 n_rcu_torture_boost_rterror++;
809 init_rcu_head_on_stack(&rbi.rcu);
810 /* Each pass through the following loop does one boost-test cycle. */
812 /* Wait for the next test interval. */
813 oldstarttime = boost_starttime;
814 while (ULONG_CMP_LT(jiffies, oldstarttime)) {
815 schedule_timeout_interruptible(oldstarttime - jiffies);
816 stutter_wait("rcu_torture_boost");
817 if (torture_must_stop())
821 /* Do one boost-test interval. */
822 endtime = oldstarttime + test_boost_duration * HZ;
823 call_rcu_time = jiffies;
824 while (ULONG_CMP_LT(jiffies, endtime)) {
825 /* If we don't have a callback in flight, post one. */
826 if (!smp_load_acquire(&rbi.inflight)) {
827 /* RCU core before ->inflight = 1. */
828 smp_store_release(&rbi.inflight, 1);
829 call_rcu(&rbi.rcu, rcu_torture_boost_cb);
830 if (jiffies - call_rcu_time >
831 test_boost_duration * HZ - HZ / 2) {
832 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed");
833 n_rcu_torture_boost_failure++;
835 call_rcu_time = jiffies;
837 stutter_wait("rcu_torture_boost");
838 if (torture_must_stop())
843 * Set the start time of the next test interval.
844 * Yes, this is vulnerable to long delays, but such
845 * delays simply cause a false negative for the next
846 * interval. Besides, we are running at RT priority,
847 * so delays should be relatively rare.
849 while (oldstarttime == boost_starttime &&
850 !kthread_should_stop()) {
851 if (mutex_trylock(&boost_mutex)) {
852 boost_starttime = jiffies +
853 test_boost_interval * HZ;
854 n_rcu_torture_boosts++;
855 mutex_unlock(&boost_mutex);
858 schedule_timeout_uninterruptible(1);
861 /* Go do the stutter. */
862 checkwait: stutter_wait("rcu_torture_boost");
863 } while (!torture_must_stop());
865 /* Clean up and exit. */
866 while (!kthread_should_stop() || smp_load_acquire(&rbi.inflight)) {
867 torture_shutdown_absorb("rcu_torture_boost");
868 schedule_timeout_uninterruptible(1);
870 destroy_rcu_head_on_stack(&rbi.rcu);
871 torture_kthread_stopping("rcu_torture_boost");
875 static void rcu_torture_cbflood_cb(struct rcu_head *rhp)
880 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls
881 * to call_rcu() or analogous, increasing the probability of occurrence
882 * of callback-overflow corner cases.
885 rcu_torture_cbflood(void *arg)
890 struct rcu_head *rhp;
892 if (cbflood_n_per_burst > 0 &&
893 cbflood_inter_holdoff > 0 &&
894 cbflood_intra_holdoff > 0 &&
896 cur_ops->cb_barrier) {
897 rhp = vmalloc(sizeof(*rhp) *
898 cbflood_n_burst * cbflood_n_per_burst);
902 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM");
905 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started");
907 schedule_timeout_interruptible(cbflood_inter_holdoff);
908 atomic_long_inc(&n_cbfloods);
909 WARN_ON(signal_pending(current));
910 for (i = 0; i < cbflood_n_burst; i++) {
911 for (j = 0; j < cbflood_n_per_burst; j++) {
912 cur_ops->call(&rhp[i * cbflood_n_per_burst + j],
913 rcu_torture_cbflood_cb);
915 schedule_timeout_interruptible(cbflood_intra_holdoff);
916 WARN_ON(signal_pending(current));
918 cur_ops->cb_barrier();
919 stutter_wait("rcu_torture_cbflood");
920 } while (!torture_must_stop());
923 torture_kthread_stopping("rcu_torture_cbflood");
928 * RCU torture force-quiescent-state kthread. Repeatedly induces
929 * bursts of calls to force_quiescent_state(), increasing the probability
930 * of occurrence of some important types of race conditions.
933 rcu_torture_fqs(void *arg)
935 unsigned long fqs_resume_time;
936 int fqs_burst_remaining;
938 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started");
940 fqs_resume_time = jiffies + fqs_stutter * HZ;
941 while (ULONG_CMP_LT(jiffies, fqs_resume_time) &&
942 !kthread_should_stop()) {
943 schedule_timeout_interruptible(1);
945 fqs_burst_remaining = fqs_duration;
946 while (fqs_burst_remaining > 0 &&
947 !kthread_should_stop()) {
950 fqs_burst_remaining -= fqs_holdoff;
952 stutter_wait("rcu_torture_fqs");
953 } while (!torture_must_stop());
954 torture_kthread_stopping("rcu_torture_fqs");
959 * RCU torture writer kthread. Repeatedly substitutes a new structure
960 * for that pointed to by rcu_torture_current, freeing the old structure
961 * after a series of grace periods (the "pipeline").
964 rcu_torture_writer(void *arg)
966 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal();
968 unsigned long gp_snap;
969 bool gp_cond1 = gp_cond, gp_exp1 = gp_exp, gp_normal1 = gp_normal;
970 bool gp_sync1 = gp_sync;
972 struct rcu_torture *rp;
973 struct rcu_torture *old_rp;
974 static DEFINE_TORTURE_RANDOM(rand);
975 int synctype[] = { RTWS_DEF_FREE, RTWS_EXP_SYNC,
976 RTWS_COND_GET, RTWS_SYNC };
979 VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
981 pr_alert("%s" TORTURE_FLAG
982 " GP expediting controlled from boot/sysfs for %s,\n",
983 torture_type, cur_ops->name);
984 pr_alert("%s" TORTURE_FLAG
985 " Disabled dynamic grace-period expediting.\n",
989 /* Initialize synctype[] array. If none set, take default. */
990 if (!gp_cond1 && !gp_exp1 && !gp_normal1 && !gp_sync1)
991 gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true;
992 if (gp_cond1 && cur_ops->get_state && cur_ops->cond_sync)
993 synctype[nsynctypes++] = RTWS_COND_GET;
994 else if (gp_cond && (!cur_ops->get_state || !cur_ops->cond_sync))
995 pr_alert("rcu_torture_writer: gp_cond without primitives.\n");
996 if (gp_exp1 && cur_ops->exp_sync)
997 synctype[nsynctypes++] = RTWS_EXP_SYNC;
998 else if (gp_exp && !cur_ops->exp_sync)
999 pr_alert("rcu_torture_writer: gp_exp without primitives.\n");
1000 if (gp_normal1 && cur_ops->deferred_free)
1001 synctype[nsynctypes++] = RTWS_DEF_FREE;
1002 else if (gp_normal && !cur_ops->deferred_free)
1003 pr_alert("rcu_torture_writer: gp_normal without primitives.\n");
1004 if (gp_sync1 && cur_ops->sync)
1005 synctype[nsynctypes++] = RTWS_SYNC;
1006 else if (gp_sync && !cur_ops->sync)
1007 pr_alert("rcu_torture_writer: gp_sync without primitives.\n");
1008 if (WARN_ONCE(nsynctypes == 0,
1009 "rcu_torture_writer: No update-side primitives.\n")) {
1011 * No updates primitives, so don't try updating.
1012 * The resulting test won't be testing much, hence the
1013 * above WARN_ONCE().
1015 rcu_torture_writer_state = RTWS_STOPPING;
1016 torture_kthread_stopping("rcu_torture_writer");
1020 rcu_torture_writer_state = RTWS_FIXED_DELAY;
1021 schedule_timeout_uninterruptible(1);
1022 rp = rcu_torture_alloc();
1025 rp->rtort_pipe_count = 0;
1026 rcu_torture_writer_state = RTWS_DELAY;
1027 udelay(torture_random(&rand) & 0x3ff);
1028 rcu_torture_writer_state = RTWS_REPLACE;
1029 old_rp = rcu_dereference_check(rcu_torture_current,
1030 current == writer_task);
1031 rp->rtort_mbtest = 1;
1032 rcu_assign_pointer(rcu_torture_current, rp);
1033 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
1035 i = old_rp->rtort_pipe_count;
1036 if (i > RCU_TORTURE_PIPE_LEN)
1037 i = RCU_TORTURE_PIPE_LEN;
1038 atomic_inc(&rcu_torture_wcount[i]);
1039 old_rp->rtort_pipe_count++;
1040 switch (synctype[torture_random(&rand) % nsynctypes]) {
1042 rcu_torture_writer_state = RTWS_DEF_FREE;
1043 cur_ops->deferred_free(old_rp);
1046 rcu_torture_writer_state = RTWS_EXP_SYNC;
1047 cur_ops->exp_sync();
1048 rcu_torture_pipe_update(old_rp);
1051 rcu_torture_writer_state = RTWS_COND_GET;
1052 gp_snap = cur_ops->get_state();
1053 i = torture_random(&rand) % 16;
1055 schedule_timeout_interruptible(i);
1056 udelay(torture_random(&rand) % 1000);
1057 rcu_torture_writer_state = RTWS_COND_SYNC;
1058 cur_ops->cond_sync(gp_snap);
1059 rcu_torture_pipe_update(old_rp);
1062 rcu_torture_writer_state = RTWS_SYNC;
1064 rcu_torture_pipe_update(old_rp);
1071 rcutorture_record_progress(++rcu_torture_current_version);
1072 /* Cycle through nesting levels of rcu_expedite_gp() calls. */
1074 !(torture_random(&rand) & 0xff & (!!expediting - 1))) {
1075 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited());
1076 if (expediting >= 0)
1079 rcu_unexpedite_gp();
1080 if (++expediting > 3)
1081 expediting = -expediting;
1083 rcu_torture_writer_state = RTWS_STUTTER;
1084 stutter_wait("rcu_torture_writer");
1085 } while (!torture_must_stop());
1086 /* Reset expediting back to unexpedited. */
1088 expediting = -expediting;
1089 while (can_expedite && expediting++ < 0)
1090 rcu_unexpedite_gp();
1091 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited());
1092 rcu_torture_writer_state = RTWS_STOPPING;
1093 torture_kthread_stopping("rcu_torture_writer");
1098 * RCU torture fake writer kthread. Repeatedly calls sync, with a random
1099 * delay between calls.
1102 rcu_torture_fakewriter(void *arg)
1104 DEFINE_TORTURE_RANDOM(rand);
1106 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
1107 set_user_nice(current, MAX_NICE);
1110 schedule_timeout_uninterruptible(1 + torture_random(&rand)%10);
1111 udelay(torture_random(&rand) & 0x3ff);
1112 if (cur_ops->cb_barrier != NULL &&
1113 torture_random(&rand) % (nfakewriters * 8) == 0) {
1114 cur_ops->cb_barrier();
1115 } else if (gp_normal == gp_exp) {
1116 if (torture_random(&rand) & 0x80)
1119 cur_ops->exp_sync();
1120 } else if (gp_normal) {
1123 cur_ops->exp_sync();
1125 stutter_wait("rcu_torture_fakewriter");
1126 } while (!torture_must_stop());
1128 torture_kthread_stopping("rcu_torture_fakewriter");
1133 * RCU torture reader from timer handler. Dereferences rcu_torture_current,
1134 * incrementing the corresponding element of the pipeline array. The
1135 * counter in the element should never be greater than 1, otherwise, the
1136 * RCU implementation is broken.
1138 static void rcu_torture_timer(unsigned long unused)
1141 unsigned long started;
1142 unsigned long completed;
1143 static DEFINE_TORTURE_RANDOM(rand);
1144 static DEFINE_SPINLOCK(rand_lock);
1145 struct rcu_torture *p;
1147 unsigned long long ts;
1149 idx = cur_ops->readlock();
1150 if (cur_ops->started)
1151 started = cur_ops->started();
1153 started = cur_ops->completed();
1154 ts = rcu_trace_clock_local();
1155 p = rcu_dereference_check(rcu_torture_current,
1156 rcu_read_lock_bh_held() ||
1157 rcu_read_lock_sched_held() ||
1158 srcu_read_lock_held(srcu_ctlp) ||
1161 /* Leave because rcu_torture_writer is not yet underway */
1162 cur_ops->readunlock(idx);
1165 if (p->rtort_mbtest == 0)
1166 atomic_inc(&n_rcu_torture_mberror);
1167 spin_lock(&rand_lock);
1168 cur_ops->read_delay(&rand);
1169 n_rcu_torture_timers++;
1170 spin_unlock(&rand_lock);
1172 pipe_count = p->rtort_pipe_count;
1173 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
1174 /* Should not happen, but... */
1175 pipe_count = RCU_TORTURE_PIPE_LEN;
1177 completed = cur_ops->completed();
1178 if (pipe_count > 1) {
1179 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts,
1180 started, completed);
1181 rcu_ftrace_dump(DUMP_ALL);
1183 __this_cpu_inc(rcu_torture_count[pipe_count]);
1184 completed = completed - started;
1185 if (cur_ops->started)
1187 if (completed > RCU_TORTURE_PIPE_LEN) {
1188 /* Should not happen, but... */
1189 completed = RCU_TORTURE_PIPE_LEN;
1191 __this_cpu_inc(rcu_torture_batch[completed]);
1193 cur_ops->readunlock(idx);
1197 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current,
1198 * incrementing the corresponding element of the pipeline array. The
1199 * counter in the element should never be greater than 1, otherwise, the
1200 * RCU implementation is broken.
1203 rcu_torture_reader(void *arg)
1205 unsigned long started;
1206 unsigned long completed;
1208 DEFINE_TORTURE_RANDOM(rand);
1209 struct rcu_torture *p;
1211 struct timer_list t;
1212 unsigned long long ts;
1214 VERBOSE_TOROUT_STRING("rcu_torture_reader task started");
1215 set_user_nice(current, MAX_NICE);
1216 if (irqreader && cur_ops->irq_capable)
1217 setup_timer_on_stack(&t, rcu_torture_timer, 0);
1220 if (irqreader && cur_ops->irq_capable) {
1221 if (!timer_pending(&t))
1222 mod_timer(&t, jiffies + 1);
1224 idx = cur_ops->readlock();
1225 if (cur_ops->started)
1226 started = cur_ops->started();
1228 started = cur_ops->completed();
1229 ts = rcu_trace_clock_local();
1230 p = rcu_dereference_check(rcu_torture_current,
1231 rcu_read_lock_bh_held() ||
1232 rcu_read_lock_sched_held() ||
1233 srcu_read_lock_held(srcu_ctlp) ||
1236 /* Wait for rcu_torture_writer to get underway */
1237 cur_ops->readunlock(idx);
1238 schedule_timeout_interruptible(HZ);
1241 if (p->rtort_mbtest == 0)
1242 atomic_inc(&n_rcu_torture_mberror);
1243 cur_ops->read_delay(&rand);
1245 pipe_count = p->rtort_pipe_count;
1246 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
1247 /* Should not happen, but... */
1248 pipe_count = RCU_TORTURE_PIPE_LEN;
1250 completed = cur_ops->completed();
1251 if (pipe_count > 1) {
1252 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
1253 ts, started, completed);
1254 rcu_ftrace_dump(DUMP_ALL);
1256 __this_cpu_inc(rcu_torture_count[pipe_count]);
1257 completed = completed - started;
1258 if (cur_ops->started)
1260 if (completed > RCU_TORTURE_PIPE_LEN) {
1261 /* Should not happen, but... */
1262 completed = RCU_TORTURE_PIPE_LEN;
1264 __this_cpu_inc(rcu_torture_batch[completed]);
1266 cur_ops->readunlock(idx);
1267 stutter_wait("rcu_torture_reader");
1268 } while (!torture_must_stop());
1269 if (irqreader && cur_ops->irq_capable) {
1271 destroy_timer_on_stack(&t);
1273 torture_kthread_stopping("rcu_torture_reader");
1278 * Print torture statistics. Caller must ensure that there is only
1279 * one call to this function at a given time!!! This is normally
1280 * accomplished by relying on the module system to only have one copy
1281 * of the module loaded, and then by giving the rcu_torture_stats
1282 * kthread full control (or the init/cleanup functions when rcu_torture_stats
1283 * thread is not running).
1286 rcu_torture_stats_print(void)
1290 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1291 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1292 static unsigned long rtcv_snap = ULONG_MAX;
1293 struct task_struct *wtp;
1295 for_each_possible_cpu(cpu) {
1296 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1297 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
1298 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
1301 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
1302 if (pipesummary[i] != 0)
1306 pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1307 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
1308 rcu_torture_current,
1309 rcu_torture_current_version,
1310 list_empty(&rcu_torture_freelist),
1311 atomic_read(&n_rcu_torture_alloc),
1312 atomic_read(&n_rcu_torture_alloc_fail),
1313 atomic_read(&n_rcu_torture_free));
1314 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ",
1315 atomic_read(&n_rcu_torture_mberror),
1316 n_rcu_torture_barrier_error,
1317 n_rcu_torture_boost_ktrerror,
1318 n_rcu_torture_boost_rterror);
1319 pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
1320 n_rcu_torture_boost_failure,
1321 n_rcu_torture_boosts,
1322 n_rcu_torture_timers);
1323 torture_onoff_stats();
1324 pr_cont("barrier: %ld/%ld:%ld ",
1325 n_barrier_successes,
1327 n_rcu_torture_barrier_error);
1328 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods));
1330 pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1331 if (atomic_read(&n_rcu_torture_mberror) != 0 ||
1332 n_rcu_torture_barrier_error != 0 ||
1333 n_rcu_torture_boost_ktrerror != 0 ||
1334 n_rcu_torture_boost_rterror != 0 ||
1335 n_rcu_torture_boost_failure != 0 ||
1337 pr_cont("%s", "!!! ");
1338 atomic_inc(&n_rcu_torture_error);
1341 pr_cont("Reader Pipe: ");
1342 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1343 pr_cont(" %ld", pipesummary[i]);
1346 pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1347 pr_cont("Reader Batch: ");
1348 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1349 pr_cont(" %ld", batchsummary[i]);
1352 pr_alert("%s%s ", torture_type, TORTURE_FLAG);
1353 pr_cont("Free-Block Circulation: ");
1354 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1355 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i]));
1361 if (rtcv_snap == rcu_torture_current_version &&
1362 rcu_torture_current != NULL) {
1363 int __maybe_unused flags = 0;
1364 unsigned long __maybe_unused gpnum = 0;
1365 unsigned long __maybe_unused completed = 0;
1367 rcutorture_get_gp_data(cur_ops->ttype,
1368 &flags, &gpnum, &completed);
1369 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp,
1370 &flags, &gpnum, &completed);
1371 wtp = READ_ONCE(writer_task);
1372 pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx\n",
1373 rcu_torture_writer_state_getname(),
1374 rcu_torture_writer_state,
1375 gpnum, completed, flags,
1376 wtp == NULL ? ~0UL : wtp->state);
1377 show_rcu_gp_kthreads();
1378 rcu_ftrace_dump(DUMP_ALL);
1380 rtcv_snap = rcu_torture_current_version;
1384 * Periodically prints torture statistics, if periodic statistics printing
1385 * was specified via the stat_interval module parameter.
1388 rcu_torture_stats(void *arg)
1390 VERBOSE_TOROUT_STRING("rcu_torture_stats task started");
1392 schedule_timeout_interruptible(stat_interval * HZ);
1393 rcu_torture_stats_print();
1394 torture_shutdown_absorb("rcu_torture_stats");
1395 } while (!torture_must_stop());
1396 torture_kthread_stopping("rcu_torture_stats");
1401 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
1403 pr_alert("%s" TORTURE_FLAG
1404 "--- %s: nreaders=%d nfakewriters=%d "
1405 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
1406 "shuffle_interval=%d stutter=%d irqreader=%d "
1407 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
1408 "test_boost=%d/%d test_boost_interval=%d "
1409 "test_boost_duration=%d shutdown_secs=%d "
1410 "stall_cpu=%d stall_cpu_holdoff=%d "
1412 "onoff_interval=%d onoff_holdoff=%d\n",
1413 torture_type, tag, nrealreaders, nfakewriters,
1414 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
1415 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
1416 test_boost, cur_ops->can_boost,
1417 test_boost_interval, test_boost_duration, shutdown_secs,
1418 stall_cpu, stall_cpu_holdoff,
1420 onoff_interval, onoff_holdoff);
1423 static int rcutorture_booster_cleanup(unsigned int cpu)
1425 struct task_struct *t;
1427 if (boost_tasks[cpu] == NULL)
1429 mutex_lock(&boost_mutex);
1430 t = boost_tasks[cpu];
1431 boost_tasks[cpu] = NULL;
1432 mutex_unlock(&boost_mutex);
1434 /* This must be outside of the mutex, otherwise deadlock! */
1435 torture_stop_kthread(rcu_torture_boost, t);
1439 static int rcutorture_booster_init(unsigned int cpu)
1443 if (boost_tasks[cpu] != NULL)
1444 return 0; /* Already created, nothing more to do. */
1446 /* Don't allow time recalculation while creating a new task. */
1447 mutex_lock(&boost_mutex);
1448 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task");
1449 boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL,
1451 "rcu_torture_boost");
1452 if (IS_ERR(boost_tasks[cpu])) {
1453 retval = PTR_ERR(boost_tasks[cpu]);
1454 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed");
1455 n_rcu_torture_boost_ktrerror++;
1456 boost_tasks[cpu] = NULL;
1457 mutex_unlock(&boost_mutex);
1460 kthread_bind(boost_tasks[cpu], cpu);
1461 wake_up_process(boost_tasks[cpu]);
1462 mutex_unlock(&boost_mutex);
1467 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then
1468 * induces a CPU stall for the time specified by stall_cpu.
1470 static int rcu_torture_stall(void *args)
1472 unsigned long stop_at;
1474 VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
1475 if (stall_cpu_holdoff > 0) {
1476 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
1477 schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
1478 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff");
1480 if (!kthread_should_stop()) {
1481 stop_at = get_seconds() + stall_cpu;
1482 /* RCU CPU stall is expected behavior in following code. */
1483 pr_alert("rcu_torture_stall start.\n");
1486 while (ULONG_CMP_LT(get_seconds(), stop_at))
1487 continue; /* Induce RCU CPU stall warning. */
1490 pr_alert("rcu_torture_stall end.\n");
1492 torture_shutdown_absorb("rcu_torture_stall");
1493 while (!kthread_should_stop())
1494 schedule_timeout_interruptible(10 * HZ);
1498 /* Spawn CPU-stall kthread, if stall_cpu specified. */
1499 static int __init rcu_torture_stall_init(void)
1503 return torture_create_kthread(rcu_torture_stall, NULL, stall_task);
1506 /* Callback function for RCU barrier testing. */
1507 static void rcu_torture_barrier_cbf(struct rcu_head *rcu)
1509 atomic_inc(&barrier_cbs_invoked);
1512 /* kthread function to register callbacks used to test RCU barriers. */
1513 static int rcu_torture_barrier_cbs(void *arg)
1515 long myid = (long)arg;
1518 struct rcu_head rcu;
1520 init_rcu_head_on_stack(&rcu);
1521 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started");
1522 set_user_nice(current, MAX_NICE);
1524 wait_event(barrier_cbs_wq[myid],
1526 smp_load_acquire(&barrier_phase)) != lastphase ||
1527 torture_must_stop());
1528 lastphase = newphase;
1529 if (torture_must_stop())
1532 * The above smp_load_acquire() ensures barrier_phase load
1533 * is ordered before the following ->call().
1535 local_irq_disable(); /* Just to test no-irq call_rcu(). */
1536 cur_ops->call(&rcu, rcu_torture_barrier_cbf);
1538 if (atomic_dec_and_test(&barrier_cbs_count))
1539 wake_up(&barrier_wq);
1540 } while (!torture_must_stop());
1541 if (cur_ops->cb_barrier != NULL)
1542 cur_ops->cb_barrier();
1543 destroy_rcu_head_on_stack(&rcu);
1544 torture_kthread_stopping("rcu_torture_barrier_cbs");
1548 /* kthread function to drive and coordinate RCU barrier testing. */
1549 static int rcu_torture_barrier(void *arg)
1553 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting");
1555 atomic_set(&barrier_cbs_invoked, 0);
1556 atomic_set(&barrier_cbs_count, n_barrier_cbs);
1557 /* Ensure barrier_phase ordered after prior assignments. */
1558 smp_store_release(&barrier_phase, !barrier_phase);
1559 for (i = 0; i < n_barrier_cbs; i++)
1560 wake_up(&barrier_cbs_wq[i]);
1561 wait_event(barrier_wq,
1562 atomic_read(&barrier_cbs_count) == 0 ||
1563 torture_must_stop());
1564 if (torture_must_stop())
1566 n_barrier_attempts++;
1567 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */
1568 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
1569 n_rcu_torture_barrier_error++;
1570 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
1571 atomic_read(&barrier_cbs_invoked),
1575 n_barrier_successes++;
1576 schedule_timeout_interruptible(HZ / 10);
1577 } while (!torture_must_stop());
1578 torture_kthread_stopping("rcu_torture_barrier");
1582 /* Initialize RCU barrier testing. */
1583 static int rcu_torture_barrier_init(void)
1588 if (n_barrier_cbs <= 0)
1590 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
1591 pr_alert("%s" TORTURE_FLAG
1592 " Call or barrier ops missing for %s,\n",
1593 torture_type, cur_ops->name);
1594 pr_alert("%s" TORTURE_FLAG
1595 " RCU barrier testing omitted from run.\n",
1599 atomic_set(&barrier_cbs_count, 0);
1600 atomic_set(&barrier_cbs_invoked, 0);
1602 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]),
1605 kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),
1607 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq)
1609 for (i = 0; i < n_barrier_cbs; i++) {
1610 init_waitqueue_head(&barrier_cbs_wq[i]);
1611 ret = torture_create_kthread(rcu_torture_barrier_cbs,
1613 barrier_cbs_tasks[i]);
1617 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task);
1620 /* Clean up after RCU barrier testing. */
1621 static void rcu_torture_barrier_cleanup(void)
1625 torture_stop_kthread(rcu_torture_barrier, barrier_task);
1626 if (barrier_cbs_tasks != NULL) {
1627 for (i = 0; i < n_barrier_cbs; i++)
1628 torture_stop_kthread(rcu_torture_barrier_cbs,
1629 barrier_cbs_tasks[i]);
1630 kfree(barrier_cbs_tasks);
1631 barrier_cbs_tasks = NULL;
1633 if (barrier_cbs_wq != NULL) {
1634 kfree(barrier_cbs_wq);
1635 barrier_cbs_wq = NULL;
1639 static enum cpuhp_state rcutor_hp;
1642 rcu_torture_cleanup(void)
1646 rcutorture_record_test_transition();
1647 if (torture_cleanup_begin()) {
1648 if (cur_ops->cb_barrier != NULL)
1649 cur_ops->cb_barrier();
1653 rcu_torture_barrier_cleanup();
1654 torture_stop_kthread(rcu_torture_stall, stall_task);
1655 torture_stop_kthread(rcu_torture_writer, writer_task);
1658 for (i = 0; i < nrealreaders; i++)
1659 torture_stop_kthread(rcu_torture_reader,
1661 kfree(reader_tasks);
1663 rcu_torture_current = NULL;
1665 if (fakewriter_tasks) {
1666 for (i = 0; i < nfakewriters; i++) {
1667 torture_stop_kthread(rcu_torture_fakewriter,
1668 fakewriter_tasks[i]);
1670 kfree(fakewriter_tasks);
1671 fakewriter_tasks = NULL;
1674 torture_stop_kthread(rcu_torture_stats, stats_task);
1675 torture_stop_kthread(rcu_torture_fqs, fqs_task);
1676 for (i = 0; i < ncbflooders; i++)
1677 torture_stop_kthread(rcu_torture_cbflood, cbflood_task[i]);
1678 if ((test_boost == 1 && cur_ops->can_boost) ||
1680 cpuhp_remove_state(rcutor_hp);
1683 * Wait for all RCU callbacks to fire, then do flavor-specific
1684 * cleanup operations.
1686 if (cur_ops->cb_barrier != NULL)
1687 cur_ops->cb_barrier();
1688 if (cur_ops->cleanup != NULL)
1691 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
1693 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
1694 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
1695 else if (torture_onoff_failures())
1696 rcu_torture_print_module_parms(cur_ops,
1697 "End of test: RCU_HOTPLUG");
1699 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
1700 torture_cleanup_end();
1703 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
1704 static void rcu_torture_leak_cb(struct rcu_head *rhp)
1708 static void rcu_torture_err_cb(struct rcu_head *rhp)
1711 * This -might- happen due to race conditions, but is unlikely.
1712 * The scenario that leads to this happening is that the
1713 * first of the pair of duplicate callbacks is queued,
1714 * someone else starts a grace period that includes that
1715 * callback, then the second of the pair must wait for the
1716 * next grace period. Unlikely, but can happen. If it
1717 * does happen, the debug-objects subsystem won't have splatted.
1719 pr_alert("rcutorture: duplicated callback was invoked.\n");
1721 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
1724 * Verify that double-free causes debug-objects to complain, but only
1725 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test
1726 * cannot be carried out.
1728 static void rcu_test_debug_objects(void)
1730 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
1731 struct rcu_head rh1;
1732 struct rcu_head rh2;
1734 init_rcu_head_on_stack(&rh1);
1735 init_rcu_head_on_stack(&rh2);
1736 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n");
1738 /* Try to queue the rh2 pair of callbacks for the same grace period. */
1739 preempt_disable(); /* Prevent preemption from interrupting test. */
1740 rcu_read_lock(); /* Make it impossible to finish a grace period. */
1741 call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */
1742 local_irq_disable(); /* Make it harder to start a new grace period. */
1743 call_rcu(&rh2, rcu_torture_leak_cb);
1744 call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
1749 /* Wait for them all to get done so we can safely return. */
1751 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n");
1752 destroy_rcu_head_on_stack(&rh1);
1753 destroy_rcu_head_on_stack(&rh2);
1754 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
1755 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n");
1756 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
1760 rcu_torture_init(void)
1765 static struct rcu_torture_ops *torture_ops[] = {
1766 &rcu_ops, &rcu_bh_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops,
1767 &sched_ops, RCUTORTURE_TASKS_OPS
1770 if (!torture_init_begin(torture_type, verbose, &torture_runnable))
1773 /* Process args and tell the world that the torturer is on the job. */
1774 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1775 cur_ops = torture_ops[i];
1776 if (strcmp(torture_type, cur_ops->name) == 0)
1779 if (i == ARRAY_SIZE(torture_ops)) {
1780 pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
1782 pr_alert("rcu-torture types:");
1783 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1784 pr_alert(" %s", torture_ops[i]->name);
1789 if (cur_ops->fqs == NULL && fqs_duration != 0) {
1790 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
1796 if (nreaders >= 0) {
1797 nrealreaders = nreaders;
1799 nrealreaders = num_online_cpus() - 2 - nreaders;
1800 if (nrealreaders <= 0)
1803 rcu_torture_print_module_parms(cur_ops, "Start of test");
1805 /* Set up the freelist. */
1807 INIT_LIST_HEAD(&rcu_torture_freelist);
1808 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
1809 rcu_tortures[i].rtort_mbtest = 0;
1810 list_add_tail(&rcu_tortures[i].rtort_free,
1811 &rcu_torture_freelist);
1814 /* Initialize the statistics so that each run gets its own numbers. */
1816 rcu_torture_current = NULL;
1817 rcu_torture_current_version = 0;
1818 atomic_set(&n_rcu_torture_alloc, 0);
1819 atomic_set(&n_rcu_torture_alloc_fail, 0);
1820 atomic_set(&n_rcu_torture_free, 0);
1821 atomic_set(&n_rcu_torture_mberror, 0);
1822 atomic_set(&n_rcu_torture_error, 0);
1823 n_rcu_torture_barrier_error = 0;
1824 n_rcu_torture_boost_ktrerror = 0;
1825 n_rcu_torture_boost_rterror = 0;
1826 n_rcu_torture_boost_failure = 0;
1827 n_rcu_torture_boosts = 0;
1828 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1829 atomic_set(&rcu_torture_wcount[i], 0);
1830 for_each_possible_cpu(cpu) {
1831 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1832 per_cpu(rcu_torture_count, cpu)[i] = 0;
1833 per_cpu(rcu_torture_batch, cpu)[i] = 0;
1837 /* Start up the kthreads. */
1839 firsterr = torture_create_kthread(rcu_torture_writer, NULL,
1843 if (nfakewriters > 0) {
1844 fakewriter_tasks = kzalloc(nfakewriters *
1845 sizeof(fakewriter_tasks[0]),
1847 if (fakewriter_tasks == NULL) {
1848 VERBOSE_TOROUT_ERRSTRING("out of memory");
1853 for (i = 0; i < nfakewriters; i++) {
1854 firsterr = torture_create_kthread(rcu_torture_fakewriter,
1855 NULL, fakewriter_tasks[i]);
1859 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
1861 if (reader_tasks == NULL) {
1862 VERBOSE_TOROUT_ERRSTRING("out of memory");
1866 for (i = 0; i < nrealreaders; i++) {
1867 firsterr = torture_create_kthread(rcu_torture_reader, NULL,
1872 if (stat_interval > 0) {
1873 firsterr = torture_create_kthread(rcu_torture_stats, NULL,
1878 if (test_no_idle_hz && shuffle_interval > 0) {
1879 firsterr = torture_shuffle_init(shuffle_interval * HZ);
1886 firsterr = torture_stutter_init(stutter * HZ);
1890 if (fqs_duration < 0)
1893 /* Create the fqs thread */
1894 firsterr = torture_create_kthread(rcu_torture_fqs, NULL,
1899 if (test_boost_interval < 1)
1900 test_boost_interval = 1;
1901 if (test_boost_duration < 2)
1902 test_boost_duration = 2;
1903 if ((test_boost == 1 && cur_ops->can_boost) ||
1906 boost_starttime = jiffies + test_boost_interval * HZ;
1908 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE",
1909 rcutorture_booster_init,
1910 rcutorture_booster_cleanup);
1913 rcutor_hp = firsterr;
1915 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup);
1918 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ);
1921 firsterr = rcu_torture_stall_init();
1924 firsterr = rcu_torture_barrier_init();
1928 rcu_test_debug_objects();
1929 if (cbflood_n_burst > 0) {
1930 /* Create the cbflood threads */
1931 ncbflooders = (num_online_cpus() + 3) / 4;
1932 cbflood_task = kcalloc(ncbflooders, sizeof(*cbflood_task),
1934 if (!cbflood_task) {
1935 VERBOSE_TOROUT_ERRSTRING("out of memory");
1939 for (i = 0; i < ncbflooders; i++) {
1940 firsterr = torture_create_kthread(rcu_torture_cbflood,
1947 rcutorture_record_test_transition();
1953 rcu_torture_cleanup();
1957 module_init(rcu_torture_init);
1958 module_exit(rcu_torture_cleanup);