1 // SPDX-License-Identifier: GPL-2.0+
3 // Scalability test comparing RCU vs other mechanisms
4 // for acquiring references on objects.
6 // Copyright (C) Google, 2020.
8 // Author: Joel Fernandes <joel@joelfernandes.org>
10 #define pr_fmt(fmt) fmt
12 #include <linux/atomic.h>
13 #include <linux/bitops.h>
14 #include <linux/completion.h>
15 #include <linux/cpu.h>
16 #include <linux/delay.h>
17 #include <linux/err.h>
18 #include <linux/init.h>
19 #include <linux/interrupt.h>
20 #include <linux/kthread.h>
21 #include <linux/kernel.h>
23 #include <linux/module.h>
24 #include <linux/moduleparam.h>
25 #include <linux/notifier.h>
26 #include <linux/percpu.h>
27 #include <linux/rcupdate.h>
28 #include <linux/rcupdate_trace.h>
29 #include <linux/reboot.h>
30 #include <linux/sched.h>
31 #include <linux/spinlock.h>
32 #include <linux/smp.h>
33 #include <linux/stat.h>
34 #include <linux/srcu.h>
35 #include <linux/slab.h>
36 #include <linux/torture.h>
37 #include <linux/types.h>
41 #define SCALE_FLAG "-ref-scale: "
43 #define SCALEOUT(s, x...) \
44 pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
46 #define VERBOSE_SCALEOUT(s, x...) \
49 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
52 static atomic_t verbose_batch_ctr;
54 #define VERBOSE_SCALEOUT_BATCH(s, x...) \
57 (verbose_batched <= 0 || \
58 !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \
59 schedule_timeout_uninterruptible(1); \
60 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
64 #define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x)
66 MODULE_LICENSE("GPL");
67 MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
69 static char *scale_type = "rcu";
70 module_param(scale_type, charp, 0444);
71 MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
73 torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
74 torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
76 // Wait until there are multiple CPUs before starting test.
77 torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
78 "Holdoff time before test start (s)");
79 // Number of loops per experiment, all readers execute operations concurrently.
80 torture_param(long, loops, 10000, "Number of loops per experiment.");
81 // Number of readers, with -1 defaulting to about 75% of the CPUs.
82 torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
84 torture_param(int, nruns, 30, "Number of experiments to run.");
85 // Reader delay in nanoseconds, 0 for no delay.
86 torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
89 # define REFSCALE_SHUTDOWN 0
91 # define REFSCALE_SHUTDOWN 1
94 torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
95 "Shutdown at end of scalability tests.");
98 struct task_struct *task;
100 wait_queue_head_t wq;
101 u64 last_duration_ns;
104 static struct task_struct *shutdown_task;
105 static wait_queue_head_t shutdown_wq;
107 static struct task_struct *main_task;
108 static wait_queue_head_t main_wq;
109 static int shutdown_start;
111 static struct reader_task *reader_tasks;
113 // Number of readers that are part of the current experiment.
114 static atomic_t nreaders_exp;
116 // Use to wait for all threads to start.
117 static atomic_t n_init;
118 static atomic_t n_started;
119 static atomic_t n_warmedup;
120 static atomic_t n_cooleddown;
122 // Track which experiment is currently running.
125 // Operations vector for selecting different types of tests.
126 struct ref_scale_ops {
128 void (*cleanup)(void);
129 void (*readsection)(const int nloops);
130 void (*delaysection)(const int nloops, const int udl, const int ndl);
134 static struct ref_scale_ops *cur_ops;
136 static void un_delay(const int udl, const int ndl)
144 static void ref_rcu_read_section(const int nloops)
148 for (i = nloops; i >= 0; i--) {
154 static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
158 for (i = nloops; i >= 0; i--) {
165 static void rcu_sync_scale_init(void)
169 static struct ref_scale_ops rcu_ops = {
170 .init = rcu_sync_scale_init,
171 .readsection = ref_rcu_read_section,
172 .delaysection = ref_rcu_delay_section,
176 // Definitions for SRCU ref scale testing.
177 DEFINE_STATIC_SRCU(srcu_refctl_scale);
178 static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
180 static void srcu_ref_scale_read_section(const int nloops)
185 for (i = nloops; i >= 0; i--) {
186 idx = srcu_read_lock(srcu_ctlp);
187 srcu_read_unlock(srcu_ctlp, idx);
191 static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
196 for (i = nloops; i >= 0; i--) {
197 idx = srcu_read_lock(srcu_ctlp);
199 srcu_read_unlock(srcu_ctlp, idx);
203 static struct ref_scale_ops srcu_ops = {
204 .init = rcu_sync_scale_init,
205 .readsection = srcu_ref_scale_read_section,
206 .delaysection = srcu_ref_scale_delay_section,
210 // Definitions for RCU Tasks ref scale testing: Empty read markers.
211 // These definitions also work for RCU Rude readers.
212 static void rcu_tasks_ref_scale_read_section(const int nloops)
216 for (i = nloops; i >= 0; i--)
220 static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
224 for (i = nloops; i >= 0; i--)
228 static struct ref_scale_ops rcu_tasks_ops = {
229 .init = rcu_sync_scale_init,
230 .readsection = rcu_tasks_ref_scale_read_section,
231 .delaysection = rcu_tasks_ref_scale_delay_section,
235 // Definitions for RCU Tasks Trace ref scale testing.
236 static void rcu_trace_ref_scale_read_section(const int nloops)
240 for (i = nloops; i >= 0; i--) {
241 rcu_read_lock_trace();
242 rcu_read_unlock_trace();
246 static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
250 for (i = nloops; i >= 0; i--) {
251 rcu_read_lock_trace();
253 rcu_read_unlock_trace();
257 static struct ref_scale_ops rcu_trace_ops = {
258 .init = rcu_sync_scale_init,
259 .readsection = rcu_trace_ref_scale_read_section,
260 .delaysection = rcu_trace_ref_scale_delay_section,
264 // Definitions for reference count
265 static atomic_t refcnt;
267 static void ref_refcnt_section(const int nloops)
271 for (i = nloops; i >= 0; i--) {
277 static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
281 for (i = nloops; i >= 0; i--) {
288 static struct ref_scale_ops refcnt_ops = {
289 .init = rcu_sync_scale_init,
290 .readsection = ref_refcnt_section,
291 .delaysection = ref_refcnt_delay_section,
295 // Definitions for rwlock
296 static rwlock_t test_rwlock;
298 static void ref_rwlock_init(void)
300 rwlock_init(&test_rwlock);
303 static void ref_rwlock_section(const int nloops)
307 for (i = nloops; i >= 0; i--) {
308 read_lock(&test_rwlock);
309 read_unlock(&test_rwlock);
313 static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
317 for (i = nloops; i >= 0; i--) {
318 read_lock(&test_rwlock);
320 read_unlock(&test_rwlock);
324 static struct ref_scale_ops rwlock_ops = {
325 .init = ref_rwlock_init,
326 .readsection = ref_rwlock_section,
327 .delaysection = ref_rwlock_delay_section,
331 // Definitions for rwsem
332 static struct rw_semaphore test_rwsem;
334 static void ref_rwsem_init(void)
336 init_rwsem(&test_rwsem);
339 static void ref_rwsem_section(const int nloops)
343 for (i = nloops; i >= 0; i--) {
344 down_read(&test_rwsem);
345 up_read(&test_rwsem);
349 static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
353 for (i = nloops; i >= 0; i--) {
354 down_read(&test_rwsem);
356 up_read(&test_rwsem);
360 static struct ref_scale_ops rwsem_ops = {
361 .init = ref_rwsem_init,
362 .readsection = ref_rwsem_section,
363 .delaysection = ref_rwsem_delay_section,
367 // Definitions for global spinlock
368 static DEFINE_SPINLOCK(test_lock);
370 static void ref_lock_section(const int nloops)
375 for (i = nloops; i >= 0; i--) {
376 spin_lock(&test_lock);
377 spin_unlock(&test_lock);
382 static void ref_lock_delay_section(const int nloops, const int udl, const int ndl)
387 for (i = nloops; i >= 0; i--) {
388 spin_lock(&test_lock);
390 spin_unlock(&test_lock);
395 static struct ref_scale_ops lock_ops = {
396 .readsection = ref_lock_section,
397 .delaysection = ref_lock_delay_section,
401 // Definitions for global irq-save spinlock
403 static void ref_lock_irq_section(const int nloops)
409 for (i = nloops; i >= 0; i--) {
410 spin_lock_irqsave(&test_lock, flags);
411 spin_unlock_irqrestore(&test_lock, flags);
416 static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl)
422 for (i = nloops; i >= 0; i--) {
423 spin_lock_irqsave(&test_lock, flags);
425 spin_unlock_irqrestore(&test_lock, flags);
430 static struct ref_scale_ops lock_irq_ops = {
431 .readsection = ref_lock_irq_section,
432 .delaysection = ref_lock_irq_delay_section,
436 // Definitions acquire-release.
437 static DEFINE_PER_CPU(unsigned long, test_acqrel);
439 static void ref_acqrel_section(const int nloops)
445 for (i = nloops; i >= 0; i--) {
446 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
447 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
452 static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl)
458 for (i = nloops; i >= 0; i--) {
459 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
461 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
466 static struct ref_scale_ops acqrel_ops = {
467 .readsection = ref_acqrel_section,
468 .delaysection = ref_acqrel_delay_section,
472 static volatile u64 stopopts;
474 static void ref_clock_section(const int nloops)
480 for (i = nloops; i >= 0; i--)
481 x += ktime_get_real_fast_ns();
486 static void ref_clock_delay_section(const int nloops, const int udl, const int ndl)
492 for (i = nloops; i >= 0; i--) {
493 x += ktime_get_real_fast_ns();
500 static struct ref_scale_ops clock_ops = {
501 .readsection = ref_clock_section,
502 .delaysection = ref_clock_delay_section,
506 static void rcu_scale_one_reader(void)
509 cur_ops->readsection(loops);
511 cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
514 // Reader kthread. Repeatedly does empty RCU read-side
515 // critical section, minimizing update-side interference.
517 ref_scale_reader(void *arg)
521 struct reader_task *rt = &(reader_tasks[me]);
525 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me);
526 WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)));
527 set_user_nice(current, MAX_NICE);
530 schedule_timeout_interruptible(holdoff * HZ);
532 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id());
534 // Wait for signal that this reader can start.
535 wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
536 torture_must_stop());
538 if (torture_must_stop())
541 // Make sure that the CPU is affinitized appropriately during testing.
542 WARN_ON_ONCE(raw_smp_processor_id() != me);
544 WRITE_ONCE(rt->start_reader, 0);
545 if (!atomic_dec_return(&n_started))
546 while (atomic_read_acquire(&n_started))
549 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx);
552 // To reduce noise, do an initial cache-warming invocation, check
553 // in, and then keep warming until everyone has checked in.
554 rcu_scale_one_reader();
555 if (!atomic_dec_return(&n_warmedup))
556 while (atomic_read_acquire(&n_warmedup))
557 rcu_scale_one_reader();
558 // Also keep interrupts disabled. This also has the effect
559 // of preventing entries into slow path for rcu_read_unlock().
560 local_irq_save(flags);
561 start = ktime_get_mono_fast_ns();
563 rcu_scale_one_reader();
565 duration = ktime_get_mono_fast_ns() - start;
566 local_irq_restore(flags);
568 rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
569 // To reduce runtime-skew noise, do maintain-load invocations until
571 if (!atomic_dec_return(&n_cooleddown))
572 while (atomic_read_acquire(&n_cooleddown))
573 rcu_scale_one_reader();
575 if (atomic_dec_and_test(&nreaders_exp))
578 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
579 me, exp_idx, atomic_read(&nreaders_exp));
581 if (!torture_must_stop())
584 torture_kthread_stopping("ref_scale_reader");
588 static void reset_readers(void)
591 struct reader_task *rt;
593 for (i = 0; i < nreaders; i++) {
594 rt = &(reader_tasks[i]);
596 rt->last_duration_ns = 0;
600 // Print the results of each reader and return the sum of all their durations.
601 static u64 process_durations(int n)
604 struct reader_task *rt;
609 buf = kmalloc(800 + 64, GFP_KERNEL);
613 sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
616 for (i = 0; i < n && !torture_must_stop(); i++) {
617 rt = &(reader_tasks[i]);
618 sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns);
622 if (strlen(buf) >= 800) {
628 sum += rt->last_duration_ns;
630 pr_alert("%s\n", buf);
636 // The main_func is the main orchestrator, it performs a bunch of
637 // experiments. For every experiment, it orders all the readers
638 // involved to start and waits for them to finish the experiment. It
639 // then reads their timestamps and starts the next experiment. Each
640 // experiment progresses from 1 concurrent reader to N of them at which
641 // point all the timestamps are printed.
642 static int main_func(void *arg)
649 set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
650 set_user_nice(current, MAX_NICE);
652 VERBOSE_SCALEOUT("main_func task started");
653 result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);
654 buf = kzalloc(800 + 64, GFP_KERNEL);
655 if (!result_avg || !buf) {
656 SCALEOUT_ERRSTRING("out of memory");
660 schedule_timeout_interruptible(holdoff * HZ);
662 // Wait for all threads to start.
664 while (atomic_read(&n_init) < nreaders + 1)
665 schedule_timeout_uninterruptible(1);
667 // Start exp readers up per experiment
668 for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
669 if (torture_must_stop())
673 atomic_set(&nreaders_exp, nreaders);
674 atomic_set(&n_started, nreaders);
675 atomic_set(&n_warmedup, nreaders);
676 atomic_set(&n_cooleddown, nreaders);
680 for (r = 0; r < nreaders; r++) {
681 smp_store_release(&reader_tasks[r].start_reader, 1);
682 wake_up(&reader_tasks[r].wq);
685 VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
689 !atomic_read(&nreaders_exp) || torture_must_stop());
691 VERBOSE_SCALEOUT("main_func: experiment ended");
693 if (torture_must_stop())
696 result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops);
699 // Print the average of all experiments
700 SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
702 pr_alert("Runs\tTime(ns)\n");
703 for (exp = 0; exp < nruns; exp++) {
707 avg = div_u64_rem(result_avg[exp], 1000, &rem);
708 sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);
710 if (strlen(buf) >= 800) {
719 // This will shutdown everything including us.
722 wake_up(&shutdown_wq);
725 // Wait for torture to stop us
726 while (!torture_must_stop())
727 schedule_timeout_uninterruptible(1);
730 torture_kthread_stopping("main_func");
737 ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag)
739 pr_alert("%s" SCALE_FLAG
740 "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
741 verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay);
745 ref_scale_cleanup(void)
749 if (torture_cleanup_begin())
753 torture_cleanup_end();
758 for (i = 0; i < nreaders; i++)
759 torture_stop_kthread("ref_scale_reader",
760 reader_tasks[i].task);
764 torture_stop_kthread("main_task", main_task);
767 // Do scale-type-specific cleanup operations.
768 if (cur_ops->cleanup != NULL)
771 torture_cleanup_end();
774 // Shutdown kthread. Just waits to be awakened, then shuts down system.
776 ref_scale_shutdown(void *arg)
778 wait_event(shutdown_wq, shutdown_start);
780 smp_mb(); // Wake before output.
792 static struct ref_scale_ops *scale_ops[] = {
793 &rcu_ops, &srcu_ops, &rcu_trace_ops, &rcu_tasks_ops, &refcnt_ops, &rwlock_ops,
794 &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops,
797 if (!torture_init_begin(scale_type, verbose))
800 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
801 cur_ops = scale_ops[i];
802 if (strcmp(scale_type, cur_ops->name) == 0)
805 if (i == ARRAY_SIZE(scale_ops)) {
806 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
807 pr_alert("rcu-scale types:");
808 for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
809 pr_cont(" %s", scale_ops[i]->name);
818 ref_scale_print_module_parms(cur_ops, "Start of test");
822 init_waitqueue_head(&shutdown_wq);
823 firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
825 if (torture_init_error(firsterr))
827 schedule_timeout_uninterruptible(1);
830 // Reader tasks (default to ~75% of online CPUs).
832 nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
833 if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops))
835 if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders))
837 if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns))
839 reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
842 SCALEOUT_ERRSTRING("out of memory");
847 VERBOSE_SCALEOUT("Starting %d reader threads", nreaders);
849 for (i = 0; i < nreaders; i++) {
850 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
851 reader_tasks[i].task);
852 if (torture_init_error(firsterr))
855 init_waitqueue_head(&(reader_tasks[i].wq));
859 init_waitqueue_head(&main_wq);
860 firsterr = torture_create_kthread(main_func, NULL, main_task);
861 if (torture_init_error(firsterr))
871 WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
877 module_init(ref_scale_init);
878 module_exit(ref_scale_cleanup);