kernel/locking/locktorture.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Module-based torture test facility for locking
   4  *
   5  * Copyright (C) IBM Corporation, 2014
   6  *
   7  * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
   8  *          Davidlohr Bueso <dave@stgolabs.net>
   9  *      Based on kernel/rcu/torture.c.
  10  */
  11
  12 #define pr_fmt(fmt) fmt
  13
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/kthread.h>
  17 #include <linux/sched/rt.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/mutex.h>
  20 #include <linux/rwsem.h>
  21 #include <linux/smp.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/sched.h>
  24 #include <uapi/linux/sched/types.h>
  25 #include <linux/rtmutex.h>
  26 #include <linux/atomic.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/delay.h>
  29 #include <linux/slab.h>
  30 #include <linux/torture.h>
  31 #include <linux/reboot.h>
  32
  33 MODULE_LICENSE("GPL");
  34 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
  35
  36 torture_param(int, nwriters_stress, -1,
  37              "Number of write-locking stress-test threads");
  38 torture_param(int, nreaders_stress, -1,
  39              "Number of read-locking stress-test threads");
  40 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
  41 torture_param(int, onoff_interval, 0,
  42              "Time between CPU hotplugs (s), 0=disable");
  43 torture_param(int, shuffle_interval, 3,
  44              "Number of jiffies between shuffles, 0=disable");
  45 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
  46 torture_param(int, stat_interval, 60,
  47              "Number of seconds between stats printk()s");
  48 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
  49 torture_param(int, verbose, 1,
  50              "Enable verbose debugging printk()s");
  51
  52 static char *torture_type = "spin_lock";
  53 module_param(torture_type, charp, 0444);
  54 MODULE_PARM_DESC(torture_type,
  55                  "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
  56
  57 static struct task_struct *stats_task;
  58 static struct task_struct **writer_tasks;
  59 static struct task_struct **reader_tasks;
  60
  61 static bool lock_is_write_held;
  62 static bool lock_is_read_held;
  63 static unsigned long last_lock_release;
  64
  65 struct lock_stress_stats {
  66         long n_lock_fail;
  67         long n_lock_acquired;
  68 };
  69
  70 /* Forward reference. */
  71 static void lock_torture_cleanup(void);
  72
  73 /*
  74  * Operations vector for selecting different types of tests.
  75  */
  76 struct lock_torture_ops {
  77         void (*init)(void);
  78         void (*exit)(void);
  79         int (*writelock)(int tid);
  80         void (*write_delay)(struct torture_random_state *trsp);
  81         void (*task_boost)(struct torture_random_state *trsp);
  82         void (*writeunlock)(int tid);
  83         int (*readlock)(int tid);
  84         void (*read_delay)(struct torture_random_state *trsp);
  85         void (*readunlock)(int tid);
  86
  87         unsigned long flags; /* for irq spinlocks */
  88         const char *name;
  89 };
  90
  91 struct lock_torture_cxt {
  92         int nrealwriters_stress;
  93         int nrealreaders_stress;
  94         bool debug_lock;
  95         bool init_called;
  96         atomic_t n_lock_torture_errors;
  97         struct lock_torture_ops *cur_ops;
  98         struct lock_stress_stats *lwsa; /* writer statistics */
  99         struct lock_stress_stats *lrsa; /* reader statistics */
 100 };
 101 static struct lock_torture_cxt cxt = { 0, 0, false, false,
 102                                        ATOMIC_INIT(0),
 103                                        NULL, NULL};
 104 /*
 105  * Definitions for lock torture testing.
 106  */
 107
 108 static int torture_lock_busted_write_lock(int tid __maybe_unused)
 109 {
 110         return 0;  /* BUGGY, do not use in real life!!! */
 111 }
 112
 113 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
 114 {
 115         const unsigned long longdelay_ms = 100;
 116
 117         /* We want a long delay occasionally to force massive contention.  */
 118         if (!(torture_random(trsp) %
 119               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 120                 mdelay(longdelay_ms);
 121         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 122                 torture_preempt_schedule();  /* Allow test to be preempted. */
 123 }
 124
 125 static void torture_lock_busted_write_unlock(int tid __maybe_unused)
 126 {
 127           /* BUGGY, do not use in real life!!! */
 128 }
 129
 130 static void torture_boost_dummy(struct torture_random_state *trsp)
 131 {
 132         /* Only rtmutexes care about priority */
 133 }
 134
 135 static struct lock_torture_ops lock_busted_ops = {
 136         .writelock      = torture_lock_busted_write_lock,
 137         .write_delay    = torture_lock_busted_write_delay,
 138         .task_boost     = torture_boost_dummy,
 139         .writeunlock    = torture_lock_busted_write_unlock,
 140         .readlock       = NULL,
 141         .read_delay     = NULL,
 142         .readunlock     = NULL,
 143         .name           = "lock_busted"
 144 };
 145
 146 static DEFINE_SPINLOCK(torture_spinlock);
 147
 148 static int torture_spin_lock_write_lock(int tid __maybe_unused)
 149 __acquires(torture_spinlock)
 150 {
 151         spin_lock(&torture_spinlock);
 152         return 0;
 153 }
 154
 155 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
 156 {
 157         const unsigned long shortdelay_us = 2;
 158         const unsigned long longdelay_ms = 100;
 159
 160         /* We want a short delay mostly to emulate likely code, and
 161          * we want a long delay occasionally to force massive contention.
 162          */
 163         if (!(torture_random(trsp) %
 164               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 165                 mdelay(longdelay_ms);
 166         if (!(torture_random(trsp) %
 167               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 168                 udelay(shortdelay_us);
 169         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 170                 torture_preempt_schedule();  /* Allow test to be preempted. */
 171 }
 172
 173 static void torture_spin_lock_write_unlock(int tid __maybe_unused)
 174 __releases(torture_spinlock)
 175 {
 176         spin_unlock(&torture_spinlock);
 177 }
 178
 179 static struct lock_torture_ops spin_lock_ops = {
 180         .writelock      = torture_spin_lock_write_lock,
 181         .write_delay    = torture_spin_lock_write_delay,
 182         .task_boost     = torture_boost_dummy,
 183         .writeunlock    = torture_spin_lock_write_unlock,
 184         .readlock       = NULL,
 185         .read_delay     = NULL,
 186         .readunlock     = NULL,
 187         .name           = "spin_lock"
 188 };
 189
 190 static int torture_spin_lock_write_lock_irq(int tid __maybe_unused)
 191 __acquires(torture_spinlock)
 192 {
 193         unsigned long flags;
 194
 195         spin_lock_irqsave(&torture_spinlock, flags);
 196         cxt.cur_ops->flags = flags;
 197         return 0;
 198 }
 199
 200 static void torture_lock_spin_write_unlock_irq(int tid __maybe_unused)
 201 __releases(torture_spinlock)
 202 {
 203         spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
 204 }
 205
 206 static struct lock_torture_ops spin_lock_irq_ops = {
 207         .writelock      = torture_spin_lock_write_lock_irq,
 208         .write_delay    = torture_spin_lock_write_delay,
 209         .task_boost     = torture_boost_dummy,
 210         .writeunlock    = torture_lock_spin_write_unlock_irq,
 211         .readlock       = NULL,
 212         .read_delay     = NULL,
 213         .readunlock     = NULL,
 214         .name           = "spin_lock_irq"
 215 };
 216
 217 static DEFINE_RWLOCK(torture_rwlock);
 218
 219 static int torture_rwlock_write_lock(int tid __maybe_unused)
 220 __acquires(torture_rwlock)
 221 {
 222         write_lock(&torture_rwlock);
 223         return 0;
 224 }
 225
 226 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
 227 {
 228         const unsigned long shortdelay_us = 2;
 229         const unsigned long longdelay_ms = 100;
 230
 231         /* We want a short delay mostly to emulate likely code, and
 232          * we want a long delay occasionally to force massive contention.
 233          */
 234         if (!(torture_random(trsp) %
 235               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 236                 mdelay(longdelay_ms);
 237         else
 238                 udelay(shortdelay_us);
 239 }
 240
 241 static void torture_rwlock_write_unlock(int tid __maybe_unused)
 242 __releases(torture_rwlock)
 243 {
 244         write_unlock(&torture_rwlock);
 245 }
 246
 247 static int torture_rwlock_read_lock(int tid __maybe_unused)
 248 __acquires(torture_rwlock)
 249 {
 250         read_lock(&torture_rwlock);
 251         return 0;
 252 }
 253
 254 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
 255 {
 256         const unsigned long shortdelay_us = 10;
 257         const unsigned long longdelay_ms = 100;
 258
 259         /* We want a short delay mostly to emulate likely code, and
 260          * we want a long delay occasionally to force massive contention.
 261          */
 262         if (!(torture_random(trsp) %
 263               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 264                 mdelay(longdelay_ms);
 265         else
 266                 udelay(shortdelay_us);
 267 }
 268
 269 static void torture_rwlock_read_unlock(int tid __maybe_unused)
 270 __releases(torture_rwlock)
 271 {
 272         read_unlock(&torture_rwlock);
 273 }
 274
 275 static struct lock_torture_ops rw_lock_ops = {
 276         .writelock      = torture_rwlock_write_lock,
 277         .write_delay    = torture_rwlock_write_delay,
 278         .task_boost     = torture_boost_dummy,
 279         .writeunlock    = torture_rwlock_write_unlock,
 280         .readlock       = torture_rwlock_read_lock,
 281         .read_delay     = torture_rwlock_read_delay,
 282         .readunlock     = torture_rwlock_read_unlock,
 283         .name           = "rw_lock"
 284 };
 285
 286 static int torture_rwlock_write_lock_irq(int tid __maybe_unused)
 287 __acquires(torture_rwlock)
 288 {
 289         unsigned long flags;
 290
 291         write_lock_irqsave(&torture_rwlock, flags);
 292         cxt.cur_ops->flags = flags;
 293         return 0;
 294 }
 295
 296 static void torture_rwlock_write_unlock_irq(int tid __maybe_unused)
 297 __releases(torture_rwlock)
 298 {
 299         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 300 }
 301
 302 static int torture_rwlock_read_lock_irq(int tid __maybe_unused)
 303 __acquires(torture_rwlock)
 304 {
 305         unsigned long flags;
 306
 307         read_lock_irqsave(&torture_rwlock, flags);
 308         cxt.cur_ops->flags = flags;
 309         return 0;
 310 }
 311
 312 static void torture_rwlock_read_unlock_irq(int tid __maybe_unused)
 313 __releases(torture_rwlock)
 314 {
 315         read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 316 }
 317
 318 static struct lock_torture_ops rw_lock_irq_ops = {
 319         .writelock      = torture_rwlock_write_lock_irq,
 320         .write_delay    = torture_rwlock_write_delay,
 321         .task_boost     = torture_boost_dummy,
 322         .writeunlock    = torture_rwlock_write_unlock_irq,
 323         .readlock       = torture_rwlock_read_lock_irq,
 324         .read_delay     = torture_rwlock_read_delay,
 325         .readunlock     = torture_rwlock_read_unlock_irq,
 326         .name           = "rw_lock_irq"
 327 };
 328
 329 static DEFINE_MUTEX(torture_mutex);
 330
 331 static int torture_mutex_lock(int tid __maybe_unused)
 332 __acquires(torture_mutex)
 333 {
 334         mutex_lock(&torture_mutex);
 335         return 0;
 336 }
 337
 338 static void torture_mutex_delay(struct torture_random_state *trsp)
 339 {
 340         const unsigned long longdelay_ms = 100;
 341
 342         /* We want a long delay occasionally to force massive contention.  */
 343         if (!(torture_random(trsp) %
 344               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 345                 mdelay(longdelay_ms * 5);
 346         else
 347                 mdelay(longdelay_ms / 5);
 348         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 349                 torture_preempt_schedule();  /* Allow test to be preempted. */
 350 }
 351
 352 static void torture_mutex_unlock(int tid __maybe_unused)
 353 __releases(torture_mutex)
 354 {
 355         mutex_unlock(&torture_mutex);
 356 }
 357
 358 static struct lock_torture_ops mutex_lock_ops = {
 359         .writelock      = torture_mutex_lock,
 360         .write_delay    = torture_mutex_delay,
 361         .task_boost     = torture_boost_dummy,
 362         .writeunlock    = torture_mutex_unlock,
 363         .readlock       = NULL,
 364         .read_delay     = NULL,
 365         .readunlock     = NULL,
 366         .name           = "mutex_lock"
 367 };
 368
 369 #include <linux/ww_mutex.h>
 370 /*
 371  * The torture ww_mutexes should belong to the same lock class as
 372  * torture_ww_class to avoid lockdep problem. The ww_mutex_init()
 373  * function is called for initialization to ensure that.
 374  */
 375 static DEFINE_WD_CLASS(torture_ww_class);
 376 static struct ww_mutex torture_ww_mutex_0, torture_ww_mutex_1, torture_ww_mutex_2;
 377 static struct ww_acquire_ctx *ww_acquire_ctxs;
 378
 379 static void torture_ww_mutex_init(void)
 380 {
 381         ww_mutex_init(&torture_ww_mutex_0, &torture_ww_class);
 382         ww_mutex_init(&torture_ww_mutex_1, &torture_ww_class);
 383         ww_mutex_init(&torture_ww_mutex_2, &torture_ww_class);
 384
 385         ww_acquire_ctxs = kmalloc_array(cxt.nrealwriters_stress,
 386                                         sizeof(*ww_acquire_ctxs),
 387                                         GFP_KERNEL);
 388         if (!ww_acquire_ctxs)
 389                 VERBOSE_TOROUT_STRING("ww_acquire_ctx: Out of memory");
 390 }
 391
 392 static void torture_ww_mutex_exit(void)
 393 {
 394         kfree(ww_acquire_ctxs);
 395 }
 396
 397 static int torture_ww_mutex_lock(int tid)
 398 __acquires(torture_ww_mutex_0)
 399 __acquires(torture_ww_mutex_1)
 400 __acquires(torture_ww_mutex_2)
 401 {
 402         LIST_HEAD(list);
 403         struct reorder_lock {
 404                 struct list_head link;
 405                 struct ww_mutex *lock;
 406         } locks[3], *ll, *ln;
 407         struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid];
 408
 409         locks[0].lock = &torture_ww_mutex_0;
 410         list_add(&locks[0].link, &list);
 411
 412         locks[1].lock = &torture_ww_mutex_1;
 413         list_add(&locks[1].link, &list);
 414
 415         locks[2].lock = &torture_ww_mutex_2;
 416         list_add(&locks[2].link, &list);
 417
 418         ww_acquire_init(ctx, &torture_ww_class);
 419
 420         list_for_each_entry(ll, &list, link) {
 421                 int err;
 422
 423                 err = ww_mutex_lock(ll->lock, ctx);
 424                 if (!err)
 425                         continue;
 426
 427                 ln = ll;
 428                 list_for_each_entry_continue_reverse(ln, &list, link)
 429                         ww_mutex_unlock(ln->lock);
 430
 431                 if (err != -EDEADLK)
 432                         return err;
 433
 434                 ww_mutex_lock_slow(ll->lock, ctx);
 435                 list_move(&ll->link, &list);
 436         }
 437
 438         return 0;
 439 }
 440
 441 static void torture_ww_mutex_unlock(int tid)
 442 __releases(torture_ww_mutex_0)
 443 __releases(torture_ww_mutex_1)
 444 __releases(torture_ww_mutex_2)
 445 {
 446         struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid];
 447
 448         ww_mutex_unlock(&torture_ww_mutex_0);
 449         ww_mutex_unlock(&torture_ww_mutex_1);
 450         ww_mutex_unlock(&torture_ww_mutex_2);
 451         ww_acquire_fini(ctx);
 452 }
 453
 454 static struct lock_torture_ops ww_mutex_lock_ops = {
 455         .init           = torture_ww_mutex_init,
 456         .exit           = torture_ww_mutex_exit,
 457         .writelock      = torture_ww_mutex_lock,
 458         .write_delay    = torture_mutex_delay,
 459         .task_boost     = torture_boost_dummy,
 460         .writeunlock    = torture_ww_mutex_unlock,
 461         .readlock       = NULL,
 462         .read_delay     = NULL,
 463         .readunlock     = NULL,
 464         .name           = "ww_mutex_lock"
 465 };
 466
 467 #ifdef CONFIG_RT_MUTEXES
 468 static DEFINE_RT_MUTEX(torture_rtmutex);
 469
 470 static int torture_rtmutex_lock(int tid __maybe_unused)
 471 __acquires(torture_rtmutex)
 472 {
 473         rt_mutex_lock(&torture_rtmutex);
 474         return 0;
 475 }
 476
 477 static void torture_rtmutex_boost(struct torture_random_state *trsp)
 478 {
 479         const unsigned int factor = 50000; /* yes, quite arbitrary */
 480
 481         if (!rt_task(current)) {
 482                 /*
 483                  * Boost priority once every ~50k operations. When the
 484                  * task tries to take the lock, the rtmutex it will account
 485                  * for the new priority, and do any corresponding pi-dance.
 486                  */
 487                 if (trsp && !(torture_random(trsp) %
 488                               (cxt.nrealwriters_stress * factor))) {
 489                         sched_set_fifo(current);
 490                 } else /* common case, do nothing */
 491                         return;
 492         } else {
 493                 /*
 494                  * The task will remain boosted for another ~500k operations,
 495                  * then restored back to its original prio, and so forth.
 496                  *
 497                  * When @trsp is nil, we want to force-reset the task for
 498                  * stopping the kthread.
 499                  */
 500                 if (!trsp || !(torture_random(trsp) %
 501                                (cxt.nrealwriters_stress * factor * 2))) {
 502                         sched_set_normal(current, 0);
 503                 } else /* common case, do nothing */
 504                         return;
 505         }
 506 }
 507
 508 static void torture_rtmutex_delay(struct torture_random_state *trsp)
 509 {
 510         const unsigned long shortdelay_us = 2;
 511         const unsigned long longdelay_ms = 100;
 512
 513         /*
 514          * We want a short delay mostly to emulate likely code, and
 515          * we want a long delay occasionally to force massive contention.
 516          */
 517         if (!(torture_random(trsp) %
 518               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 519                 mdelay(longdelay_ms);
 520         if (!(torture_random(trsp) %
 521               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 522                 udelay(shortdelay_us);
 523         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 524                 torture_preempt_schedule();  /* Allow test to be preempted. */
 525 }
 526
 527 static void torture_rtmutex_unlock(int tid __maybe_unused)
 528 __releases(torture_rtmutex)
 529 {
 530         rt_mutex_unlock(&torture_rtmutex);
 531 }
 532
 533 static struct lock_torture_ops rtmutex_lock_ops = {
 534         .writelock      = torture_rtmutex_lock,
 535         .write_delay    = torture_rtmutex_delay,
 536         .task_boost     = torture_rtmutex_boost,
 537         .writeunlock    = torture_rtmutex_unlock,
 538         .readlock       = NULL,
 539         .read_delay     = NULL,
 540         .readunlock     = NULL,
 541         .name           = "rtmutex_lock"
 542 };
 543 #endif
 544
 545 static DECLARE_RWSEM(torture_rwsem);
 546 static int torture_rwsem_down_write(int tid __maybe_unused)
 547 __acquires(torture_rwsem)
 548 {
 549         down_write(&torture_rwsem);
 550         return 0;
 551 }
 552
 553 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
 554 {
 555         const unsigned long longdelay_ms = 100;
 556
 557         /* We want a long delay occasionally to force massive contention.  */
 558         if (!(torture_random(trsp) %
 559               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 560                 mdelay(longdelay_ms * 10);
 561         else
 562                 mdelay(longdelay_ms / 10);
 563         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 564                 torture_preempt_schedule();  /* Allow test to be preempted. */
 565 }
 566
 567 static void torture_rwsem_up_write(int tid __maybe_unused)
 568 __releases(torture_rwsem)
 569 {
 570         up_write(&torture_rwsem);
 571 }
 572
 573 static int torture_rwsem_down_read(int tid __maybe_unused)
 574 __acquires(torture_rwsem)
 575 {
 576         down_read(&torture_rwsem);
 577         return 0;
 578 }
 579
 580 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
 581 {
 582         const unsigned long longdelay_ms = 100;
 583
 584         /* We want a long delay occasionally to force massive contention.  */
 585         if (!(torture_random(trsp) %
 586               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 587                 mdelay(longdelay_ms * 2);
 588         else
 589                 mdelay(longdelay_ms / 2);
 590         if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
 591                 torture_preempt_schedule();  /* Allow test to be preempted. */
 592 }
 593
 594 static void torture_rwsem_up_read(int tid __maybe_unused)
 595 __releases(torture_rwsem)
 596 {
 597         up_read(&torture_rwsem);
 598 }
 599
 600 static struct lock_torture_ops rwsem_lock_ops = {
 601         .writelock      = torture_rwsem_down_write,
 602         .write_delay    = torture_rwsem_write_delay,
 603         .task_boost     = torture_boost_dummy,
 604         .writeunlock    = torture_rwsem_up_write,
 605         .readlock       = torture_rwsem_down_read,
 606         .read_delay     = torture_rwsem_read_delay,
 607         .readunlock     = torture_rwsem_up_read,
 608         .name           = "rwsem_lock"
 609 };
 610
 611 #include <linux/percpu-rwsem.h>
 612 static struct percpu_rw_semaphore pcpu_rwsem;
 613
 614 static void torture_percpu_rwsem_init(void)
 615 {
 616         BUG_ON(percpu_init_rwsem(&pcpu_rwsem));
 617 }
 618
 619 static void torture_percpu_rwsem_exit(void)
 620 {
 621         percpu_free_rwsem(&pcpu_rwsem);
 622 }
 623
 624 static int torture_percpu_rwsem_down_write(int tid __maybe_unused)
 625 __acquires(pcpu_rwsem)
 626 {
 627         percpu_down_write(&pcpu_rwsem);
 628         return 0;
 629 }
 630
 631 static void torture_percpu_rwsem_up_write(int tid __maybe_unused)
 632 __releases(pcpu_rwsem)
 633 {
 634         percpu_up_write(&pcpu_rwsem);
 635 }
 636
 637 static int torture_percpu_rwsem_down_read(int tid __maybe_unused)
 638 __acquires(pcpu_rwsem)
 639 {
 640         percpu_down_read(&pcpu_rwsem);
 641         return 0;
 642 }
 643
 644 static void torture_percpu_rwsem_up_read(int tid __maybe_unused)
 645 __releases(pcpu_rwsem)
 646 {
 647         percpu_up_read(&pcpu_rwsem);
 648 }
 649
 650 static struct lock_torture_ops percpu_rwsem_lock_ops = {
 651         .init           = torture_percpu_rwsem_init,
 652         .exit           = torture_percpu_rwsem_exit,
 653         .writelock      = torture_percpu_rwsem_down_write,
 654         .write_delay    = torture_rwsem_write_delay,
 655         .task_boost     = torture_boost_dummy,
 656         .writeunlock    = torture_percpu_rwsem_up_write,
 657         .readlock       = torture_percpu_rwsem_down_read,
 658         .read_delay     = torture_rwsem_read_delay,
 659         .readunlock     = torture_percpu_rwsem_up_read,
 660         .name           = "percpu_rwsem_lock"
 661 };
 662
 663 /*
 664  * Lock torture writer kthread.  Repeatedly acquires and releases
 665  * the lock, checking for duplicate acquisitions.
 666  */
 667 static int lock_torture_writer(void *arg)
 668 {
 669         struct lock_stress_stats *lwsp = arg;
 670         int tid = lwsp - cxt.lwsa;
 671         DEFINE_TORTURE_RANDOM(rand);
 672
 673         VERBOSE_TOROUT_STRING("lock_torture_writer task started");
 674         set_user_nice(current, MAX_NICE);
 675
 676         do {
 677                 if ((torture_random(&rand) & 0xfffff) == 0)
 678                         schedule_timeout_uninterruptible(1);
 679
 680                 cxt.cur_ops->task_boost(&rand);
 681                 cxt.cur_ops->writelock(tid);
 682                 if (WARN_ON_ONCE(lock_is_write_held))
 683                         lwsp->n_lock_fail++;
 684                 lock_is_write_held = true;
 685                 if (WARN_ON_ONCE(lock_is_read_held))
 686                         lwsp->n_lock_fail++; /* rare, but... */
 687
 688                 lwsp->n_lock_acquired++;
 689                 cxt.cur_ops->write_delay(&rand);
 690                 lock_is_write_held = false;
 691                 WRITE_ONCE(last_lock_release, jiffies);
 692                 cxt.cur_ops->writeunlock(tid);
 693
 694                 stutter_wait("lock_torture_writer");
 695         } while (!torture_must_stop());
 696
 697         cxt.cur_ops->task_boost(NULL); /* reset prio */
 698         torture_kthread_stopping("lock_torture_writer");
 699         return 0;
 700 }
 701
 702 /*
 703  * Lock torture reader kthread.  Repeatedly acquires and releases
 704  * the reader lock.
 705  */
 706 static int lock_torture_reader(void *arg)
 707 {
 708         struct lock_stress_stats *lrsp = arg;
 709         int tid = lrsp - cxt.lrsa;
 710         DEFINE_TORTURE_RANDOM(rand);
 711
 712         VERBOSE_TOROUT_STRING("lock_torture_reader task started");
 713         set_user_nice(current, MAX_NICE);
 714
 715         do {
 716                 if ((torture_random(&rand) & 0xfffff) == 0)
 717                         schedule_timeout_uninterruptible(1);
 718
 719                 cxt.cur_ops->readlock(tid);
 720                 lock_is_read_held = true;
 721                 if (WARN_ON_ONCE(lock_is_write_held))
 722                         lrsp->n_lock_fail++; /* rare, but... */
 723
 724                 lrsp->n_lock_acquired++;
 725                 cxt.cur_ops->read_delay(&rand);
 726                 lock_is_read_held = false;
 727                 cxt.cur_ops->readunlock(tid);
 728
 729                 stutter_wait("lock_torture_reader");
 730         } while (!torture_must_stop());
 731         torture_kthread_stopping("lock_torture_reader");
 732         return 0;
 733 }
 734
 735 /*
 736  * Create an lock-torture-statistics message in the specified buffer.
 737  */
 738 static void __torture_print_stats(char *page,
 739                                   struct lock_stress_stats *statp, bool write)
 740 {
 741         bool fail = false;
 742         int i, n_stress;
 743         long max = 0, min = statp ? statp[0].n_lock_acquired : 0;
 744         long long sum = 0;
 745
 746         n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
 747         for (i = 0; i < n_stress; i++) {
 748                 if (statp[i].n_lock_fail)
 749                         fail = true;
 750                 sum += statp[i].n_lock_acquired;
 751                 if (max < statp[i].n_lock_acquired)
 752                         max = statp[i].n_lock_acquired;
 753                 if (min > statp[i].n_lock_acquired)
 754                         min = statp[i].n_lock_acquired;
 755         }
 756         page += sprintf(page,
 757                         "%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
 758                         write ? "Writes" : "Reads ",
 759                         sum, max, min,
 760                         !onoff_interval && max / 2 > min ? "???" : "",
 761                         fail, fail ? "!!!" : "");
 762         if (fail)
 763                 atomic_inc(&cxt.n_lock_torture_errors);
 764 }
 765
 766 /*
 767  * Print torture statistics.  Caller must ensure that there is only one
 768  * call to this function at a given time!!!  This is normally accomplished
 769  * by relying on the module system to only have one copy of the module
 770  * loaded, and then by giving the lock_torture_stats kthread full control
 771  * (or the init/cleanup functions when lock_torture_stats thread is not
 772  * running).
 773  */
 774 static void lock_torture_stats_print(void)
 775 {
 776         int size = cxt.nrealwriters_stress * 200 + 8192;
 777         char *buf;
 778
 779         if (cxt.cur_ops->readlock)
 780                 size += cxt.nrealreaders_stress * 200 + 8192;
 781
 782         buf = kmalloc(size, GFP_KERNEL);
 783         if (!buf) {
 784                 pr_err("lock_torture_stats_print: Out of memory, need: %d",
 785                        size);
 786                 return;
 787         }
 788
 789         __torture_print_stats(buf, cxt.lwsa, true);
 790         pr_alert("%s", buf);
 791         kfree(buf);
 792
 793         if (cxt.cur_ops->readlock) {
 794                 buf = kmalloc(size, GFP_KERNEL);
 795                 if (!buf) {
 796                         pr_err("lock_torture_stats_print: Out of memory, need: %d",
 797                                size);
 798                         return;
 799                 }
 800
 801                 __torture_print_stats(buf, cxt.lrsa, false);
 802                 pr_alert("%s", buf);
 803                 kfree(buf);
 804         }
 805 }
 806
 807 /*
 808  * Periodically prints torture statistics, if periodic statistics printing
 809  * was specified via the stat_interval module parameter.
 810  *
 811  * No need to worry about fullstop here, since this one doesn't reference
 812  * volatile state or register callbacks.
 813  */
 814 static int lock_torture_stats(void *arg)
 815 {
 816         VERBOSE_TOROUT_STRING("lock_torture_stats task started");
 817         do {
 818                 schedule_timeout_interruptible(stat_interval * HZ);
 819                 lock_torture_stats_print();
 820                 torture_shutdown_absorb("lock_torture_stats");
 821         } while (!torture_must_stop());
 822         torture_kthread_stopping("lock_torture_stats");
 823         return 0;
 824 }
 825
 826 static inline void
 827 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
 828                                 const char *tag)
 829 {
 830         pr_alert("%s" TORTURE_FLAG
 831                  "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
 832                  torture_type, tag, cxt.debug_lock ? " [debug]": "",
 833                  cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
 834                  verbose, shuffle_interval, stutter, shutdown_secs,
 835                  onoff_interval, onoff_holdoff);
 836 }
 837
 838 static void lock_torture_cleanup(void)
 839 {
 840         int i;
 841
 842         if (torture_cleanup_begin())
 843                 return;
 844
 845         /*
 846          * Indicates early cleanup, meaning that the test has not run,
 847          * such as when passing bogus args when loading the module.
 848          * However cxt->cur_ops.init() may have been invoked, so beside
 849          * perform the underlying torture-specific cleanups, cur_ops.exit()
 850          * will be invoked if needed.
 851          */
 852         if (!cxt.lwsa && !cxt.lrsa)
 853                 goto end;
 854
 855         if (writer_tasks) {
 856                 for (i = 0; i < cxt.nrealwriters_stress; i++)
 857                         torture_stop_kthread(lock_torture_writer,
 858                                              writer_tasks[i]);
 859                 kfree(writer_tasks);
 860                 writer_tasks = NULL;
 861         }
 862
 863         if (reader_tasks) {
 864                 for (i = 0; i < cxt.nrealreaders_stress; i++)
 865                         torture_stop_kthread(lock_torture_reader,
 866                                              reader_tasks[i]);
 867                 kfree(reader_tasks);
 868                 reader_tasks = NULL;
 869         }
 870
 871         torture_stop_kthread(lock_torture_stats, stats_task);
 872         lock_torture_stats_print();  /* -After- the stats thread is stopped! */
 873
 874         if (atomic_read(&cxt.n_lock_torture_errors))
 875                 lock_torture_print_module_parms(cxt.cur_ops,
 876                                                 "End of test: FAILURE");
 877         else if (torture_onoff_failures())
 878                 lock_torture_print_module_parms(cxt.cur_ops,
 879                                                 "End of test: LOCK_HOTPLUG");
 880         else
 881                 lock_torture_print_module_parms(cxt.cur_ops,
 882                                                 "End of test: SUCCESS");
 883
 884         kfree(cxt.lwsa);
 885         cxt.lwsa = NULL;
 886         kfree(cxt.lrsa);
 887         cxt.lrsa = NULL;
 888
 889 end:
 890         if (cxt.init_called) {
 891                 if (cxt.cur_ops->exit)
 892                         cxt.cur_ops->exit();
 893                 cxt.init_called = false;
 894         }
 895         torture_cleanup_end();
 896 }
 897
 898 static int __init lock_torture_init(void)
 899 {
 900         int i, j;
 901         int firsterr = 0;
 902         static struct lock_torture_ops *torture_ops[] = {
 903                 &lock_busted_ops,
 904                 &spin_lock_ops, &spin_lock_irq_ops,
 905                 &rw_lock_ops, &rw_lock_irq_ops,
 906                 &mutex_lock_ops,
 907                 &ww_mutex_lock_ops,
 908 #ifdef CONFIG_RT_MUTEXES
 909                 &rtmutex_lock_ops,
 910 #endif
 911                 &rwsem_lock_ops,
 912                 &percpu_rwsem_lock_ops,
 913         };
 914
 915         if (!torture_init_begin(torture_type, verbose))
 916                 return -EBUSY;
 917
 918         /* Process args and tell the world that the torturer is on the job. */
 919         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
 920                 cxt.cur_ops = torture_ops[i];
 921                 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
 922                         break;
 923         }
 924         if (i == ARRAY_SIZE(torture_ops)) {
 925                 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
 926                          torture_type);
 927                 pr_alert("lock-torture types:");
 928                 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
 929                         pr_alert(" %s", torture_ops[i]->name);
 930                 pr_alert("\n");
 931                 firsterr = -EINVAL;
 932                 goto unwind;
 933         }
 934
 935         if (nwriters_stress == 0 &&
 936             (!cxt.cur_ops->readlock || nreaders_stress == 0)) {
 937                 pr_alert("lock-torture: must run at least one locking thread\n");
 938                 firsterr = -EINVAL;
 939                 goto unwind;
 940         }
 941
 942         if (nwriters_stress >= 0)
 943                 cxt.nrealwriters_stress = nwriters_stress;
 944         else
 945                 cxt.nrealwriters_stress = 2 * num_online_cpus();
 946
 947         if (cxt.cur_ops->init) {
 948                 cxt.cur_ops->init();
 949                 cxt.init_called = true;
 950         }
 951
 952 #ifdef CONFIG_DEBUG_MUTEXES
 953         if (str_has_prefix(torture_type, "mutex"))
 954                 cxt.debug_lock = true;
 955 #endif
 956 #ifdef CONFIG_DEBUG_RT_MUTEXES
 957         if (str_has_prefix(torture_type, "rtmutex"))
 958                 cxt.debug_lock = true;
 959 #endif
 960 #ifdef CONFIG_DEBUG_SPINLOCK
 961         if ((str_has_prefix(torture_type, "spin")) ||
 962             (str_has_prefix(torture_type, "rw_lock")))
 963                 cxt.debug_lock = true;
 964 #endif
 965
 966         /* Initialize the statistics so that each run gets its own numbers. */
 967         if (nwriters_stress) {
 968                 lock_is_write_held = false;
 969                 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress,
 970                                          sizeof(*cxt.lwsa),
 971                                          GFP_KERNEL);
 972                 if (cxt.lwsa == NULL) {
 973                         VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
 974                         firsterr = -ENOMEM;
 975                         goto unwind;
 976                 }
 977
 978                 for (i = 0; i < cxt.nrealwriters_stress; i++) {
 979                         cxt.lwsa[i].n_lock_fail = 0;
 980                         cxt.lwsa[i].n_lock_acquired = 0;
 981                 }
 982         }
 983
 984         if (cxt.cur_ops->readlock) {
 985                 if (nreaders_stress >= 0)
 986                         cxt.nrealreaders_stress = nreaders_stress;
 987                 else {
 988                         /*
 989                          * By default distribute evenly the number of
 990                          * readers and writers. We still run the same number
 991                          * of threads as the writer-only locks default.
 992                          */
 993                         if (nwriters_stress < 0) /* user doesn't care */
 994                                 cxt.nrealwriters_stress = num_online_cpus();
 995                         cxt.nrealreaders_stress = cxt.nrealwriters_stress;
 996                 }
 997
 998                 if (nreaders_stress) {
 999                         lock_is_read_held = false;
1000                         cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress,
1001                                                  sizeof(*cxt.lrsa),
1002                                                  GFP_KERNEL);
1003                         if (cxt.lrsa == NULL) {
1004                                 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
1005                                 firsterr = -ENOMEM;
1006                                 kfree(cxt.lwsa);
1007                                 cxt.lwsa = NULL;
1008                                 goto unwind;
1009                         }
1010
1011                         for (i = 0; i < cxt.nrealreaders_stress; i++) {
1012                                 cxt.lrsa[i].n_lock_fail = 0;
1013                                 cxt.lrsa[i].n_lock_acquired = 0;
1014                         }
1015                 }
1016         }
1017
1018         lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
1019
1020         /* Prepare torture context. */
1021         if (onoff_interval > 0) {
1022                 firsterr = torture_onoff_init(onoff_holdoff * HZ,
1023                                               onoff_interval * HZ, NULL);
1024                 if (firsterr)
1025                         goto unwind;
1026         }
1027         if (shuffle_interval > 0) {
1028                 firsterr = torture_shuffle_init(shuffle_interval);
1029                 if (firsterr)
1030                         goto unwind;
1031         }
1032         if (shutdown_secs > 0) {
1033                 firsterr = torture_shutdown_init(shutdown_secs,
1034                                                  lock_torture_cleanup);
1035                 if (firsterr)
1036                         goto unwind;
1037         }
1038         if (stutter > 0) {
1039                 firsterr = torture_stutter_init(stutter, stutter);
1040                 if (firsterr)
1041                         goto unwind;
1042         }
1043
1044         if (nwriters_stress) {
1045                 writer_tasks = kcalloc(cxt.nrealwriters_stress,
1046                                        sizeof(writer_tasks[0]),
1047                                        GFP_KERNEL);
1048                 if (writer_tasks == NULL) {
1049                         VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
1050                         firsterr = -ENOMEM;
1051                         goto unwind;
1052                 }
1053         }
1054
1055         if (cxt.cur_ops->readlock) {
1056                 reader_tasks = kcalloc(cxt.nrealreaders_stress,
1057                                        sizeof(reader_tasks[0]),
1058                                        GFP_KERNEL);
1059                 if (reader_tasks == NULL) {
1060                         VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
1061                         kfree(writer_tasks);
1062                         writer_tasks = NULL;
1063                         firsterr = -ENOMEM;
1064                         goto unwind;
1065                 }
1066         }
1067
1068         /*
1069          * Create the kthreads and start torturing (oh, those poor little locks).
1070          *
1071          * TODO: Note that we interleave writers with readers, giving writers a
1072          * slight advantage, by creating its kthread first. This can be modified
1073          * for very specific needs, or even let the user choose the policy, if
1074          * ever wanted.
1075          */
1076         for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
1077                     j < cxt.nrealreaders_stress; i++, j++) {
1078                 if (i >= cxt.nrealwriters_stress)
1079                         goto create_reader;
1080
1081                 /* Create writer. */
1082                 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
1083                                                   writer_tasks[i]);
1084                 if (firsterr)
1085                         goto unwind;
1086
1087         create_reader:
1088                 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
1089                         continue;
1090                 /* Create reader. */
1091                 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
1092                                                   reader_tasks[j]);
1093                 if (firsterr)
1094                         goto unwind;
1095         }
1096         if (stat_interval > 0) {
1097                 firsterr = torture_create_kthread(lock_torture_stats, NULL,
1098                                                   stats_task);
1099                 if (firsterr)
1100                         goto unwind;
1101         }
1102         torture_init_end();
1103         return 0;
1104
1105 unwind:
1106         torture_init_end();
1107         lock_torture_cleanup();
1108         if (shutdown_secs) {
1109                 WARN_ON(!IS_MODULE(CONFIG_LOCK_TORTURE_TEST));
1110                 kernel_power_off();
1111         }
1112         return firsterr;
1113 }
1114
1115 module_init(lock_torture_init);
1116 module_exit(lock_torture_cleanup);