kernel/locking/locktorture.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Module-based torture test facility for locking
   4  *
   5  * Copyright (C) IBM Corporation, 2014
   6  *
   7  * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
   8  *          Davidlohr Bueso <dave@stgolabs.net>
   9  *      Based on kernel/rcu/torture.c.
  10  */
  11
  12 #define pr_fmt(fmt) fmt
  13
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/kthread.h>
  17 #include <linux/sched/rt.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/mutex.h>
  20 #include <linux/rwsem.h>
  21 #include <linux/smp.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/sched.h>
  24 #include <uapi/linux/sched/types.h>
  25 #include <linux/rtmutex.h>
  26 #include <linux/atomic.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/delay.h>
  29 #include <linux/slab.h>
  30 #include <linux/torture.h>
  31 #include <linux/reboot.h>
  32
  33 MODULE_LICENSE("GPL");
  34 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
  35
  36 torture_param(int, nwriters_stress, -1,
  37              "Number of write-locking stress-test threads");
  38 torture_param(int, nreaders_stress, -1,
  39              "Number of read-locking stress-test threads");
  40 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
  41 torture_param(int, onoff_interval, 0,
  42              "Time between CPU hotplugs (s), 0=disable");
  43 torture_param(int, shuffle_interval, 3,
  44              "Number of jiffies between shuffles, 0=disable");
  45 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
  46 torture_param(int, stat_interval, 60,
  47              "Number of seconds between stats printk()s");
  48 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
  49 torture_param(int, verbose, 1,
  50              "Enable verbose debugging printk()s");
  51
  52 static char *torture_type = "spin_lock";
  53 module_param(torture_type, charp, 0444);
  54 MODULE_PARM_DESC(torture_type,
  55                  "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
  56
  57 static struct task_struct *stats_task;
  58 static struct task_struct **writer_tasks;
  59 static struct task_struct **reader_tasks;
  60
  61 static bool lock_is_write_held;
  62 static bool lock_is_read_held;
  63 static unsigned long last_lock_release;
  64
  65 struct lock_stress_stats {
  66         long n_lock_fail;
  67         long n_lock_acquired;
  68 };
  69
  70 /* Forward reference. */
  71 static void lock_torture_cleanup(void);
  72
  73 /*
  74  * Operations vector for selecting different types of tests.
  75  */
  76 struct lock_torture_ops {
  77         void (*init)(void);
  78         void (*exit)(void);
  79         int (*writelock)(void);
  80         void (*write_delay)(struct torture_random_state *trsp);
  81         void (*task_boost)(struct torture_random_state *trsp);
  82         void (*writeunlock)(void);
  83         int (*readlock)(void);
  84         void (*read_delay)(struct torture_random_state *trsp);
  85         void (*readunlock)(void);
  86
  87         unsigned long flags; /* for irq spinlocks */
  88         const char *name;
  89 };
  90
  91 struct lock_torture_cxt {
  92         int nrealwriters_stress;
  93         int nrealreaders_stress;
  94         bool debug_lock;
  95         bool init_called;
  96         atomic_t n_lock_torture_errors;
  97         struct lock_torture_ops *cur_ops;
  98         struct lock_stress_stats *lwsa; /* writer statistics */
  99         struct lock_stress_stats *lrsa; /* reader statistics */
 100 };
 101 static struct lock_torture_cxt cxt = { 0, 0, false, false,
 102                                        ATOMIC_INIT(0),
 103                                        NULL, NULL};
 104 /*
 105  * Definitions for lock torture testing.
 106  */
 107
 108 static int torture_lock_busted_write_lock(void)
 109 {
 110         return 0;  /* BUGGY, do not use in real life!!! */
 111 }
 112
 113 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
 114 {
 115         const unsigned long longdelay_ms = 100;
 116
 117         /* We want a long delay occasionally to force massive contention.  */
 118         if (!(torture_random(trsp) %
 119               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 120                 mdelay(longdelay_ms);
 121         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 122                 torture_preempt_schedule();  /* Allow test to be preempted. */
 123 }
 124
 125 static void torture_lock_busted_write_unlock(void)
 126 {
 127           /* BUGGY, do not use in real life!!! */
 128 }
 129
 130 static void torture_boost_dummy(struct torture_random_state *trsp)
 131 {
 132         /* Only rtmutexes care about priority */
 133 }
 134
 135 static struct lock_torture_ops lock_busted_ops = {
 136         .writelock      = torture_lock_busted_write_lock,
 137         .write_delay    = torture_lock_busted_write_delay,
 138         .task_boost     = torture_boost_dummy,
 139         .writeunlock    = torture_lock_busted_write_unlock,
 140         .readlock       = NULL,
 141         .read_delay     = NULL,
 142         .readunlock     = NULL,
 143         .name           = "lock_busted"
 144 };
 145
 146 static DEFINE_SPINLOCK(torture_spinlock);
 147
 148 static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock)
 149 {
 150         spin_lock(&torture_spinlock);
 151         return 0;
 152 }
 153
 154 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
 155 {
 156         const unsigned long shortdelay_us = 2;
 157         const unsigned long longdelay_ms = 100;
 158
 159         /* We want a short delay mostly to emulate likely code, and
 160          * we want a long delay occasionally to force massive contention.
 161          */
 162         if (!(torture_random(trsp) %
 163               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 164                 mdelay(longdelay_ms);
 165         if (!(torture_random(trsp) %
 166               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 167                 udelay(shortdelay_us);
 168         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 169                 torture_preempt_schedule();  /* Allow test to be preempted. */
 170 }
 171
 172 static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock)
 173 {
 174         spin_unlock(&torture_spinlock);
 175 }
 176
 177 static struct lock_torture_ops spin_lock_ops = {
 178         .writelock      = torture_spin_lock_write_lock,
 179         .write_delay    = torture_spin_lock_write_delay,
 180         .task_boost     = torture_boost_dummy,
 181         .writeunlock    = torture_spin_lock_write_unlock,
 182         .readlock       = NULL,
 183         .read_delay     = NULL,
 184         .readunlock     = NULL,
 185         .name           = "spin_lock"
 186 };
 187
 188 static int torture_spin_lock_write_lock_irq(void)
 189 __acquires(torture_spinlock)
 190 {
 191         unsigned long flags;
 192
 193         spin_lock_irqsave(&torture_spinlock, flags);
 194         cxt.cur_ops->flags = flags;
 195         return 0;
 196 }
 197
 198 static void torture_lock_spin_write_unlock_irq(void)
 199 __releases(torture_spinlock)
 200 {
 201         spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
 202 }
 203
 204 static struct lock_torture_ops spin_lock_irq_ops = {
 205         .writelock      = torture_spin_lock_write_lock_irq,
 206         .write_delay    = torture_spin_lock_write_delay,
 207         .task_boost     = torture_boost_dummy,
 208         .writeunlock    = torture_lock_spin_write_unlock_irq,
 209         .readlock       = NULL,
 210         .read_delay     = NULL,
 211         .readunlock     = NULL,
 212         .name           = "spin_lock_irq"
 213 };
 214
 215 static DEFINE_RWLOCK(torture_rwlock);
 216
 217 static int torture_rwlock_write_lock(void) __acquires(torture_rwlock)
 218 {
 219         write_lock(&torture_rwlock);
 220         return 0;
 221 }
 222
 223 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
 224 {
 225         const unsigned long shortdelay_us = 2;
 226         const unsigned long longdelay_ms = 100;
 227
 228         /* We want a short delay mostly to emulate likely code, and
 229          * we want a long delay occasionally to force massive contention.
 230          */
 231         if (!(torture_random(trsp) %
 232               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 233                 mdelay(longdelay_ms);
 234         else
 235                 udelay(shortdelay_us);
 236 }
 237
 238 static void torture_rwlock_write_unlock(void) __releases(torture_rwlock)
 239 {
 240         write_unlock(&torture_rwlock);
 241 }
 242
 243 static int torture_rwlock_read_lock(void) __acquires(torture_rwlock)
 244 {
 245         read_lock(&torture_rwlock);
 246         return 0;
 247 }
 248
 249 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
 250 {
 251         const unsigned long shortdelay_us = 10;
 252         const unsigned long longdelay_ms = 100;
 253
 254         /* We want a short delay mostly to emulate likely code, and
 255          * we want a long delay occasionally to force massive contention.
 256          */
 257         if (!(torture_random(trsp) %
 258               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 259                 mdelay(longdelay_ms);
 260         else
 261                 udelay(shortdelay_us);
 262 }
 263
 264 static void torture_rwlock_read_unlock(void) __releases(torture_rwlock)
 265 {
 266         read_unlock(&torture_rwlock);
 267 }
 268
 269 static struct lock_torture_ops rw_lock_ops = {
 270         .writelock      = torture_rwlock_write_lock,
 271         .write_delay    = torture_rwlock_write_delay,
 272         .task_boost     = torture_boost_dummy,
 273         .writeunlock    = torture_rwlock_write_unlock,
 274         .readlock       = torture_rwlock_read_lock,
 275         .read_delay     = torture_rwlock_read_delay,
 276         .readunlock     = torture_rwlock_read_unlock,
 277         .name           = "rw_lock"
 278 };
 279
 280 static int torture_rwlock_write_lock_irq(void) __acquires(torture_rwlock)
 281 {
 282         unsigned long flags;
 283
 284         write_lock_irqsave(&torture_rwlock, flags);
 285         cxt.cur_ops->flags = flags;
 286         return 0;
 287 }
 288
 289 static void torture_rwlock_write_unlock_irq(void)
 290 __releases(torture_rwlock)
 291 {
 292         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 293 }
 294
 295 static int torture_rwlock_read_lock_irq(void) __acquires(torture_rwlock)
 296 {
 297         unsigned long flags;
 298
 299         read_lock_irqsave(&torture_rwlock, flags);
 300         cxt.cur_ops->flags = flags;
 301         return 0;
 302 }
 303
 304 static void torture_rwlock_read_unlock_irq(void)
 305 __releases(torture_rwlock)
 306 {
 307         read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
 308 }
 309
 310 static struct lock_torture_ops rw_lock_irq_ops = {
 311         .writelock      = torture_rwlock_write_lock_irq,
 312         .write_delay    = torture_rwlock_write_delay,
 313         .task_boost     = torture_boost_dummy,
 314         .writeunlock    = torture_rwlock_write_unlock_irq,
 315         .readlock       = torture_rwlock_read_lock_irq,
 316         .read_delay     = torture_rwlock_read_delay,
 317         .readunlock     = torture_rwlock_read_unlock_irq,
 318         .name           = "rw_lock_irq"
 319 };
 320
 321 static DEFINE_MUTEX(torture_mutex);
 322
 323 static int torture_mutex_lock(void) __acquires(torture_mutex)
 324 {
 325         mutex_lock(&torture_mutex);
 326         return 0;
 327 }
 328
 329 static void torture_mutex_delay(struct torture_random_state *trsp)
 330 {
 331         const unsigned long longdelay_ms = 100;
 332
 333         /* We want a long delay occasionally to force massive contention.  */
 334         if (!(torture_random(trsp) %
 335               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 336                 mdelay(longdelay_ms * 5);
 337         else
 338                 mdelay(longdelay_ms / 5);
 339         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 340                 torture_preempt_schedule();  /* Allow test to be preempted. */
 341 }
 342
 343 static void torture_mutex_unlock(void) __releases(torture_mutex)
 344 {
 345         mutex_unlock(&torture_mutex);
 346 }
 347
 348 static struct lock_torture_ops mutex_lock_ops = {
 349         .writelock      = torture_mutex_lock,
 350         .write_delay    = torture_mutex_delay,
 351         .task_boost     = torture_boost_dummy,
 352         .writeunlock    = torture_mutex_unlock,
 353         .readlock       = NULL,
 354         .read_delay     = NULL,
 355         .readunlock     = NULL,
 356         .name           = "mutex_lock"
 357 };
 358
 359 #include <linux/ww_mutex.h>
 360 static DEFINE_WD_CLASS(torture_ww_class);
 361 static DEFINE_WW_MUTEX(torture_ww_mutex_0, &torture_ww_class);
 362 static DEFINE_WW_MUTEX(torture_ww_mutex_1, &torture_ww_class);
 363 static DEFINE_WW_MUTEX(torture_ww_mutex_2, &torture_ww_class);
 364
 365 static int torture_ww_mutex_lock(void)
 366 __acquires(torture_ww_mutex_0)
 367 __acquires(torture_ww_mutex_1)
 368 __acquires(torture_ww_mutex_2)
 369 {
 370         LIST_HEAD(list);
 371         struct reorder_lock {
 372                 struct list_head link;
 373                 struct ww_mutex *lock;
 374         } locks[3], *ll, *ln;
 375         struct ww_acquire_ctx ctx;
 376
 377         locks[0].lock = &torture_ww_mutex_0;
 378         list_add(&locks[0].link, &list);
 379
 380         locks[1].lock = &torture_ww_mutex_1;
 381         list_add(&locks[1].link, &list);
 382
 383         locks[2].lock = &torture_ww_mutex_2;
 384         list_add(&locks[2].link, &list);
 385
 386         ww_acquire_init(&ctx, &torture_ww_class);
 387
 388         list_for_each_entry(ll, &list, link) {
 389                 int err;
 390
 391                 err = ww_mutex_lock(ll->lock, &ctx);
 392                 if (!err)
 393                         continue;
 394
 395                 ln = ll;
 396                 list_for_each_entry_continue_reverse(ln, &list, link)
 397                         ww_mutex_unlock(ln->lock);
 398
 399                 if (err != -EDEADLK)
 400                         return err;
 401
 402                 ww_mutex_lock_slow(ll->lock, &ctx);
 403                 list_move(&ll->link, &list);
 404         }
 405
 406         ww_acquire_fini(&ctx);
 407         return 0;
 408 }
 409
 410 static void torture_ww_mutex_unlock(void)
 411 __releases(torture_ww_mutex_0)
 412 __releases(torture_ww_mutex_1)
 413 __releases(torture_ww_mutex_2)
 414 {
 415         ww_mutex_unlock(&torture_ww_mutex_0);
 416         ww_mutex_unlock(&torture_ww_mutex_1);
 417         ww_mutex_unlock(&torture_ww_mutex_2);
 418 }
 419
 420 static struct lock_torture_ops ww_mutex_lock_ops = {
 421         .writelock      = torture_ww_mutex_lock,
 422         .write_delay    = torture_mutex_delay,
 423         .task_boost     = torture_boost_dummy,
 424         .writeunlock    = torture_ww_mutex_unlock,
 425         .readlock       = NULL,
 426         .read_delay     = NULL,
 427         .readunlock     = NULL,
 428         .name           = "ww_mutex_lock"
 429 };
 430
 431 #ifdef CONFIG_RT_MUTEXES
 432 static DEFINE_RT_MUTEX(torture_rtmutex);
 433
 434 static int torture_rtmutex_lock(void) __acquires(torture_rtmutex)
 435 {
 436         rt_mutex_lock(&torture_rtmutex);
 437         return 0;
 438 }
 439
 440 static void torture_rtmutex_boost(struct torture_random_state *trsp)
 441 {
 442         const unsigned int factor = 50000; /* yes, quite arbitrary */
 443
 444         if (!rt_task(current)) {
 445                 /*
 446                  * Boost priority once every ~50k operations. When the
 447                  * task tries to take the lock, the rtmutex it will account
 448                  * for the new priority, and do any corresponding pi-dance.
 449                  */
 450                 if (trsp && !(torture_random(trsp) %
 451                               (cxt.nrealwriters_stress * factor))) {
 452                         sched_set_fifo(current);
 453                 } else /* common case, do nothing */
 454                         return;
 455         } else {
 456                 /*
 457                  * The task will remain boosted for another ~500k operations,
 458                  * then restored back to its original prio, and so forth.
 459                  *
 460                  * When @trsp is nil, we want to force-reset the task for
 461                  * stopping the kthread.
 462                  */
 463                 if (!trsp || !(torture_random(trsp) %
 464                                (cxt.nrealwriters_stress * factor * 2))) {
 465                         sched_set_normal(current, 0);
 466                 } else /* common case, do nothing */
 467                         return;
 468         }
 469 }
 470
 471 static void torture_rtmutex_delay(struct torture_random_state *trsp)
 472 {
 473         const unsigned long shortdelay_us = 2;
 474         const unsigned long longdelay_ms = 100;
 475
 476         /*
 477          * We want a short delay mostly to emulate likely code, and
 478          * we want a long delay occasionally to force massive contention.
 479          */
 480         if (!(torture_random(trsp) %
 481               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 482                 mdelay(longdelay_ms);
 483         if (!(torture_random(trsp) %
 484               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
 485                 udelay(shortdelay_us);
 486         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 487                 torture_preempt_schedule();  /* Allow test to be preempted. */
 488 }
 489
 490 static void torture_rtmutex_unlock(void) __releases(torture_rtmutex)
 491 {
 492         rt_mutex_unlock(&torture_rtmutex);
 493 }
 494
 495 static struct lock_torture_ops rtmutex_lock_ops = {
 496         .writelock      = torture_rtmutex_lock,
 497         .write_delay    = torture_rtmutex_delay,
 498         .task_boost     = torture_rtmutex_boost,
 499         .writeunlock    = torture_rtmutex_unlock,
 500         .readlock       = NULL,
 501         .read_delay     = NULL,
 502         .readunlock     = NULL,
 503         .name           = "rtmutex_lock"
 504 };
 505 #endif
 506
 507 static DECLARE_RWSEM(torture_rwsem);
 508 static int torture_rwsem_down_write(void) __acquires(torture_rwsem)
 509 {
 510         down_write(&torture_rwsem);
 511         return 0;
 512 }
 513
 514 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
 515 {
 516         const unsigned long longdelay_ms = 100;
 517
 518         /* We want a long delay occasionally to force massive contention.  */
 519         if (!(torture_random(trsp) %
 520               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
 521                 mdelay(longdelay_ms * 10);
 522         else
 523                 mdelay(longdelay_ms / 10);
 524         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
 525                 torture_preempt_schedule();  /* Allow test to be preempted. */
 526 }
 527
 528 static void torture_rwsem_up_write(void) __releases(torture_rwsem)
 529 {
 530         up_write(&torture_rwsem);
 531 }
 532
 533 static int torture_rwsem_down_read(void) __acquires(torture_rwsem)
 534 {
 535         down_read(&torture_rwsem);
 536         return 0;
 537 }
 538
 539 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
 540 {
 541         const unsigned long longdelay_ms = 100;
 542
 543         /* We want a long delay occasionally to force massive contention.  */
 544         if (!(torture_random(trsp) %
 545               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
 546                 mdelay(longdelay_ms * 2);
 547         else
 548                 mdelay(longdelay_ms / 2);
 549         if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
 550                 torture_preempt_schedule();  /* Allow test to be preempted. */
 551 }
 552
 553 static void torture_rwsem_up_read(void) __releases(torture_rwsem)
 554 {
 555         up_read(&torture_rwsem);
 556 }
 557
 558 static struct lock_torture_ops rwsem_lock_ops = {
 559         .writelock      = torture_rwsem_down_write,
 560         .write_delay    = torture_rwsem_write_delay,
 561         .task_boost     = torture_boost_dummy,
 562         .writeunlock    = torture_rwsem_up_write,
 563         .readlock       = torture_rwsem_down_read,
 564         .read_delay     = torture_rwsem_read_delay,
 565         .readunlock     = torture_rwsem_up_read,
 566         .name           = "rwsem_lock"
 567 };
 568
 569 #include <linux/percpu-rwsem.h>
 570 static struct percpu_rw_semaphore pcpu_rwsem;
 571
 572 static void torture_percpu_rwsem_init(void)
 573 {
 574         BUG_ON(percpu_init_rwsem(&pcpu_rwsem));
 575 }
 576
 577 static void torture_percpu_rwsem_exit(void)
 578 {
 579         percpu_free_rwsem(&pcpu_rwsem);
 580 }
 581
 582 static int torture_percpu_rwsem_down_write(void) __acquires(pcpu_rwsem)
 583 {
 584         percpu_down_write(&pcpu_rwsem);
 585         return 0;
 586 }
 587
 588 static void torture_percpu_rwsem_up_write(void) __releases(pcpu_rwsem)
 589 {
 590         percpu_up_write(&pcpu_rwsem);
 591 }
 592
 593 static int torture_percpu_rwsem_down_read(void) __acquires(pcpu_rwsem)
 594 {
 595         percpu_down_read(&pcpu_rwsem);
 596         return 0;
 597 }
 598
 599 static void torture_percpu_rwsem_up_read(void) __releases(pcpu_rwsem)
 600 {
 601         percpu_up_read(&pcpu_rwsem);
 602 }
 603
 604 static struct lock_torture_ops percpu_rwsem_lock_ops = {
 605         .init           = torture_percpu_rwsem_init,
 606         .exit           = torture_percpu_rwsem_exit,
 607         .writelock      = torture_percpu_rwsem_down_write,
 608         .write_delay    = torture_rwsem_write_delay,
 609         .task_boost     = torture_boost_dummy,
 610         .writeunlock    = torture_percpu_rwsem_up_write,
 611         .readlock       = torture_percpu_rwsem_down_read,
 612         .read_delay     = torture_rwsem_read_delay,
 613         .readunlock     = torture_percpu_rwsem_up_read,
 614         .name           = "percpu_rwsem_lock"
 615 };
 616
 617 /*
 618  * Lock torture writer kthread.  Repeatedly acquires and releases
 619  * the lock, checking for duplicate acquisitions.
 620  */
 621 static int lock_torture_writer(void *arg)
 622 {
 623         struct lock_stress_stats *lwsp = arg;
 624         DEFINE_TORTURE_RANDOM(rand);
 625
 626         VERBOSE_TOROUT_STRING("lock_torture_writer task started");
 627         set_user_nice(current, MAX_NICE);
 628
 629         do {
 630                 if ((torture_random(&rand) & 0xfffff) == 0)
 631                         schedule_timeout_uninterruptible(1);
 632
 633                 cxt.cur_ops->task_boost(&rand);
 634                 cxt.cur_ops->writelock();
 635                 if (WARN_ON_ONCE(lock_is_write_held))
 636                         lwsp->n_lock_fail++;
 637                 lock_is_write_held = true;
 638                 if (WARN_ON_ONCE(lock_is_read_held))
 639                         lwsp->n_lock_fail++; /* rare, but... */
 640
 641                 lwsp->n_lock_acquired++;
 642                 cxt.cur_ops->write_delay(&rand);
 643                 lock_is_write_held = false;
 644                 WRITE_ONCE(last_lock_release, jiffies);
 645                 cxt.cur_ops->writeunlock();
 646
 647                 stutter_wait("lock_torture_writer");
 648         } while (!torture_must_stop());
 649
 650         cxt.cur_ops->task_boost(NULL); /* reset prio */
 651         torture_kthread_stopping("lock_torture_writer");
 652         return 0;
 653 }
 654
 655 /*
 656  * Lock torture reader kthread.  Repeatedly acquires and releases
 657  * the reader lock.
 658  */
 659 static int lock_torture_reader(void *arg)
 660 {
 661         struct lock_stress_stats *lrsp = arg;
 662         DEFINE_TORTURE_RANDOM(rand);
 663
 664         VERBOSE_TOROUT_STRING("lock_torture_reader task started");
 665         set_user_nice(current, MAX_NICE);
 666
 667         do {
 668                 if ((torture_random(&rand) & 0xfffff) == 0)
 669                         schedule_timeout_uninterruptible(1);
 670
 671                 cxt.cur_ops->readlock();
 672                 lock_is_read_held = true;
 673                 if (WARN_ON_ONCE(lock_is_write_held))
 674                         lrsp->n_lock_fail++; /* rare, but... */
 675
 676                 lrsp->n_lock_acquired++;
 677                 cxt.cur_ops->read_delay(&rand);
 678                 lock_is_read_held = false;
 679                 cxt.cur_ops->readunlock();
 680
 681                 stutter_wait("lock_torture_reader");
 682         } while (!torture_must_stop());
 683         torture_kthread_stopping("lock_torture_reader");
 684         return 0;
 685 }
 686
 687 /*
 688  * Create an lock-torture-statistics message in the specified buffer.
 689  */
 690 static void __torture_print_stats(char *page,
 691                                   struct lock_stress_stats *statp, bool write)
 692 {
 693         bool fail = false;
 694         int i, n_stress;
 695         long max = 0, min = statp ? statp[0].n_lock_acquired : 0;
 696         long long sum = 0;
 697
 698         n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
 699         for (i = 0; i < n_stress; i++) {
 700                 if (statp[i].n_lock_fail)
 701                         fail = true;
 702                 sum += statp[i].n_lock_acquired;
 703                 if (max < statp[i].n_lock_acquired)
 704                         max = statp[i].n_lock_acquired;
 705                 if (min > statp[i].n_lock_acquired)
 706                         min = statp[i].n_lock_acquired;
 707         }
 708         page += sprintf(page,
 709                         "%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
 710                         write ? "Writes" : "Reads ",
 711                         sum, max, min,
 712                         !onoff_interval && max / 2 > min ? "???" : "",
 713                         fail, fail ? "!!!" : "");
 714         if (fail)
 715                 atomic_inc(&cxt.n_lock_torture_errors);
 716 }
 717
 718 /*
 719  * Print torture statistics.  Caller must ensure that there is only one
 720  * call to this function at a given time!!!  This is normally accomplished
 721  * by relying on the module system to only have one copy of the module
 722  * loaded, and then by giving the lock_torture_stats kthread full control
 723  * (or the init/cleanup functions when lock_torture_stats thread is not
 724  * running).
 725  */
 726 static void lock_torture_stats_print(void)
 727 {
 728         int size = cxt.nrealwriters_stress * 200 + 8192;
 729         char *buf;
 730
 731         if (cxt.cur_ops->readlock)
 732                 size += cxt.nrealreaders_stress * 200 + 8192;
 733
 734         buf = kmalloc(size, GFP_KERNEL);
 735         if (!buf) {
 736                 pr_err("lock_torture_stats_print: Out of memory, need: %d",
 737                        size);
 738                 return;
 739         }
 740
 741         __torture_print_stats(buf, cxt.lwsa, true);
 742         pr_alert("%s", buf);
 743         kfree(buf);
 744
 745         if (cxt.cur_ops->readlock) {
 746                 buf = kmalloc(size, GFP_KERNEL);
 747                 if (!buf) {
 748                         pr_err("lock_torture_stats_print: Out of memory, need: %d",
 749                                size);
 750                         return;
 751                 }
 752
 753                 __torture_print_stats(buf, cxt.lrsa, false);
 754                 pr_alert("%s", buf);
 755                 kfree(buf);
 756         }
 757 }
 758
 759 /*
 760  * Periodically prints torture statistics, if periodic statistics printing
 761  * was specified via the stat_interval module parameter.
 762  *
 763  * No need to worry about fullstop here, since this one doesn't reference
 764  * volatile state or register callbacks.
 765  */
 766 static int lock_torture_stats(void *arg)
 767 {
 768         VERBOSE_TOROUT_STRING("lock_torture_stats task started");
 769         do {
 770                 schedule_timeout_interruptible(stat_interval * HZ);
 771                 lock_torture_stats_print();
 772                 torture_shutdown_absorb("lock_torture_stats");
 773         } while (!torture_must_stop());
 774         torture_kthread_stopping("lock_torture_stats");
 775         return 0;
 776 }
 777
 778 static inline void
 779 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
 780                                 const char *tag)
 781 {
 782         pr_alert("%s" TORTURE_FLAG
 783                  "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
 784                  torture_type, tag, cxt.debug_lock ? " [debug]": "",
 785                  cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
 786                  verbose, shuffle_interval, stutter, shutdown_secs,
 787                  onoff_interval, onoff_holdoff);
 788 }
 789
 790 static void lock_torture_cleanup(void)
 791 {
 792         int i;
 793
 794         if (torture_cleanup_begin())
 795                 return;
 796
 797         /*
 798          * Indicates early cleanup, meaning that the test has not run,
 799          * such as when passing bogus args when loading the module.
 800          * However cxt->cur_ops.init() may have been invoked, so beside
 801          * perform the underlying torture-specific cleanups, cur_ops.exit()
 802          * will be invoked if needed.
 803          */
 804         if (!cxt.lwsa && !cxt.lrsa)
 805                 goto end;
 806
 807         if (writer_tasks) {
 808                 for (i = 0; i < cxt.nrealwriters_stress; i++)
 809                         torture_stop_kthread(lock_torture_writer,
 810                                              writer_tasks[i]);
 811                 kfree(writer_tasks);
 812                 writer_tasks = NULL;
 813         }
 814
 815         if (reader_tasks) {
 816                 for (i = 0; i < cxt.nrealreaders_stress; i++)
 817                         torture_stop_kthread(lock_torture_reader,
 818                                              reader_tasks[i]);
 819                 kfree(reader_tasks);
 820                 reader_tasks = NULL;
 821         }
 822
 823         torture_stop_kthread(lock_torture_stats, stats_task);
 824         lock_torture_stats_print();  /* -After- the stats thread is stopped! */
 825
 826         if (atomic_read(&cxt.n_lock_torture_errors))
 827                 lock_torture_print_module_parms(cxt.cur_ops,
 828                                                 "End of test: FAILURE");
 829         else if (torture_onoff_failures())
 830                 lock_torture_print_module_parms(cxt.cur_ops,
 831                                                 "End of test: LOCK_HOTPLUG");
 832         else
 833                 lock_torture_print_module_parms(cxt.cur_ops,
 834                                                 "End of test: SUCCESS");
 835
 836         kfree(cxt.lwsa);
 837         cxt.lwsa = NULL;
 838         kfree(cxt.lrsa);
 839         cxt.lrsa = NULL;
 840
 841 end:
 842         if (cxt.init_called) {
 843                 if (cxt.cur_ops->exit)
 844                         cxt.cur_ops->exit();
 845                 cxt.init_called = false;
 846         }
 847         torture_cleanup_end();
 848 }
 849
 850 static int __init lock_torture_init(void)
 851 {
 852         int i, j;
 853         int firsterr = 0;
 854         static struct lock_torture_ops *torture_ops[] = {
 855                 &lock_busted_ops,
 856                 &spin_lock_ops, &spin_lock_irq_ops,
 857                 &rw_lock_ops, &rw_lock_irq_ops,
 858                 &mutex_lock_ops,
 859                 &ww_mutex_lock_ops,
 860 #ifdef CONFIG_RT_MUTEXES
 861                 &rtmutex_lock_ops,
 862 #endif
 863                 &rwsem_lock_ops,
 864                 &percpu_rwsem_lock_ops,
 865         };
 866
 867         if (!torture_init_begin(torture_type, verbose))
 868                 return -EBUSY;
 869
 870         /* Process args and tell the world that the torturer is on the job. */
 871         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
 872                 cxt.cur_ops = torture_ops[i];
 873                 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
 874                         break;
 875         }
 876         if (i == ARRAY_SIZE(torture_ops)) {
 877                 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
 878                          torture_type);
 879                 pr_alert("lock-torture types:");
 880                 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
 881                         pr_alert(" %s", torture_ops[i]->name);
 882                 pr_alert("\n");
 883                 firsterr = -EINVAL;
 884                 goto unwind;
 885         }
 886
 887         if (nwriters_stress == 0 &&
 888             (!cxt.cur_ops->readlock || nreaders_stress == 0)) {
 889                 pr_alert("lock-torture: must run at least one locking thread\n");
 890                 firsterr = -EINVAL;
 891                 goto unwind;
 892         }
 893
 894         if (cxt.cur_ops->init) {
 895                 cxt.cur_ops->init();
 896                 cxt.init_called = true;
 897         }
 898
 899         if (nwriters_stress >= 0)
 900                 cxt.nrealwriters_stress = nwriters_stress;
 901         else
 902                 cxt.nrealwriters_stress = 2 * num_online_cpus();
 903
 904 #ifdef CONFIG_DEBUG_MUTEXES
 905         if (str_has_prefix(torture_type, "mutex"))
 906                 cxt.debug_lock = true;
 907 #endif
 908 #ifdef CONFIG_DEBUG_RT_MUTEXES
 909         if (str_has_prefix(torture_type, "rtmutex"))
 910                 cxt.debug_lock = true;
 911 #endif
 912 #ifdef CONFIG_DEBUG_SPINLOCK
 913         if ((str_has_prefix(torture_type, "spin")) ||
 914             (str_has_prefix(torture_type, "rw_lock")))
 915                 cxt.debug_lock = true;
 916 #endif
 917
 918         /* Initialize the statistics so that each run gets its own numbers. */
 919         if (nwriters_stress) {
 920                 lock_is_write_held = false;
 921                 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress,
 922                                          sizeof(*cxt.lwsa),
 923                                          GFP_KERNEL);
 924                 if (cxt.lwsa == NULL) {
 925                         VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
 926                         firsterr = -ENOMEM;
 927                         goto unwind;
 928                 }
 929
 930                 for (i = 0; i < cxt.nrealwriters_stress; i++) {
 931                         cxt.lwsa[i].n_lock_fail = 0;
 932                         cxt.lwsa[i].n_lock_acquired = 0;
 933                 }
 934         }
 935
 936         if (cxt.cur_ops->readlock) {
 937                 if (nreaders_stress >= 0)
 938                         cxt.nrealreaders_stress = nreaders_stress;
 939                 else {
 940                         /*
 941                          * By default distribute evenly the number of
 942                          * readers and writers. We still run the same number
 943                          * of threads as the writer-only locks default.
 944                          */
 945                         if (nwriters_stress < 0) /* user doesn't care */
 946                                 cxt.nrealwriters_stress = num_online_cpus();
 947                         cxt.nrealreaders_stress = cxt.nrealwriters_stress;
 948                 }
 949
 950                 if (nreaders_stress) {
 951                         lock_is_read_held = false;
 952                         cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress,
 953                                                  sizeof(*cxt.lrsa),
 954                                                  GFP_KERNEL);
 955                         if (cxt.lrsa == NULL) {
 956                                 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
 957                                 firsterr = -ENOMEM;
 958                                 kfree(cxt.lwsa);
 959                                 cxt.lwsa = NULL;
 960                                 goto unwind;
 961                         }
 962
 963                         for (i = 0; i < cxt.nrealreaders_stress; i++) {
 964                                 cxt.lrsa[i].n_lock_fail = 0;
 965                                 cxt.lrsa[i].n_lock_acquired = 0;
 966                         }
 967                 }
 968         }
 969
 970         lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
 971
 972         /* Prepare torture context. */
 973         if (onoff_interval > 0) {
 974                 firsterr = torture_onoff_init(onoff_holdoff * HZ,
 975                                               onoff_interval * HZ, NULL);
 976                 if (firsterr)
 977                         goto unwind;
 978         }
 979         if (shuffle_interval > 0) {
 980                 firsterr = torture_shuffle_init(shuffle_interval);
 981                 if (firsterr)
 982                         goto unwind;
 983         }
 984         if (shutdown_secs > 0) {
 985                 firsterr = torture_shutdown_init(shutdown_secs,
 986                                                  lock_torture_cleanup);
 987                 if (firsterr)
 988                         goto unwind;
 989         }
 990         if (stutter > 0) {
 991                 firsterr = torture_stutter_init(stutter, stutter);
 992                 if (firsterr)
 993                         goto unwind;
 994         }
 995
 996         if (nwriters_stress) {
 997                 writer_tasks = kcalloc(cxt.nrealwriters_stress,
 998                                        sizeof(writer_tasks[0]),
 999                                        GFP_KERNEL);
1000                 if (writer_tasks == NULL) {
1001                         VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
1002                         firsterr = -ENOMEM;
1003                         goto unwind;
1004                 }
1005         }
1006
1007         if (cxt.cur_ops->readlock) {
1008                 reader_tasks = kcalloc(cxt.nrealreaders_stress,
1009                                        sizeof(reader_tasks[0]),
1010                                        GFP_KERNEL);
1011                 if (reader_tasks == NULL) {
1012                         VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
1013                         kfree(writer_tasks);
1014                         writer_tasks = NULL;
1015                         firsterr = -ENOMEM;
1016                         goto unwind;
1017                 }
1018         }
1019
1020         /*
1021          * Create the kthreads and start torturing (oh, those poor little locks).
1022          *
1023          * TODO: Note that we interleave writers with readers, giving writers a
1024          * slight advantage, by creating its kthread first. This can be modified
1025          * for very specific needs, or even let the user choose the policy, if
1026          * ever wanted.
1027          */
1028         for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
1029                     j < cxt.nrealreaders_stress; i++, j++) {
1030                 if (i >= cxt.nrealwriters_stress)
1031                         goto create_reader;
1032
1033                 /* Create writer. */
1034                 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
1035                                                   writer_tasks[i]);
1036                 if (firsterr)
1037                         goto unwind;
1038
1039         create_reader:
1040                 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
1041                         continue;
1042                 /* Create reader. */
1043                 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
1044                                                   reader_tasks[j]);
1045                 if (firsterr)
1046                         goto unwind;
1047         }
1048         if (stat_interval > 0) {
1049                 firsterr = torture_create_kthread(lock_torture_stats, NULL,
1050                                                   stats_task);
1051                 if (firsterr)
1052                         goto unwind;
1053         }
1054         torture_init_end();
1055         return 0;
1056
1057 unwind:
1058         torture_init_end();
1059         lock_torture_cleanup();
1060         if (shutdown_secs) {
1061                 WARN_ON(!IS_MODULE(CONFIG_LOCK_TORTURE_TEST));
1062                 kernel_power_off();
1063         }
1064         return firsterr;
1065 }
1066
1067 module_init(lock_torture_init);
1068 module_exit(lock_torture_cleanup);