lib/locking-selftest.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * lib/locking-selftest.c
   4  *
   5  * Testsuite for various locking APIs: spinlocks, rwlocks,
   6  * mutexes and rw-semaphores.
   7  *
   8  * It is checking both false positives and false negatives.
   9  *
  10  * Started by Ingo Molnar:
  11  *
  12  *  Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  13  */
  14 #include <linux/rwsem.h>
  15 #include <linux/mutex.h>
  16 #include <linux/ww_mutex.h>
  17 #include <linux/sched.h>
  18 #include <linux/delay.h>
  19 #include <linux/lockdep.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/kallsyms.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/debug_locks.h>
  24 #include <linux/irqflags.h>
  25 #include <linux/rtmutex.h>
  26
  27 /*
  28  * Change this to 1 if you want to see the failure printouts:
  29  */
  30 static unsigned int debug_locks_verbose;
  31 unsigned int force_read_lock_recursive;
  32
  33 static DEFINE_WD_CLASS(ww_lockdep);
  34
  35 static int __init setup_debug_locks_verbose(char *str)
  36 {
  37         get_option(&str, &debug_locks_verbose);
  38
  39         return 1;
  40 }
  41
  42 __setup("debug_locks_verbose=", setup_debug_locks_verbose);
  43
  44 #define FAILURE         0
  45 #define SUCCESS         1
  46
  47 #define LOCKTYPE_SPIN   0x1
  48 #define LOCKTYPE_RWLOCK 0x2
  49 #define LOCKTYPE_MUTEX  0x4
  50 #define LOCKTYPE_RWSEM  0x8
  51 #define LOCKTYPE_WW     0x10
  52 #define LOCKTYPE_RTMUTEX 0x20
  53
  54 static struct ww_acquire_ctx t, t2;
  55 static struct ww_mutex o, o2, o3;
  56
  57 /*
  58  * Normal standalone locks, for the circular and irq-context
  59  * dependency tests:
  60  */
  61 static DEFINE_RAW_SPINLOCK(lock_A);
  62 static DEFINE_RAW_SPINLOCK(lock_B);
  63 static DEFINE_RAW_SPINLOCK(lock_C);
  64 static DEFINE_RAW_SPINLOCK(lock_D);
  65
  66 static DEFINE_RWLOCK(rwlock_A);
  67 static DEFINE_RWLOCK(rwlock_B);
  68 static DEFINE_RWLOCK(rwlock_C);
  69 static DEFINE_RWLOCK(rwlock_D);
  70
  71 static DEFINE_MUTEX(mutex_A);
  72 static DEFINE_MUTEX(mutex_B);
  73 static DEFINE_MUTEX(mutex_C);
  74 static DEFINE_MUTEX(mutex_D);
  75
  76 static DECLARE_RWSEM(rwsem_A);
  77 static DECLARE_RWSEM(rwsem_B);
  78 static DECLARE_RWSEM(rwsem_C);
  79 static DECLARE_RWSEM(rwsem_D);
  80
  81 #ifdef CONFIG_RT_MUTEXES
  82
  83 static DEFINE_RT_MUTEX(rtmutex_A);
  84 static DEFINE_RT_MUTEX(rtmutex_B);
  85 static DEFINE_RT_MUTEX(rtmutex_C);
  86 static DEFINE_RT_MUTEX(rtmutex_D);
  87
  88 #endif
  89
  90 /*
  91  * Locks that we initialize dynamically as well so that
  92  * e.g. X1 and X2 becomes two instances of the same class,
  93  * but X* and Y* are different classes. We do this so that
  94  * we do not trigger a real lockup:
  95  */
  96 static DEFINE_RAW_SPINLOCK(lock_X1);
  97 static DEFINE_RAW_SPINLOCK(lock_X2);
  98 static DEFINE_RAW_SPINLOCK(lock_Y1);
  99 static DEFINE_RAW_SPINLOCK(lock_Y2);
 100 static DEFINE_RAW_SPINLOCK(lock_Z1);
 101 static DEFINE_RAW_SPINLOCK(lock_Z2);
 102
 103 static DEFINE_RWLOCK(rwlock_X1);
 104 static DEFINE_RWLOCK(rwlock_X2);
 105 static DEFINE_RWLOCK(rwlock_Y1);
 106 static DEFINE_RWLOCK(rwlock_Y2);
 107 static DEFINE_RWLOCK(rwlock_Z1);
 108 static DEFINE_RWLOCK(rwlock_Z2);
 109
 110 static DEFINE_MUTEX(mutex_X1);
 111 static DEFINE_MUTEX(mutex_X2);
 112 static DEFINE_MUTEX(mutex_Y1);
 113 static DEFINE_MUTEX(mutex_Y2);
 114 static DEFINE_MUTEX(mutex_Z1);
 115 static DEFINE_MUTEX(mutex_Z2);
 116
 117 static DECLARE_RWSEM(rwsem_X1);
 118 static DECLARE_RWSEM(rwsem_X2);
 119 static DECLARE_RWSEM(rwsem_Y1);
 120 static DECLARE_RWSEM(rwsem_Y2);
 121 static DECLARE_RWSEM(rwsem_Z1);
 122 static DECLARE_RWSEM(rwsem_Z2);
 123
 124 #ifdef CONFIG_RT_MUTEXES
 125
 126 static DEFINE_RT_MUTEX(rtmutex_X1);
 127 static DEFINE_RT_MUTEX(rtmutex_X2);
 128 static DEFINE_RT_MUTEX(rtmutex_Y1);
 129 static DEFINE_RT_MUTEX(rtmutex_Y2);
 130 static DEFINE_RT_MUTEX(rtmutex_Z1);
 131 static DEFINE_RT_MUTEX(rtmutex_Z2);
 132
 133 #endif
 134
 135 /*
 136  * non-inlined runtime initializers, to let separate locks share
 137  * the same lock-class:
 138  */
 139 #define INIT_CLASS_FUNC(class)                          \
 140 static noinline void                                    \
 141 init_class_##class(raw_spinlock_t *lock, rwlock_t *rwlock, \
 142         struct mutex *mutex, struct rw_semaphore *rwsem)\
 143 {                                                       \
 144         raw_spin_lock_init(lock);                       \
 145         rwlock_init(rwlock);                            \
 146         mutex_init(mutex);                              \
 147         init_rwsem(rwsem);                              \
 148 }
 149
 150 INIT_CLASS_FUNC(X)
 151 INIT_CLASS_FUNC(Y)
 152 INIT_CLASS_FUNC(Z)
 153
 154 static void init_shared_classes(void)
 155 {
 156 #ifdef CONFIG_RT_MUTEXES
 157         static struct lock_class_key rt_X, rt_Y, rt_Z;
 158
 159         __rt_mutex_init(&rtmutex_X1, __func__, &rt_X);
 160         __rt_mutex_init(&rtmutex_X2, __func__, &rt_X);
 161         __rt_mutex_init(&rtmutex_Y1, __func__, &rt_Y);
 162         __rt_mutex_init(&rtmutex_Y2, __func__, &rt_Y);
 163         __rt_mutex_init(&rtmutex_Z1, __func__, &rt_Z);
 164         __rt_mutex_init(&rtmutex_Z2, __func__, &rt_Z);
 165 #endif
 166
 167         init_class_X(&lock_X1, &rwlock_X1, &mutex_X1, &rwsem_X1);
 168         init_class_X(&lock_X2, &rwlock_X2, &mutex_X2, &rwsem_X2);
 169
 170         init_class_Y(&lock_Y1, &rwlock_Y1, &mutex_Y1, &rwsem_Y1);
 171         init_class_Y(&lock_Y2, &rwlock_Y2, &mutex_Y2, &rwsem_Y2);
 172
 173         init_class_Z(&lock_Z1, &rwlock_Z1, &mutex_Z1, &rwsem_Z1);
 174         init_class_Z(&lock_Z2, &rwlock_Z2, &mutex_Z2, &rwsem_Z2);
 175 }
 176
 177 /*
 178  * For spinlocks and rwlocks we also do hardirq-safe / softirq-safe tests.
 179  * The following functions use a lock from a simulated hardirq/softirq
 180  * context, causing the locks to be marked as hardirq-safe/softirq-safe:
 181  */
 182
 183 #define HARDIRQ_DISABLE         local_irq_disable
 184 #define HARDIRQ_ENABLE          local_irq_enable
 185
 186 #define HARDIRQ_ENTER()                         \
 187         local_irq_disable();                    \
 188         __irq_enter();                          \
 189         WARN_ON(!in_irq());
 190
 191 #define HARDIRQ_EXIT()                          \
 192         __irq_exit();                           \
 193         local_irq_enable();
 194
 195 #define SOFTIRQ_DISABLE         local_bh_disable
 196 #define SOFTIRQ_ENABLE          local_bh_enable
 197
 198 #define SOFTIRQ_ENTER()                         \
 199                 local_bh_disable();             \
 200                 local_irq_disable();            \
 201                 lockdep_softirq_enter();        \
 202                 WARN_ON(!in_softirq());
 203
 204 #define SOFTIRQ_EXIT()                          \
 205                 lockdep_softirq_exit();         \
 206                 local_irq_enable();             \
 207                 local_bh_enable();
 208
 209 /*
 210  * Shortcuts for lock/unlock API variants, to keep
 211  * the testcases compact:
 212  */
 213 #define L(x)                    raw_spin_lock(&lock_##x)
 214 #define U(x)                    raw_spin_unlock(&lock_##x)
 215 #define LU(x)                   L(x); U(x)
 216 #define SI(x)                   raw_spin_lock_init(&lock_##x)
 217
 218 #define WL(x)                   write_lock(&rwlock_##x)
 219 #define WU(x)                   write_unlock(&rwlock_##x)
 220 #define WLU(x)                  WL(x); WU(x)
 221
 222 #define RL(x)                   read_lock(&rwlock_##x)
 223 #define RU(x)                   read_unlock(&rwlock_##x)
 224 #define RLU(x)                  RL(x); RU(x)
 225 #define RWI(x)                  rwlock_init(&rwlock_##x)
 226
 227 #define ML(x)                   mutex_lock(&mutex_##x)
 228 #define MU(x)                   mutex_unlock(&mutex_##x)
 229 #define MI(x)                   mutex_init(&mutex_##x)
 230
 231 #define RTL(x)                  rt_mutex_lock(&rtmutex_##x)
 232 #define RTU(x)                  rt_mutex_unlock(&rtmutex_##x)
 233 #define RTI(x)                  rt_mutex_init(&rtmutex_##x)
 234
 235 #define WSL(x)                  down_write(&rwsem_##x)
 236 #define WSU(x)                  up_write(&rwsem_##x)
 237
 238 #define RSL(x)                  down_read(&rwsem_##x)
 239 #define RSU(x)                  up_read(&rwsem_##x)
 240 #define RWSI(x)                 init_rwsem(&rwsem_##x)
 241
 242 #ifndef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
 243 #define WWAI(x)                 ww_acquire_init(x, &ww_lockdep)
 244 #else
 245 #define WWAI(x)                 do { ww_acquire_init(x, &ww_lockdep); (x)->deadlock_inject_countdown = ~0U; } while (0)
 246 #endif
 247 #define WWAD(x)                 ww_acquire_done(x)
 248 #define WWAF(x)                 ww_acquire_fini(x)
 249
 250 #define WWL(x, c)               ww_mutex_lock(x, c)
 251 #define WWT(x)                  ww_mutex_trylock(x)
 252 #define WWL1(x)                 ww_mutex_lock(x, NULL)
 253 #define WWU(x)                  ww_mutex_unlock(x)
 254
 255
 256 #define LOCK_UNLOCK_2(x,y)      LOCK(x); LOCK(y); UNLOCK(y); UNLOCK(x)
 257
 258 /*
 259  * Generate different permutations of the same testcase, using
 260  * the same basic lock-dependency/state events:
 261  */
 262
 263 #define GENERATE_TESTCASE(name)                 \
 264                                                 \
 265 static void name(void) { E(); }
 266
 267 #define GENERATE_PERMUTATIONS_2_EVENTS(name)    \
 268                                                 \
 269 static void name##_12(void) { E1(); E2(); }     \
 270 static void name##_21(void) { E2(); E1(); }
 271
 272 #define GENERATE_PERMUTATIONS_3_EVENTS(name)            \
 273                                                         \
 274 static void name##_123(void) { E1(); E2(); E3(); }      \
 275 static void name##_132(void) { E1(); E3(); E2(); }      \
 276 static void name##_213(void) { E2(); E1(); E3(); }      \
 277 static void name##_231(void) { E2(); E3(); E1(); }      \
 278 static void name##_312(void) { E3(); E1(); E2(); }      \
 279 static void name##_321(void) { E3(); E2(); E1(); }
 280
 281 /*
 282  * AA deadlock:
 283  */
 284
 285 #define E()                                     \
 286                                                 \
 287         LOCK(X1);                               \
 288         LOCK(X2); /* this one should fail */
 289
 290 /*
 291  * 6 testcases:
 292  */
 293 #include "locking-selftest-spin.h"
 294 GENERATE_TESTCASE(AA_spin)
 295 #include "locking-selftest-wlock.h"
 296 GENERATE_TESTCASE(AA_wlock)
 297 #include "locking-selftest-rlock.h"
 298 GENERATE_TESTCASE(AA_rlock)
 299 #include "locking-selftest-mutex.h"
 300 GENERATE_TESTCASE(AA_mutex)
 301 #include "locking-selftest-wsem.h"
 302 GENERATE_TESTCASE(AA_wsem)
 303 #include "locking-selftest-rsem.h"
 304 GENERATE_TESTCASE(AA_rsem)
 305
 306 #ifdef CONFIG_RT_MUTEXES
 307 #include "locking-selftest-rtmutex.h"
 308 GENERATE_TESTCASE(AA_rtmutex);
 309 #endif
 310
 311 #undef E
 312
 313 /*
 314  * Special-case for read-locking, they are
 315  * allowed to recurse on the same lock class:
 316  */
 317 static void rlock_AA1(void)
 318 {
 319         RL(X1);
 320         RL(X1); // this one should NOT fail
 321 }
 322
 323 static void rlock_AA1B(void)
 324 {
 325         RL(X1);
 326         RL(X2); // this one should NOT fail
 327 }
 328
 329 static void rsem_AA1(void)
 330 {
 331         RSL(X1);
 332         RSL(X1); // this one should fail
 333 }
 334
 335 static void rsem_AA1B(void)
 336 {
 337         RSL(X1);
 338         RSL(X2); // this one should fail
 339 }
 340 /*
 341  * The mixing of read and write locks is not allowed:
 342  */
 343 static void rlock_AA2(void)
 344 {
 345         RL(X1);
 346         WL(X2); // this one should fail
 347 }
 348
 349 static void rsem_AA2(void)
 350 {
 351         RSL(X1);
 352         WSL(X2); // this one should fail
 353 }
 354
 355 static void rlock_AA3(void)
 356 {
 357         WL(X1);
 358         RL(X2); // this one should fail
 359 }
 360
 361 static void rsem_AA3(void)
 362 {
 363         WSL(X1);
 364         RSL(X2); // this one should fail
 365 }
 366
 367 /*
 368  * read_lock(A)
 369  * spin_lock(B)
 370  *              spin_lock(B)
 371  *              write_lock(A)
 372  */
 373 static void rlock_ABBA1(void)
 374 {
 375         RL(X1);
 376         L(Y1);
 377         U(Y1);
 378         RU(X1);
 379
 380         L(Y1);
 381         WL(X1);
 382         WU(X1);
 383         U(Y1); // should fail
 384 }
 385
 386 static void rwsem_ABBA1(void)
 387 {
 388         RSL(X1);
 389         ML(Y1);
 390         MU(Y1);
 391         RSU(X1);
 392
 393         ML(Y1);
 394         WSL(X1);
 395         WSU(X1);
 396         MU(Y1); // should fail
 397 }
 398
 399 /*
 400  * read_lock(A)
 401  * spin_lock(B)
 402  *              spin_lock(B)
 403  *              write_lock(A)
 404  *
 405  * This test case is aimed at poking whether the chain cache prevents us from
 406  * detecting a read-lock/lock-write deadlock: if the chain cache doesn't differ
 407  * read/write locks, the following case may happen
 408  *
 409  *      { read_lock(A)->lock(B) dependency exists }
 410  *
 411  *      P0:
 412  *      lock(B);
 413  *      read_lock(A);
 414  *
 415  *      { Not a deadlock, B -> A is added in the chain cache }
 416  *
 417  *      P1:
 418  *      lock(B);
 419  *      write_lock(A);
 420  *
 421  *      { B->A found in chain cache, not reported as a deadlock }
 422  *
 423  */
 424 static void rlock_chaincache_ABBA1(void)
 425 {
 426         RL(X1);
 427         L(Y1);
 428         U(Y1);
 429         RU(X1);
 430
 431         L(Y1);
 432         RL(X1);
 433         RU(X1);
 434         U(Y1);
 435
 436         L(Y1);
 437         WL(X1);
 438         WU(X1);
 439         U(Y1); // should fail
 440 }
 441
 442 /*
 443  * read_lock(A)
 444  * spin_lock(B)
 445  *              spin_lock(B)
 446  *              read_lock(A)
 447  */
 448 static void rlock_ABBA2(void)
 449 {
 450         RL(X1);
 451         L(Y1);
 452         U(Y1);
 453         RU(X1);
 454
 455         L(Y1);
 456         RL(X1);
 457         RU(X1);
 458         U(Y1); // should NOT fail
 459 }
 460
 461 static void rwsem_ABBA2(void)
 462 {
 463         RSL(X1);
 464         ML(Y1);
 465         MU(Y1);
 466         RSU(X1);
 467
 468         ML(Y1);
 469         RSL(X1);
 470         RSU(X1);
 471         MU(Y1); // should fail
 472 }
 473
 474
 475 /*
 476  * write_lock(A)
 477  * spin_lock(B)
 478  *              spin_lock(B)
 479  *              write_lock(A)
 480  */
 481 static void rlock_ABBA3(void)
 482 {
 483         WL(X1);
 484         L(Y1);
 485         U(Y1);
 486         WU(X1);
 487
 488         L(Y1);
 489         WL(X1);
 490         WU(X1);
 491         U(Y1); // should fail
 492 }
 493
 494 static void rwsem_ABBA3(void)
 495 {
 496         WSL(X1);
 497         ML(Y1);
 498         MU(Y1);
 499         WSU(X1);
 500
 501         ML(Y1);
 502         WSL(X1);
 503         WSU(X1);
 504         MU(Y1); // should fail
 505 }
 506
 507 /*
 508  * ABBA deadlock:
 509  */
 510
 511 #define E()                                     \
 512                                                 \
 513         LOCK_UNLOCK_2(A, B);                    \
 514         LOCK_UNLOCK_2(B, A); /* fail */
 515
 516 /*
 517  * 6 testcases:
 518  */
 519 #include "locking-selftest-spin.h"
 520 GENERATE_TESTCASE(ABBA_spin)
 521 #include "locking-selftest-wlock.h"
 522 GENERATE_TESTCASE(ABBA_wlock)
 523 #include "locking-selftest-rlock.h"
 524 GENERATE_TESTCASE(ABBA_rlock)
 525 #include "locking-selftest-mutex.h"
 526 GENERATE_TESTCASE(ABBA_mutex)
 527 #include "locking-selftest-wsem.h"
 528 GENERATE_TESTCASE(ABBA_wsem)
 529 #include "locking-selftest-rsem.h"
 530 GENERATE_TESTCASE(ABBA_rsem)
 531
 532 #ifdef CONFIG_RT_MUTEXES
 533 #include "locking-selftest-rtmutex.h"
 534 GENERATE_TESTCASE(ABBA_rtmutex);
 535 #endif
 536
 537 #undef E
 538
 539 /*
 540  * AB BC CA deadlock:
 541  */
 542
 543 #define E()                                     \
 544                                                 \
 545         LOCK_UNLOCK_2(A, B);                    \
 546         LOCK_UNLOCK_2(B, C);                    \
 547         LOCK_UNLOCK_2(C, A); /* fail */
 548
 549 /*
 550  * 6 testcases:
 551  */
 552 #include "locking-selftest-spin.h"
 553 GENERATE_TESTCASE(ABBCCA_spin)
 554 #include "locking-selftest-wlock.h"
 555 GENERATE_TESTCASE(ABBCCA_wlock)
 556 #include "locking-selftest-rlock.h"
 557 GENERATE_TESTCASE(ABBCCA_rlock)
 558 #include "locking-selftest-mutex.h"
 559 GENERATE_TESTCASE(ABBCCA_mutex)
 560 #include "locking-selftest-wsem.h"
 561 GENERATE_TESTCASE(ABBCCA_wsem)
 562 #include "locking-selftest-rsem.h"
 563 GENERATE_TESTCASE(ABBCCA_rsem)
 564
 565 #ifdef CONFIG_RT_MUTEXES
 566 #include "locking-selftest-rtmutex.h"
 567 GENERATE_TESTCASE(ABBCCA_rtmutex);
 568 #endif
 569
 570 #undef E
 571
 572 /*
 573  * AB CA BC deadlock:
 574  */
 575
 576 #define E()                                     \
 577                                                 \
 578         LOCK_UNLOCK_2(A, B);                    \
 579         LOCK_UNLOCK_2(C, A);                    \
 580         LOCK_UNLOCK_2(B, C); /* fail */
 581
 582 /*
 583  * 6 testcases:
 584  */
 585 #include "locking-selftest-spin.h"
 586 GENERATE_TESTCASE(ABCABC_spin)
 587 #include "locking-selftest-wlock.h"
 588 GENERATE_TESTCASE(ABCABC_wlock)
 589 #include "locking-selftest-rlock.h"
 590 GENERATE_TESTCASE(ABCABC_rlock)
 591 #include "locking-selftest-mutex.h"
 592 GENERATE_TESTCASE(ABCABC_mutex)
 593 #include "locking-selftest-wsem.h"
 594 GENERATE_TESTCASE(ABCABC_wsem)
 595 #include "locking-selftest-rsem.h"
 596 GENERATE_TESTCASE(ABCABC_rsem)
 597
 598 #ifdef CONFIG_RT_MUTEXES
 599 #include "locking-selftest-rtmutex.h"
 600 GENERATE_TESTCASE(ABCABC_rtmutex);
 601 #endif
 602
 603 #undef E
 604
 605 /*
 606  * AB BC CD DA deadlock:
 607  */
 608
 609 #define E()                                     \
 610                                                 \
 611         LOCK_UNLOCK_2(A, B);                    \
 612         LOCK_UNLOCK_2(B, C);                    \
 613         LOCK_UNLOCK_2(C, D);                    \
 614         LOCK_UNLOCK_2(D, A); /* fail */
 615
 616 /*
 617  * 6 testcases:
 618  */
 619 #include "locking-selftest-spin.h"
 620 GENERATE_TESTCASE(ABBCCDDA_spin)
 621 #include "locking-selftest-wlock.h"
 622 GENERATE_TESTCASE(ABBCCDDA_wlock)
 623 #include "locking-selftest-rlock.h"
 624 GENERATE_TESTCASE(ABBCCDDA_rlock)
 625 #include "locking-selftest-mutex.h"
 626 GENERATE_TESTCASE(ABBCCDDA_mutex)
 627 #include "locking-selftest-wsem.h"
 628 GENERATE_TESTCASE(ABBCCDDA_wsem)
 629 #include "locking-selftest-rsem.h"
 630 GENERATE_TESTCASE(ABBCCDDA_rsem)
 631
 632 #ifdef CONFIG_RT_MUTEXES
 633 #include "locking-selftest-rtmutex.h"
 634 GENERATE_TESTCASE(ABBCCDDA_rtmutex);
 635 #endif
 636
 637 #undef E
 638
 639 /*
 640  * AB CD BD DA deadlock:
 641  */
 642 #define E()                                     \
 643                                                 \
 644         LOCK_UNLOCK_2(A, B);                    \
 645         LOCK_UNLOCK_2(C, D);                    \
 646         LOCK_UNLOCK_2(B, D);                    \
 647         LOCK_UNLOCK_2(D, A); /* fail */
 648
 649 /*
 650  * 6 testcases:
 651  */
 652 #include "locking-selftest-spin.h"
 653 GENERATE_TESTCASE(ABCDBDDA_spin)
 654 #include "locking-selftest-wlock.h"
 655 GENERATE_TESTCASE(ABCDBDDA_wlock)
 656 #include "locking-selftest-rlock.h"
 657 GENERATE_TESTCASE(ABCDBDDA_rlock)
 658 #include "locking-selftest-mutex.h"
 659 GENERATE_TESTCASE(ABCDBDDA_mutex)
 660 #include "locking-selftest-wsem.h"
 661 GENERATE_TESTCASE(ABCDBDDA_wsem)
 662 #include "locking-selftest-rsem.h"
 663 GENERATE_TESTCASE(ABCDBDDA_rsem)
 664
 665 #ifdef CONFIG_RT_MUTEXES
 666 #include "locking-selftest-rtmutex.h"
 667 GENERATE_TESTCASE(ABCDBDDA_rtmutex);
 668 #endif
 669
 670 #undef E
 671
 672 /*
 673  * AB CD BC DA deadlock:
 674  */
 675 #define E()                                     \
 676                                                 \
 677         LOCK_UNLOCK_2(A, B);                    \
 678         LOCK_UNLOCK_2(C, D);                    \
 679         LOCK_UNLOCK_2(B, C);                    \
 680         LOCK_UNLOCK_2(D, A); /* fail */
 681
 682 /*
 683  * 6 testcases:
 684  */
 685 #include "locking-selftest-spin.h"
 686 GENERATE_TESTCASE(ABCDBCDA_spin)
 687 #include "locking-selftest-wlock.h"
 688 GENERATE_TESTCASE(ABCDBCDA_wlock)
 689 #include "locking-selftest-rlock.h"
 690 GENERATE_TESTCASE(ABCDBCDA_rlock)
 691 #include "locking-selftest-mutex.h"
 692 GENERATE_TESTCASE(ABCDBCDA_mutex)
 693 #include "locking-selftest-wsem.h"
 694 GENERATE_TESTCASE(ABCDBCDA_wsem)
 695 #include "locking-selftest-rsem.h"
 696 GENERATE_TESTCASE(ABCDBCDA_rsem)
 697
 698 #ifdef CONFIG_RT_MUTEXES
 699 #include "locking-selftest-rtmutex.h"
 700 GENERATE_TESTCASE(ABCDBCDA_rtmutex);
 701 #endif
 702
 703 #undef E
 704
 705 /*
 706  * Double unlock:
 707  */
 708 #define E()                                     \
 709                                                 \
 710         LOCK(A);                                \
 711         UNLOCK(A);                              \
 712         UNLOCK(A); /* fail */
 713
 714 /*
 715  * 6 testcases:
 716  */
 717 #include "locking-selftest-spin.h"
 718 GENERATE_TESTCASE(double_unlock_spin)
 719 #include "locking-selftest-wlock.h"
 720 GENERATE_TESTCASE(double_unlock_wlock)
 721 #include "locking-selftest-rlock.h"
 722 GENERATE_TESTCASE(double_unlock_rlock)
 723 #include "locking-selftest-mutex.h"
 724 GENERATE_TESTCASE(double_unlock_mutex)
 725 #include "locking-selftest-wsem.h"
 726 GENERATE_TESTCASE(double_unlock_wsem)
 727 #include "locking-selftest-rsem.h"
 728 GENERATE_TESTCASE(double_unlock_rsem)
 729
 730 #ifdef CONFIG_RT_MUTEXES
 731 #include "locking-selftest-rtmutex.h"
 732 GENERATE_TESTCASE(double_unlock_rtmutex);
 733 #endif
 734
 735 #undef E
 736
 737 /*
 738  * initializing a held lock:
 739  */
 740 #define E()                                     \
 741                                                 \
 742         LOCK(A);                                \
 743         INIT(A); /* fail */
 744
 745 /*
 746  * 6 testcases:
 747  */
 748 #include "locking-selftest-spin.h"
 749 GENERATE_TESTCASE(init_held_spin)
 750 #include "locking-selftest-wlock.h"
 751 GENERATE_TESTCASE(init_held_wlock)
 752 #include "locking-selftest-rlock.h"
 753 GENERATE_TESTCASE(init_held_rlock)
 754 #include "locking-selftest-mutex.h"
 755 GENERATE_TESTCASE(init_held_mutex)
 756 #include "locking-selftest-wsem.h"
 757 GENERATE_TESTCASE(init_held_wsem)
 758 #include "locking-selftest-rsem.h"
 759 GENERATE_TESTCASE(init_held_rsem)
 760
 761 #ifdef CONFIG_RT_MUTEXES
 762 #include "locking-selftest-rtmutex.h"
 763 GENERATE_TESTCASE(init_held_rtmutex);
 764 #endif
 765
 766 #undef E
 767
 768 /*
 769  * locking an irq-safe lock with irqs enabled:
 770  */
 771 #define E1()                            \
 772                                         \
 773         IRQ_ENTER();                    \
 774         LOCK(A);                        \
 775         UNLOCK(A);                      \
 776         IRQ_EXIT();
 777
 778 #define E2()                            \
 779                                         \
 780         LOCK(A);                        \
 781         UNLOCK(A);
 782
 783 /*
 784  * Generate 24 testcases:
 785  */
 786 #include "locking-selftest-spin-hardirq.h"
 787 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin)
 788
 789 #include "locking-selftest-rlock-hardirq.h"
 790 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock)
 791
 792 #include "locking-selftest-wlock-hardirq.h"
 793 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_wlock)
 794
 795 #include "locking-selftest-spin-softirq.h"
 796 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_spin)
 797
 798 #include "locking-selftest-rlock-softirq.h"
 799 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_rlock)
 800
 801 #include "locking-selftest-wlock-softirq.h"
 802 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock)
 803
 804 #undef E1
 805 #undef E2
 806
 807 /*
 808  * Enabling hardirqs with a softirq-safe lock held:
 809  */
 810 #define E1()                            \
 811                                         \
 812         SOFTIRQ_ENTER();                \
 813         LOCK(A);                        \
 814         UNLOCK(A);                      \
 815         SOFTIRQ_EXIT();
 816
 817 #define E2()                            \
 818                                         \
 819         HARDIRQ_DISABLE();              \
 820         LOCK(A);                        \
 821         HARDIRQ_ENABLE();               \
 822         UNLOCK(A);
 823
 824 /*
 825  * Generate 12 testcases:
 826  */
 827 #include "locking-selftest-spin.h"
 828 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_spin)
 829
 830 #include "locking-selftest-wlock.h"
 831 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_wlock)
 832
 833 #include "locking-selftest-rlock.h"
 834 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_rlock)
 835
 836 #undef E1
 837 #undef E2
 838
 839 /*
 840  * Enabling irqs with an irq-safe lock held:
 841  */
 842 #define E1()                            \
 843                                         \
 844         IRQ_ENTER();                    \
 845         LOCK(A);                        \
 846         UNLOCK(A);                      \
 847         IRQ_EXIT();
 848
 849 #define E2()                            \
 850                                         \
 851         IRQ_DISABLE();                  \
 852         LOCK(A);                        \
 853         IRQ_ENABLE();                   \
 854         UNLOCK(A);
 855
 856 /*
 857  * Generate 24 testcases:
 858  */
 859 #include "locking-selftest-spin-hardirq.h"
 860 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin)
 861
 862 #include "locking-selftest-rlock-hardirq.h"
 863 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock)
 864
 865 #include "locking-selftest-wlock-hardirq.h"
 866 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_wlock)
 867
 868 #include "locking-selftest-spin-softirq.h"
 869 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_spin)
 870
 871 #include "locking-selftest-rlock-softirq.h"
 872 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_rlock)
 873
 874 #include "locking-selftest-wlock-softirq.h"
 875 GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock)
 876
 877 #undef E1
 878 #undef E2
 879
 880 /*
 881  * Acquiring a irq-unsafe lock while holding an irq-safe-lock:
 882  */
 883 #define E1()                            \
 884                                         \
 885         LOCK(A);                        \
 886         LOCK(B);                        \
 887         UNLOCK(B);                      \
 888         UNLOCK(A);                      \
 889
 890 #define E2()                            \
 891                                         \
 892         LOCK(B);                        \
 893         UNLOCK(B);
 894
 895 #define E3()                            \
 896                                         \
 897         IRQ_ENTER();                    \
 898         LOCK(A);                        \
 899         UNLOCK(A);                      \
 900         IRQ_EXIT();
 901
 902 /*
 903  * Generate 36 testcases:
 904  */
 905 #include "locking-selftest-spin-hardirq.h"
 906 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin)
 907
 908 #include "locking-selftest-rlock-hardirq.h"
 909 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock)
 910
 911 #include "locking-selftest-wlock-hardirq.h"
 912 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_wlock)
 913
 914 #include "locking-selftest-spin-softirq.h"
 915 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_spin)
 916
 917 #include "locking-selftest-rlock-softirq.h"
 918 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_rlock)
 919
 920 #include "locking-selftest-wlock-softirq.h"
 921 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock)
 922
 923 #undef E1
 924 #undef E2
 925 #undef E3
 926
 927 /*
 928  * If a lock turns into softirq-safe, but earlier it took
 929  * a softirq-unsafe lock:
 930  */
 931
 932 #define E1()                            \
 933         IRQ_DISABLE();                  \
 934         LOCK(A);                        \
 935         LOCK(B);                        \
 936         UNLOCK(B);                      \
 937         UNLOCK(A);                      \
 938         IRQ_ENABLE();
 939
 940 #define E2()                            \
 941         LOCK(B);                        \
 942         UNLOCK(B);
 943
 944 #define E3()                            \
 945         IRQ_ENTER();                    \
 946         LOCK(A);                        \
 947         UNLOCK(A);                      \
 948         IRQ_EXIT();
 949
 950 /*
 951  * Generate 36 testcases:
 952  */
 953 #include "locking-selftest-spin-hardirq.h"
 954 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin)
 955
 956 #include "locking-selftest-rlock-hardirq.h"
 957 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock)
 958
 959 #include "locking-selftest-wlock-hardirq.h"
 960 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_wlock)
 961
 962 #include "locking-selftest-spin-softirq.h"
 963 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_spin)
 964
 965 #include "locking-selftest-rlock-softirq.h"
 966 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_rlock)
 967
 968 #include "locking-selftest-wlock-softirq.h"
 969 GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock)
 970
 971 #undef E1
 972 #undef E2
 973 #undef E3
 974
 975 /*
 976  * read-lock / write-lock irq inversion.
 977  *
 978  * Deadlock scenario:
 979  *
 980  * CPU#1 is at #1, i.e. it has write-locked A, but has not
 981  * taken B yet.
 982  *
 983  * CPU#2 is at #2, i.e. it has locked B.
 984  *
 985  * Hardirq hits CPU#2 at point #2 and is trying to read-lock A.
 986  *
 987  * The deadlock occurs because CPU#1 will spin on B, and CPU#2
 988  * will spin on A.
 989  */
 990
 991 #define E1()                            \
 992                                         \
 993         IRQ_DISABLE();                  \
 994         WL(A);                          \
 995         LOCK(B);                        \
 996         UNLOCK(B);                      \
 997         WU(A);                          \
 998         IRQ_ENABLE();
 999
1000 #define E2()                            \
1001                                         \
1002         LOCK(B);                        \
1003         UNLOCK(B);
1004
1005 #define E3()                            \
1006                                         \
1007         IRQ_ENTER();                    \
1008         RL(A);                          \
1009         RU(A);                          \
1010         IRQ_EXIT();
1011
1012 /*
1013  * Generate 36 testcases:
1014  */
1015 #include "locking-selftest-spin-hardirq.h"
1016 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_spin)
1017
1018 #include "locking-selftest-rlock-hardirq.h"
1019 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_rlock)
1020
1021 #include "locking-selftest-wlock-hardirq.h"
1022 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_wlock)
1023
1024 #include "locking-selftest-spin-softirq.h"
1025 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_spin)
1026
1027 #include "locking-selftest-rlock-softirq.h"
1028 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_rlock)
1029
1030 #include "locking-selftest-wlock-softirq.h"
1031 GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_wlock)
1032
1033 #undef E1
1034 #undef E2
1035 #undef E3
1036
1037 /*
1038  * write-read / write-read / write-read deadlock even if read is recursive
1039  */
1040
1041 #define E1()                            \
1042                                         \
1043         WL(X1);                         \
1044         RL(Y1);                         \
1045         RU(Y1);                         \
1046         WU(X1);
1047
1048 #define E2()                            \
1049                                         \
1050         WL(Y1);                         \
1051         RL(Z1);                         \
1052         RU(Z1);                         \
1053         WU(Y1);
1054
1055 #define E3()                            \
1056                                         \
1057         WL(Z1);                         \
1058         RL(X1);                         \
1059         RU(X1);                         \
1060         WU(Z1);
1061
1062 #include "locking-selftest-rlock.h"
1063 GENERATE_PERMUTATIONS_3_EVENTS(W1R2_W2R3_W3R1)
1064
1065 #undef E1
1066 #undef E2
1067 #undef E3
1068
1069 /*
1070  * write-write / read-read / write-read deadlock even if read is recursive
1071  */
1072
1073 #define E1()                            \
1074                                         \
1075         WL(X1);                         \
1076         WL(Y1);                         \
1077         WU(Y1);                         \
1078         WU(X1);
1079
1080 #define E2()                            \
1081                                         \
1082         RL(Y1);                         \
1083         RL(Z1);                         \
1084         RU(Z1);                         \
1085         RU(Y1);
1086
1087 #define E3()                            \
1088                                         \
1089         WL(Z1);                         \
1090         RL(X1);                         \
1091         RU(X1);                         \
1092         WU(Z1);
1093
1094 #include "locking-selftest-rlock.h"
1095 GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_W3R1)
1096
1097 #undef E1
1098 #undef E2
1099 #undef E3
1100
1101 /*
1102  * write-write / read-read / read-write is not deadlock when read is recursive
1103  */
1104
1105 #define E1()                            \
1106                                         \
1107         WL(X1);                         \
1108         WL(Y1);                         \
1109         WU(Y1);                         \
1110         WU(X1);
1111
1112 #define E2()                            \
1113                                         \
1114         RL(Y1);                         \
1115         RL(Z1);                         \
1116         RU(Z1);                         \
1117         RU(Y1);
1118
1119 #define E3()                            \
1120                                         \
1121         RL(Z1);                         \
1122         WL(X1);                         \
1123         WU(X1);                         \
1124         RU(Z1);
1125
1126 #include "locking-selftest-rlock.h"
1127 GENERATE_PERMUTATIONS_3_EVENTS(W1R2_R2R3_W3W1)
1128
1129 #undef E1
1130 #undef E2
1131 #undef E3
1132
1133 /*
1134  * write-read / read-read / write-write is not deadlock when read is recursive
1135  */
1136
1137 #define E1()                            \
1138                                         \
1139         WL(X1);                         \
1140         RL(Y1);                         \
1141         RU(Y1);                         \
1142         WU(X1);
1143
1144 #define E2()                            \
1145                                         \
1146         RL(Y1);                         \
1147         RL(Z1);                         \
1148         RU(Z1);                         \
1149         RU(Y1);
1150
1151 #define E3()                            \
1152                                         \
1153         WL(Z1);                         \
1154         WL(X1);                         \
1155         WU(X1);                         \
1156         WU(Z1);
1157
1158 #include "locking-selftest-rlock.h"
1159 GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_R3W1)
1160
1161 #undef E1
1162 #undef E2
1163 #undef E3
1164 /*
1165  * read-lock / write-lock recursion that is actually safe.
1166  */
1167
1168 #define E1()                            \
1169                                         \
1170         IRQ_DISABLE();                  \
1171         WL(A);                          \
1172         WU(A);                          \
1173         IRQ_ENABLE();
1174
1175 #define E2()                            \
1176                                         \
1177         RL(A);                          \
1178         RU(A);                          \
1179
1180 #define E3()                            \
1181                                         \
1182         IRQ_ENTER();                    \
1183         LOCK(A);                        \
1184         L(B);                           \
1185         U(B);                           \
1186         UNLOCK(A);                      \
1187         IRQ_EXIT();
1188
1189 /*
1190  * Generate 24 testcases:
1191  */
1192 #include "locking-selftest-hardirq.h"
1193 #include "locking-selftest-rlock.h"
1194 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_rlock)
1195
1196 #include "locking-selftest-wlock.h"
1197 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_wlock)
1198
1199 #include "locking-selftest-softirq.h"
1200 #include "locking-selftest-rlock.h"
1201 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_rlock)
1202
1203 #include "locking-selftest-wlock.h"
1204 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_wlock)
1205
1206 #undef E1
1207 #undef E2
1208 #undef E3
1209
1210 /*
1211  * read-lock / write-lock recursion that is unsafe.
1212  */
1213
1214 #define E1()                            \
1215                                         \
1216         IRQ_DISABLE();                  \
1217         L(B);                           \
1218         LOCK(A);                        \
1219         UNLOCK(A);                      \
1220         U(B);                           \
1221         IRQ_ENABLE();
1222
1223 #define E2()                            \
1224                                         \
1225         RL(A);                          \
1226         RU(A);                          \
1227
1228 #define E3()                            \
1229                                         \
1230         IRQ_ENTER();                    \
1231         L(B);                           \
1232         U(B);                           \
1233         IRQ_EXIT();
1234
1235 /*
1236  * Generate 24 testcases:
1237  */
1238 #include "locking-selftest-hardirq.h"
1239 #include "locking-selftest-rlock.h"
1240 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_rlock)
1241
1242 #include "locking-selftest-wlock.h"
1243 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_wlock)
1244
1245 #include "locking-selftest-softirq.h"
1246 #include "locking-selftest-rlock.h"
1247 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_rlock)
1248
1249 #include "locking-selftest-wlock.h"
1250 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_wlock)
1251
1252 #undef E1
1253 #undef E2
1254 #undef E3
1255 /*
1256  * read-lock / write-lock recursion that is unsafe.
1257  *
1258  * A is a ENABLED_*_READ lock
1259  * B is a USED_IN_*_READ lock
1260  *
1261  * read_lock(A);
1262  *                      write_lock(B);
1263  * <interrupt>
1264  * read_lock(B);
1265  *                      write_lock(A); // if this one is read_lock(), no deadlock
1266  */
1267
1268 #define E1()                            \
1269                                         \
1270         IRQ_DISABLE();                  \
1271         WL(B);                          \
1272         LOCK(A);                        \
1273         UNLOCK(A);                      \
1274         WU(B);                          \
1275         IRQ_ENABLE();
1276
1277 #define E2()                            \
1278                                         \
1279         RL(A);                          \
1280         RU(A);                          \
1281
1282 #define E3()                            \
1283                                         \
1284         IRQ_ENTER();                    \
1285         RL(B);                          \
1286         RU(B);                          \
1287         IRQ_EXIT();
1288
1289 /*
1290  * Generate 24 testcases:
1291  */
1292 #include "locking-selftest-hardirq.h"
1293 #include "locking-selftest-rlock.h"
1294 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_rlock)
1295
1296 #include "locking-selftest-wlock.h"
1297 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_wlock)
1298
1299 #include "locking-selftest-softirq.h"
1300 #include "locking-selftest-rlock.h"
1301 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_rlock)
1302
1303 #include "locking-selftest-wlock.h"
1304 GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_wlock)
1305
1306 #ifdef CONFIG_DEBUG_LOCK_ALLOC
1307 # define I_SPINLOCK(x)  lockdep_reset_lock(&lock_##x.dep_map)
1308 # define I_RWLOCK(x)    lockdep_reset_lock(&rwlock_##x.dep_map)
1309 # define I_MUTEX(x)     lockdep_reset_lock(&mutex_##x.dep_map)
1310 # define I_RWSEM(x)     lockdep_reset_lock(&rwsem_##x.dep_map)
1311 # define I_WW(x)        lockdep_reset_lock(&x.dep_map)
1312 #ifdef CONFIG_RT_MUTEXES
1313 # define I_RTMUTEX(x)   lockdep_reset_lock(&rtmutex_##x.dep_map)
1314 #endif
1315 #else
1316 # define I_SPINLOCK(x)
1317 # define I_RWLOCK(x)
1318 # define I_MUTEX(x)
1319 # define I_RWSEM(x)
1320 # define I_WW(x)
1321 #endif
1322
1323 #ifndef I_RTMUTEX
1324 # define I_RTMUTEX(x)
1325 #endif
1326
1327 #ifdef CONFIG_RT_MUTEXES
1328 #define I2_RTMUTEX(x)   rt_mutex_init(&rtmutex_##x)
1329 #else
1330 #define I2_RTMUTEX(x)
1331 #endif
1332
1333 #define I1(x)                                   \
1334         do {                                    \
1335                 I_SPINLOCK(x);                  \
1336                 I_RWLOCK(x);                    \
1337                 I_MUTEX(x);                     \
1338                 I_RWSEM(x);                     \
1339                 I_RTMUTEX(x);                   \
1340         } while (0)
1341
1342 #define I2(x)                                   \
1343         do {                                    \
1344                 raw_spin_lock_init(&lock_##x);  \
1345                 rwlock_init(&rwlock_##x);       \
1346                 mutex_init(&mutex_##x);         \
1347                 init_rwsem(&rwsem_##x);         \
1348                 I2_RTMUTEX(x);                  \
1349         } while (0)
1350
1351 static void reset_locks(void)
1352 {
1353         local_irq_disable();
1354         lockdep_free_key_range(&ww_lockdep.acquire_key, 1);
1355         lockdep_free_key_range(&ww_lockdep.mutex_key, 1);
1356
1357         I1(A); I1(B); I1(C); I1(D);
1358         I1(X1); I1(X2); I1(Y1); I1(Y2); I1(Z1); I1(Z2);
1359         I_WW(t); I_WW(t2); I_WW(o.base); I_WW(o2.base); I_WW(o3.base);
1360         lockdep_reset();
1361         I2(A); I2(B); I2(C); I2(D);
1362         init_shared_classes();
1363
1364         ww_mutex_init(&o, &ww_lockdep); ww_mutex_init(&o2, &ww_lockdep); ww_mutex_init(&o3, &ww_lockdep);
1365         memset(&t, 0, sizeof(t)); memset(&t2, 0, sizeof(t2));
1366         memset(&ww_lockdep.acquire_key, 0, sizeof(ww_lockdep.acquire_key));
1367         memset(&ww_lockdep.mutex_key, 0, sizeof(ww_lockdep.mutex_key));
1368         local_irq_enable();
1369 }
1370
1371 #undef I
1372
1373 static int testcase_total;
1374 static int testcase_successes;
1375 static int expected_testcase_failures;
1376 static int unexpected_testcase_failures;
1377
1378 static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
1379 {
1380         unsigned long saved_preempt_count = preempt_count();
1381
1382         WARN_ON(irqs_disabled());
1383
1384         testcase_fn();
1385         /*
1386          * Filter out expected failures:
1387          */
1388 #ifndef CONFIG_PROVE_LOCKING
1389         if (expected == FAILURE && debug_locks) {
1390                 expected_testcase_failures++;
1391                 pr_cont("failed|");
1392         }
1393         else
1394 #endif
1395         if (debug_locks != expected) {
1396                 unexpected_testcase_failures++;
1397                 pr_cont("FAILED|");
1398         } else {
1399                 testcase_successes++;
1400                 pr_cont("  ok  |");
1401         }
1402         testcase_total++;
1403
1404         if (debug_locks_verbose)
1405                 pr_cont(" lockclass mask: %x, debug_locks: %d, expected: %d\n",
1406                         lockclass_mask, debug_locks, expected);
1407         /*
1408          * Some tests (e.g. double-unlock) might corrupt the preemption
1409          * count, so restore it:
1410          */
1411         preempt_count_set(saved_preempt_count);
1412 #ifdef CONFIG_TRACE_IRQFLAGS
1413         if (softirq_count())
1414                 current->softirqs_enabled = 0;
1415         else
1416                 current->softirqs_enabled = 1;
1417 #endif
1418
1419         reset_locks();
1420 }
1421
1422 #ifdef CONFIG_RT_MUTEXES
1423 #define dotest_rt(fn, e, m)     dotest((fn), (e), (m))
1424 #else
1425 #define dotest_rt(fn, e, m)
1426 #endif
1427
1428 static inline void print_testname(const char *testname)
1429 {
1430         printk("%33s:", testname);
1431 }
1432
1433 #define DO_TESTCASE_1(desc, name, nr)                           \
1434         print_testname(desc"/"#nr);                             \
1435         dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK);          \
1436         pr_cont("\n");
1437
1438 #define DO_TESTCASE_1B(desc, name, nr)                          \
1439         print_testname(desc"/"#nr);                             \
1440         dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK);          \
1441         pr_cont("\n");
1442
1443 #define DO_TESTCASE_1RR(desc, name, nr)                         \
1444         print_testname(desc"/"#nr);                             \
1445         pr_cont("             |");                              \
1446         dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK);          \
1447         pr_cont("\n");
1448
1449 #define DO_TESTCASE_1RRB(desc, name, nr)                        \
1450         print_testname(desc"/"#nr);                             \
1451         pr_cont("             |");                              \
1452         dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK);          \
1453         pr_cont("\n");
1454
1455
1456 #define DO_TESTCASE_3(desc, name, nr)                           \
1457         print_testname(desc"/"#nr);                             \
1458         dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN);       \
1459         dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);    \
1460         dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);    \
1461         pr_cont("\n");
1462
1463 #define DO_TESTCASE_3RW(desc, name, nr)                         \
1464         print_testname(desc"/"#nr);                             \
1465         dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN|LOCKTYPE_RWLOCK);\
1466         dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);    \
1467         dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);    \
1468         pr_cont("\n");
1469
1470 #define DO_TESTCASE_2RW(desc, name, nr)                         \
1471         print_testname(desc"/"#nr);                             \
1472         pr_cont("      |");                                     \
1473         dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);    \
1474         dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);    \
1475         pr_cont("\n");
1476
1477 #define DO_TESTCASE_2x2RW(desc, name, nr)                       \
1478         DO_TESTCASE_2RW("hard-"desc, name##_hard, nr)           \
1479         DO_TESTCASE_2RW("soft-"desc, name##_soft, nr)           \
1480
1481 #define DO_TESTCASE_6x2x2RW(desc, name)                         \
1482         DO_TESTCASE_2x2RW(desc, name, 123);                     \
1483         DO_TESTCASE_2x2RW(desc, name, 132);                     \
1484         DO_TESTCASE_2x2RW(desc, name, 213);                     \
1485         DO_TESTCASE_2x2RW(desc, name, 231);                     \
1486         DO_TESTCASE_2x2RW(desc, name, 312);                     \
1487         DO_TESTCASE_2x2RW(desc, name, 321);
1488
1489 #define DO_TESTCASE_6(desc, name)                               \
1490         print_testname(desc);                                   \
1491         dotest(name##_spin, FAILURE, LOCKTYPE_SPIN);            \
1492         dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK);         \
1493         dotest(name##_rlock, FAILURE, LOCKTYPE_RWLOCK);         \
1494         dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX);          \
1495         dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM);           \
1496         dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM);           \
1497         dotest_rt(name##_rtmutex, FAILURE, LOCKTYPE_RTMUTEX);   \
1498         pr_cont("\n");
1499
1500 #define DO_TESTCASE_6_SUCCESS(desc, name)                       \
1501         print_testname(desc);                                   \
1502         dotest(name##_spin, SUCCESS, LOCKTYPE_SPIN);            \
1503         dotest(name##_wlock, SUCCESS, LOCKTYPE_RWLOCK);         \
1504         dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK);         \
1505         dotest(name##_mutex, SUCCESS, LOCKTYPE_MUTEX);          \
1506         dotest(name##_wsem, SUCCESS, LOCKTYPE_RWSEM);           \
1507         dotest(name##_rsem, SUCCESS, LOCKTYPE_RWSEM);           \
1508         dotest_rt(name##_rtmutex, SUCCESS, LOCKTYPE_RTMUTEX);   \
1509         pr_cont("\n");
1510
1511 /*
1512  * 'read' variant: rlocks must not trigger.
1513  */
1514 #define DO_TESTCASE_6R(desc, name)                              \
1515         print_testname(desc);                                   \
1516         dotest(name##_spin, FAILURE, LOCKTYPE_SPIN);            \
1517         dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK);         \
1518         dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK);         \
1519         dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX);          \
1520         dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM);           \
1521         dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM);           \
1522         dotest_rt(name##_rtmutex, FAILURE, LOCKTYPE_RTMUTEX);   \
1523         pr_cont("\n");
1524
1525 #define DO_TESTCASE_2I(desc, name, nr)                          \
1526         DO_TESTCASE_1("hard-"desc, name##_hard, nr);            \
1527         DO_TESTCASE_1("soft-"desc, name##_soft, nr);
1528
1529 #define DO_TESTCASE_2IB(desc, name, nr)                         \
1530         DO_TESTCASE_1B("hard-"desc, name##_hard, nr);           \
1531         DO_TESTCASE_1B("soft-"desc, name##_soft, nr);
1532
1533 #define DO_TESTCASE_6I(desc, name, nr)                          \
1534         DO_TESTCASE_3("hard-"desc, name##_hard, nr);            \
1535         DO_TESTCASE_3("soft-"desc, name##_soft, nr);
1536
1537 #define DO_TESTCASE_6IRW(desc, name, nr)                        \
1538         DO_TESTCASE_3RW("hard-"desc, name##_hard, nr);          \
1539         DO_TESTCASE_3RW("soft-"desc, name##_soft, nr);
1540
1541 #define DO_TESTCASE_2x3(desc, name)                             \
1542         DO_TESTCASE_3(desc, name, 12);                          \
1543         DO_TESTCASE_3(desc, name, 21);
1544
1545 #define DO_TESTCASE_2x6(desc, name)                             \
1546         DO_TESTCASE_6I(desc, name, 12);                         \
1547         DO_TESTCASE_6I(desc, name, 21);
1548
1549 #define DO_TESTCASE_6x2(desc, name)                             \
1550         DO_TESTCASE_2I(desc, name, 123);                        \
1551         DO_TESTCASE_2I(desc, name, 132);                        \
1552         DO_TESTCASE_2I(desc, name, 213);                        \
1553         DO_TESTCASE_2I(desc, name, 231);                        \
1554         DO_TESTCASE_2I(desc, name, 312);                        \
1555         DO_TESTCASE_2I(desc, name, 321);
1556
1557 #define DO_TESTCASE_6x2B(desc, name)                            \
1558         DO_TESTCASE_2IB(desc, name, 123);                       \
1559         DO_TESTCASE_2IB(desc, name, 132);                       \
1560         DO_TESTCASE_2IB(desc, name, 213);                       \
1561         DO_TESTCASE_2IB(desc, name, 231);                       \
1562         DO_TESTCASE_2IB(desc, name, 312);                       \
1563         DO_TESTCASE_2IB(desc, name, 321);
1564
1565 #define DO_TESTCASE_6x1RR(desc, name)                           \
1566         DO_TESTCASE_1RR(desc, name, 123);                       \
1567         DO_TESTCASE_1RR(desc, name, 132);                       \
1568         DO_TESTCASE_1RR(desc, name, 213);                       \
1569         DO_TESTCASE_1RR(desc, name, 231);                       \
1570         DO_TESTCASE_1RR(desc, name, 312);                       \
1571         DO_TESTCASE_1RR(desc, name, 321);
1572
1573 #define DO_TESTCASE_6x1RRB(desc, name)                          \
1574         DO_TESTCASE_1RRB(desc, name, 123);                      \
1575         DO_TESTCASE_1RRB(desc, name, 132);                      \
1576         DO_TESTCASE_1RRB(desc, name, 213);                      \
1577         DO_TESTCASE_1RRB(desc, name, 231);                      \
1578         DO_TESTCASE_1RRB(desc, name, 312);                      \
1579         DO_TESTCASE_1RRB(desc, name, 321);
1580
1581 #define DO_TESTCASE_6x6(desc, name)                             \
1582         DO_TESTCASE_6I(desc, name, 123);                        \
1583         DO_TESTCASE_6I(desc, name, 132);                        \
1584         DO_TESTCASE_6I(desc, name, 213);                        \
1585         DO_TESTCASE_6I(desc, name, 231);                        \
1586         DO_TESTCASE_6I(desc, name, 312);                        \
1587         DO_TESTCASE_6I(desc, name, 321);
1588
1589 #define DO_TESTCASE_6x6RW(desc, name)                           \
1590         DO_TESTCASE_6IRW(desc, name, 123);                      \
1591         DO_TESTCASE_6IRW(desc, name, 132);                      \
1592         DO_TESTCASE_6IRW(desc, name, 213);                      \
1593         DO_TESTCASE_6IRW(desc, name, 231);                      \
1594         DO_TESTCASE_6IRW(desc, name, 312);                      \
1595         DO_TESTCASE_6IRW(desc, name, 321);
1596
1597 static void ww_test_fail_acquire(void)
1598 {
1599         int ret;
1600
1601         WWAI(&t);
1602         t.stamp++;
1603
1604         ret = WWL(&o, &t);
1605
1606         if (WARN_ON(!o.ctx) ||
1607             WARN_ON(ret))
1608                 return;
1609
1610         /* No lockdep test, pure API */
1611         ret = WWL(&o, &t);
1612         WARN_ON(ret != -EALREADY);
1613
1614         ret = WWT(&o);
1615         WARN_ON(ret);
1616
1617         t2 = t;
1618         t2.stamp++;
1619         ret = WWL(&o, &t2);
1620         WARN_ON(ret != -EDEADLK);
1621         WWU(&o);
1622
1623         if (WWT(&o))
1624                 WWU(&o);
1625 #ifdef CONFIG_DEBUG_LOCK_ALLOC
1626         else
1627                 DEBUG_LOCKS_WARN_ON(1);
1628 #endif
1629 }
1630
1631 static void ww_test_normal(void)
1632 {
1633         int ret;
1634
1635         WWAI(&t);
1636
1637         /*
1638          * None of the ww_mutex codepaths should be taken in the 'normal'
1639          * mutex calls. The easiest way to verify this is by using the
1640          * normal mutex calls, and making sure o.ctx is unmodified.
1641          */
1642
1643         /* mutex_lock (and indirectly, mutex_lock_nested) */
1644         o.ctx = (void *)~0UL;
1645         mutex_lock(&o.base);
1646         mutex_unlock(&o.base);
1647         WARN_ON(o.ctx != (void *)~0UL);
1648
1649         /* mutex_lock_interruptible (and *_nested) */
1650         o.ctx = (void *)~0UL;
1651         ret = mutex_lock_interruptible(&o.base);
1652         if (!ret)
1653                 mutex_unlock(&o.base);
1654         else
1655                 WARN_ON(1);
1656         WARN_ON(o.ctx != (void *)~0UL);
1657
1658         /* mutex_lock_killable (and *_nested) */
1659         o.ctx = (void *)~0UL;
1660         ret = mutex_lock_killable(&o.base);
1661         if (!ret)
1662                 mutex_unlock(&o.base);
1663         else
1664                 WARN_ON(1);
1665         WARN_ON(o.ctx != (void *)~0UL);
1666
1667         /* trylock, succeeding */
1668         o.ctx = (void *)~0UL;
1669         ret = mutex_trylock(&o.base);
1670         WARN_ON(!ret);
1671         if (ret)
1672                 mutex_unlock(&o.base);
1673         else
1674                 WARN_ON(1);
1675         WARN_ON(o.ctx != (void *)~0UL);
1676
1677         /* trylock, failing */
1678         o.ctx = (void *)~0UL;
1679         mutex_lock(&o.base);
1680         ret = mutex_trylock(&o.base);
1681         WARN_ON(ret);
1682         mutex_unlock(&o.base);
1683         WARN_ON(o.ctx != (void *)~0UL);
1684
1685         /* nest_lock */
1686         o.ctx = (void *)~0UL;
1687         mutex_lock_nest_lock(&o.base, &t);
1688         mutex_unlock(&o.base);
1689         WARN_ON(o.ctx != (void *)~0UL);
1690 }
1691
1692 static void ww_test_two_contexts(void)
1693 {
1694         WWAI(&t);
1695         WWAI(&t2);
1696 }
1697
1698 static void ww_test_diff_class(void)
1699 {
1700         WWAI(&t);
1701 #ifdef CONFIG_DEBUG_MUTEXES
1702         t.ww_class = NULL;
1703 #endif
1704         WWL(&o, &t);
1705 }
1706
1707 static void ww_test_context_done_twice(void)
1708 {
1709         WWAI(&t);
1710         WWAD(&t);
1711         WWAD(&t);
1712         WWAF(&t);
1713 }
1714
1715 static void ww_test_context_unlock_twice(void)
1716 {
1717         WWAI(&t);
1718         WWAD(&t);
1719         WWAF(&t);
1720         WWAF(&t);
1721 }
1722
1723 static void ww_test_context_fini_early(void)
1724 {
1725         WWAI(&t);
1726         WWL(&o, &t);
1727         WWAD(&t);
1728         WWAF(&t);
1729 }
1730
1731 static void ww_test_context_lock_after_done(void)
1732 {
1733         WWAI(&t);
1734         WWAD(&t);
1735         WWL(&o, &t);
1736 }
1737
1738 static void ww_test_object_unlock_twice(void)
1739 {
1740         WWL1(&o);
1741         WWU(&o);
1742         WWU(&o);
1743 }
1744
1745 static void ww_test_object_lock_unbalanced(void)
1746 {
1747         WWAI(&t);
1748         WWL(&o, &t);
1749         t.acquired = 0;
1750         WWU(&o);
1751         WWAF(&t);
1752 }
1753
1754 static void ww_test_object_lock_stale_context(void)
1755 {
1756         WWAI(&t);
1757         o.ctx = &t2;
1758         WWL(&o, &t);
1759 }
1760
1761 static void ww_test_edeadlk_normal(void)
1762 {
1763         int ret;
1764
1765         mutex_lock(&o2.base);
1766         o2.ctx = &t2;
1767         mutex_release(&o2.base.dep_map, _THIS_IP_);
1768
1769         WWAI(&t);
1770         t2 = t;
1771         t2.stamp--;
1772
1773         ret = WWL(&o, &t);
1774         WARN_ON(ret);
1775
1776         ret = WWL(&o2, &t);
1777         WARN_ON(ret != -EDEADLK);
1778
1779         o2.ctx = NULL;
1780         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1781         mutex_unlock(&o2.base);
1782         WWU(&o);
1783
1784         WWL(&o2, &t);
1785 }
1786
1787 static void ww_test_edeadlk_normal_slow(void)
1788 {
1789         int ret;
1790
1791         mutex_lock(&o2.base);
1792         mutex_release(&o2.base.dep_map, _THIS_IP_);
1793         o2.ctx = &t2;
1794
1795         WWAI(&t);
1796         t2 = t;
1797         t2.stamp--;
1798
1799         ret = WWL(&o, &t);
1800         WARN_ON(ret);
1801
1802         ret = WWL(&o2, &t);
1803         WARN_ON(ret != -EDEADLK);
1804
1805         o2.ctx = NULL;
1806         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1807         mutex_unlock(&o2.base);
1808         WWU(&o);
1809
1810         ww_mutex_lock_slow(&o2, &t);
1811 }
1812
1813 static void ww_test_edeadlk_no_unlock(void)
1814 {
1815         int ret;
1816
1817         mutex_lock(&o2.base);
1818         o2.ctx = &t2;
1819         mutex_release(&o2.base.dep_map, _THIS_IP_);
1820
1821         WWAI(&t);
1822         t2 = t;
1823         t2.stamp--;
1824
1825         ret = WWL(&o, &t);
1826         WARN_ON(ret);
1827
1828         ret = WWL(&o2, &t);
1829         WARN_ON(ret != -EDEADLK);
1830
1831         o2.ctx = NULL;
1832         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1833         mutex_unlock(&o2.base);
1834
1835         WWL(&o2, &t);
1836 }
1837
1838 static void ww_test_edeadlk_no_unlock_slow(void)
1839 {
1840         int ret;
1841
1842         mutex_lock(&o2.base);
1843         mutex_release(&o2.base.dep_map, _THIS_IP_);
1844         o2.ctx = &t2;
1845
1846         WWAI(&t);
1847         t2 = t;
1848         t2.stamp--;
1849
1850         ret = WWL(&o, &t);
1851         WARN_ON(ret);
1852
1853         ret = WWL(&o2, &t);
1854         WARN_ON(ret != -EDEADLK);
1855
1856         o2.ctx = NULL;
1857         mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1858         mutex_unlock(&o2.base);
1859
1860         ww_mutex_lock_slow(&o2, &t);
1861 }
1862
1863 static void ww_test_edeadlk_acquire_more(void)
1864 {
1865         int ret;
1866
1867         mutex_lock(&o2.base);
1868         mutex_release(&o2.base.dep_map, _THIS_IP_);
1869         o2.ctx = &t2;
1870
1871         WWAI(&t);
1872         t2 = t;
1873         t2.stamp--;
1874
1875         ret = WWL(&o, &t);
1876         WARN_ON(ret);
1877
1878         ret = WWL(&o2, &t);
1879         WARN_ON(ret != -EDEADLK);
1880
1881         ret = WWL(&o3, &t);
1882 }
1883
1884 static void ww_test_edeadlk_acquire_more_slow(void)
1885 {
1886         int ret;
1887
1888         mutex_lock(&o2.base);
1889         mutex_release(&o2.base.dep_map, _THIS_IP_);
1890         o2.ctx = &t2;
1891
1892         WWAI(&t);
1893         t2 = t;
1894         t2.stamp--;
1895
1896         ret = WWL(&o, &t);
1897         WARN_ON(ret);
1898
1899         ret = WWL(&o2, &t);
1900         WARN_ON(ret != -EDEADLK);
1901
1902         ww_mutex_lock_slow(&o3, &t);
1903 }
1904
1905 static void ww_test_edeadlk_acquire_more_edeadlk(void)
1906 {
1907         int ret;
1908
1909         mutex_lock(&o2.base);
1910         mutex_release(&o2.base.dep_map, _THIS_IP_);
1911         o2.ctx = &t2;
1912
1913         mutex_lock(&o3.base);
1914         mutex_release(&o3.base.dep_map, _THIS_IP_);
1915         o3.ctx = &t2;
1916
1917         WWAI(&t);
1918         t2 = t;
1919         t2.stamp--;
1920
1921         ret = WWL(&o, &t);
1922         WARN_ON(ret);
1923
1924         ret = WWL(&o2, &t);
1925         WARN_ON(ret != -EDEADLK);
1926
1927         ret = WWL(&o3, &t);
1928         WARN_ON(ret != -EDEADLK);
1929 }
1930
1931 static void ww_test_edeadlk_acquire_more_edeadlk_slow(void)
1932 {
1933         int ret;
1934
1935         mutex_lock(&o2.base);
1936         mutex_release(&o2.base.dep_map, _THIS_IP_);
1937         o2.ctx = &t2;
1938
1939         mutex_lock(&o3.base);
1940         mutex_release(&o3.base.dep_map, _THIS_IP_);
1941         o3.ctx = &t2;
1942
1943         WWAI(&t);
1944         t2 = t;
1945         t2.stamp--;
1946
1947         ret = WWL(&o, &t);
1948         WARN_ON(ret);
1949
1950         ret = WWL(&o2, &t);
1951         WARN_ON(ret != -EDEADLK);
1952
1953         ww_mutex_lock_slow(&o3, &t);
1954 }
1955
1956 static void ww_test_edeadlk_acquire_wrong(void)
1957 {
1958         int ret;
1959
1960         mutex_lock(&o2.base);
1961         mutex_release(&o2.base.dep_map, _THIS_IP_);
1962         o2.ctx = &t2;
1963
1964         WWAI(&t);
1965         t2 = t;
1966         t2.stamp--;
1967
1968         ret = WWL(&o, &t);
1969         WARN_ON(ret);
1970
1971         ret = WWL(&o2, &t);
1972         WARN_ON(ret != -EDEADLK);
1973         if (!ret)
1974                 WWU(&o2);
1975
1976         WWU(&o);
1977
1978         ret = WWL(&o3, &t);
1979 }
1980
1981 static void ww_test_edeadlk_acquire_wrong_slow(void)
1982 {
1983         int ret;
1984
1985         mutex_lock(&o2.base);
1986         mutex_release(&o2.base.dep_map, _THIS_IP_);
1987         o2.ctx = &t2;
1988
1989         WWAI(&t);
1990         t2 = t;
1991         t2.stamp--;
1992
1993         ret = WWL(&o, &t);
1994         WARN_ON(ret);
1995
1996         ret = WWL(&o2, &t);
1997         WARN_ON(ret != -EDEADLK);
1998         if (!ret)
1999                 WWU(&o2);
2000
2001         WWU(&o);
2002
2003         ww_mutex_lock_slow(&o3, &t);
2004 }
2005
2006 static void ww_test_spin_nest_unlocked(void)
2007 {
2008         raw_spin_lock_nest_lock(&lock_A, &o.base);
2009         U(A);
2010 }
2011
2012 static void ww_test_unneeded_slow(void)
2013 {
2014         WWAI(&t);
2015
2016         ww_mutex_lock_slow(&o, &t);
2017 }
2018
2019 static void ww_test_context_block(void)
2020 {
2021         int ret;
2022
2023         WWAI(&t);
2024
2025         ret = WWL(&o, &t);
2026         WARN_ON(ret);
2027         WWL1(&o2);
2028 }
2029
2030 static void ww_test_context_try(void)
2031 {
2032         int ret;
2033
2034         WWAI(&t);
2035
2036         ret = WWL(&o, &t);
2037         WARN_ON(ret);
2038
2039         ret = WWT(&o2);
2040         WARN_ON(!ret);
2041         WWU(&o2);
2042         WWU(&o);
2043 }
2044
2045 static void ww_test_context_context(void)
2046 {
2047         int ret;
2048
2049         WWAI(&t);
2050
2051         ret = WWL(&o, &t);
2052         WARN_ON(ret);
2053
2054         ret = WWL(&o2, &t);
2055         WARN_ON(ret);
2056
2057         WWU(&o2);
2058         WWU(&o);
2059 }
2060
2061 static void ww_test_try_block(void)
2062 {
2063         bool ret;
2064
2065         ret = WWT(&o);
2066         WARN_ON(!ret);
2067
2068         WWL1(&o2);
2069         WWU(&o2);
2070         WWU(&o);
2071 }
2072
2073 static void ww_test_try_try(void)
2074 {
2075         bool ret;
2076
2077         ret = WWT(&o);
2078         WARN_ON(!ret);
2079         ret = WWT(&o2);
2080         WARN_ON(!ret);
2081         WWU(&o2);
2082         WWU(&o);
2083 }
2084
2085 static void ww_test_try_context(void)
2086 {
2087         int ret;
2088
2089         ret = WWT(&o);
2090         WARN_ON(!ret);
2091
2092         WWAI(&t);
2093
2094         ret = WWL(&o2, &t);
2095         WARN_ON(ret);
2096 }
2097
2098 static void ww_test_block_block(void)
2099 {
2100         WWL1(&o);
2101         WWL1(&o2);
2102 }
2103
2104 static void ww_test_block_try(void)
2105 {
2106         bool ret;
2107
2108         WWL1(&o);
2109         ret = WWT(&o2);
2110         WARN_ON(!ret);
2111 }
2112
2113 static void ww_test_block_context(void)
2114 {
2115         int ret;
2116
2117         WWL1(&o);
2118         WWAI(&t);
2119
2120         ret = WWL(&o2, &t);
2121         WARN_ON(ret);
2122 }
2123
2124 static void ww_test_spin_block(void)
2125 {
2126         L(A);
2127         U(A);
2128
2129         WWL1(&o);
2130         L(A);
2131         U(A);
2132         WWU(&o);
2133
2134         L(A);
2135         WWL1(&o);
2136         WWU(&o);
2137         U(A);
2138 }
2139
2140 static void ww_test_spin_try(void)
2141 {
2142         bool ret;
2143
2144         L(A);
2145         U(A);
2146
2147         ret = WWT(&o);
2148         WARN_ON(!ret);
2149         L(A);
2150         U(A);
2151         WWU(&o);
2152
2153         L(A);
2154         ret = WWT(&o);
2155         WARN_ON(!ret);
2156         WWU(&o);
2157         U(A);
2158 }
2159
2160 static void ww_test_spin_context(void)
2161 {
2162         int ret;
2163
2164         L(A);
2165         U(A);
2166
2167         WWAI(&t);
2168
2169         ret = WWL(&o, &t);
2170         WARN_ON(ret);
2171         L(A);
2172         U(A);
2173         WWU(&o);
2174
2175         L(A);
2176         ret = WWL(&o, &t);
2177         WARN_ON(ret);
2178         WWU(&o);
2179         U(A);
2180 }
2181
2182 static void ww_tests(void)
2183 {
2184         printk("  --------------------------------------------------------------------------\n");
2185         printk("  | Wound/wait tests |\n");
2186         printk("  ---------------------\n");
2187
2188         print_testname("ww api failures");
2189         dotest(ww_test_fail_acquire, SUCCESS, LOCKTYPE_WW);
2190         dotest(ww_test_normal, SUCCESS, LOCKTYPE_WW);
2191         dotest(ww_test_unneeded_slow, FAILURE, LOCKTYPE_WW);
2192         pr_cont("\n");
2193
2194         print_testname("ww contexts mixing");
2195         dotest(ww_test_two_contexts, FAILURE, LOCKTYPE_WW);
2196         dotest(ww_test_diff_class, FAILURE, LOCKTYPE_WW);
2197         pr_cont("\n");
2198
2199         print_testname("finishing ww context");
2200         dotest(ww_test_context_done_twice, FAILURE, LOCKTYPE_WW);
2201         dotest(ww_test_context_unlock_twice, FAILURE, LOCKTYPE_WW);
2202         dotest(ww_test_context_fini_early, FAILURE, LOCKTYPE_WW);
2203         dotest(ww_test_context_lock_after_done, FAILURE, LOCKTYPE_WW);
2204         pr_cont("\n");
2205
2206         print_testname("locking mismatches");
2207         dotest(ww_test_object_unlock_twice, FAILURE, LOCKTYPE_WW);
2208         dotest(ww_test_object_lock_unbalanced, FAILURE, LOCKTYPE_WW);
2209         dotest(ww_test_object_lock_stale_context, FAILURE, LOCKTYPE_WW);
2210         pr_cont("\n");
2211
2212         print_testname("EDEADLK handling");
2213         dotest(ww_test_edeadlk_normal, SUCCESS, LOCKTYPE_WW);
2214         dotest(ww_test_edeadlk_normal_slow, SUCCESS, LOCKTYPE_WW);
2215         dotest(ww_test_edeadlk_no_unlock, FAILURE, LOCKTYPE_WW);
2216         dotest(ww_test_edeadlk_no_unlock_slow, FAILURE, LOCKTYPE_WW);
2217         dotest(ww_test_edeadlk_acquire_more, FAILURE, LOCKTYPE_WW);
2218         dotest(ww_test_edeadlk_acquire_more_slow, FAILURE, LOCKTYPE_WW);
2219         dotest(ww_test_edeadlk_acquire_more_edeadlk, FAILURE, LOCKTYPE_WW);
2220         dotest(ww_test_edeadlk_acquire_more_edeadlk_slow, FAILURE, LOCKTYPE_WW);
2221         dotest(ww_test_edeadlk_acquire_wrong, FAILURE, LOCKTYPE_WW);
2222         dotest(ww_test_edeadlk_acquire_wrong_slow, FAILURE, LOCKTYPE_WW);
2223         pr_cont("\n");
2224
2225         print_testname("spinlock nest unlocked");
2226         dotest(ww_test_spin_nest_unlocked, FAILURE, LOCKTYPE_WW);
2227         pr_cont("\n");
2228
2229         printk("  -----------------------------------------------------\n");
2230         printk("                                 |block | try  |context|\n");
2231         printk("  -----------------------------------------------------\n");
2232
2233         print_testname("context");
2234         dotest(ww_test_context_block, FAILURE, LOCKTYPE_WW);
2235         dotest(ww_test_context_try, SUCCESS, LOCKTYPE_WW);
2236         dotest(ww_test_context_context, SUCCESS, LOCKTYPE_WW);
2237         pr_cont("\n");
2238
2239         print_testname("try");
2240         dotest(ww_test_try_block, FAILURE, LOCKTYPE_WW);
2241         dotest(ww_test_try_try, SUCCESS, LOCKTYPE_WW);
2242         dotest(ww_test_try_context, FAILURE, LOCKTYPE_WW);
2243         pr_cont("\n");
2244
2245         print_testname("block");
2246         dotest(ww_test_block_block, FAILURE, LOCKTYPE_WW);
2247         dotest(ww_test_block_try, SUCCESS, LOCKTYPE_WW);
2248         dotest(ww_test_block_context, FAILURE, LOCKTYPE_WW);
2249         pr_cont("\n");
2250
2251         print_testname("spinlock");
2252         dotest(ww_test_spin_block, FAILURE, LOCKTYPE_WW);
2253         dotest(ww_test_spin_try, SUCCESS, LOCKTYPE_WW);
2254         dotest(ww_test_spin_context, FAILURE, LOCKTYPE_WW);
2255         pr_cont("\n");
2256 }
2257
2258
2259 /*
2260  * <in hardirq handler>
2261  * read_lock(&A);
2262  *                      <hardirq disable>
2263  *                      spin_lock(&B);
2264  * spin_lock(&B);
2265  *                      read_lock(&A);
2266  *
2267  * is a deadlock.
2268  */
2269 static void queued_read_lock_hardirq_RE_Er(void)
2270 {
2271         HARDIRQ_ENTER();
2272         read_lock(&rwlock_A);
2273         LOCK(B);
2274         UNLOCK(B);
2275         read_unlock(&rwlock_A);
2276         HARDIRQ_EXIT();
2277
2278         HARDIRQ_DISABLE();
2279         LOCK(B);
2280         read_lock(&rwlock_A);
2281         read_unlock(&rwlock_A);
2282         UNLOCK(B);
2283         HARDIRQ_ENABLE();
2284 }
2285
2286 /*
2287  * <in hardirq handler>
2288  * spin_lock(&B);
2289  *                      <hardirq disable>
2290  *                      read_lock(&A);
2291  * read_lock(&A);
2292  *                      spin_lock(&B);
2293  *
2294  * is not a deadlock.
2295  */
2296 static void queued_read_lock_hardirq_ER_rE(void)
2297 {
2298         HARDIRQ_ENTER();
2299         LOCK(B);
2300         read_lock(&rwlock_A);
2301         read_unlock(&rwlock_A);
2302         UNLOCK(B);
2303         HARDIRQ_EXIT();
2304
2305         HARDIRQ_DISABLE();
2306         read_lock(&rwlock_A);
2307         LOCK(B);
2308         UNLOCK(B);
2309         read_unlock(&rwlock_A);
2310         HARDIRQ_ENABLE();
2311 }
2312
2313 /*
2314  * <hardirq disable>
2315  * spin_lock(&B);
2316  *                      read_lock(&A);
2317  *                      <in hardirq handler>
2318  *                      spin_lock(&B);
2319  * read_lock(&A);
2320  *
2321  * is a deadlock. Because the two read_lock()s are both non-recursive readers.
2322  */
2323 static void queued_read_lock_hardirq_inversion(void)
2324 {
2325
2326         HARDIRQ_ENTER();
2327         LOCK(B);
2328         UNLOCK(B);
2329         HARDIRQ_EXIT();
2330
2331         HARDIRQ_DISABLE();
2332         LOCK(B);
2333         read_lock(&rwlock_A);
2334         read_unlock(&rwlock_A);
2335         UNLOCK(B);
2336         HARDIRQ_ENABLE();
2337
2338         read_lock(&rwlock_A);
2339         read_unlock(&rwlock_A);
2340 }
2341
2342 static void queued_read_lock_tests(void)
2343 {
2344         printk("  --------------------------------------------------------------------------\n");
2345         printk("  | queued read lock tests |\n");
2346         printk("  ---------------------------\n");
2347         print_testname("hardirq read-lock/lock-read");
2348         dotest(queued_read_lock_hardirq_RE_Er, FAILURE, LOCKTYPE_RWLOCK);
2349         pr_cont("\n");
2350
2351         print_testname("hardirq lock-read/read-lock");
2352         dotest(queued_read_lock_hardirq_ER_rE, SUCCESS, LOCKTYPE_RWLOCK);
2353         pr_cont("\n");
2354
2355         print_testname("hardirq inversion");
2356         dotest(queued_read_lock_hardirq_inversion, FAILURE, LOCKTYPE_RWLOCK);
2357         pr_cont("\n");
2358 }
2359
2360 void locking_selftest(void)
2361 {
2362         /*
2363          * Got a locking failure before the selftest ran?
2364          */
2365         if (!debug_locks) {
2366                 printk("----------------------------------\n");
2367                 printk("| Locking API testsuite disabled |\n");
2368                 printk("----------------------------------\n");
2369                 return;
2370         }
2371
2372         /*
2373          * treats read_lock() as recursive read locks for testing purpose
2374          */
2375         force_read_lock_recursive = 1;
2376
2377         /*
2378          * Run the testsuite:
2379          */
2380         printk("------------------------\n");
2381         printk("| Locking API testsuite:\n");
2382         printk("----------------------------------------------------------------------------\n");
2383         printk("                                 | spin |wlock |rlock |mutex | wsem | rsem |\n");
2384         printk("  --------------------------------------------------------------------------\n");
2385
2386         init_shared_classes();
2387         debug_locks_silent = !debug_locks_verbose;
2388         lockdep_set_selftest_task(current);
2389
2390         DO_TESTCASE_6R("A-A deadlock", AA);
2391         DO_TESTCASE_6R("A-B-B-A deadlock", ABBA);
2392         DO_TESTCASE_6R("A-B-B-C-C-A deadlock", ABBCCA);
2393         DO_TESTCASE_6R("A-B-C-A-B-C deadlock", ABCABC);
2394         DO_TESTCASE_6R("A-B-B-C-C-D-D-A deadlock", ABBCCDDA);
2395         DO_TESTCASE_6R("A-B-C-D-B-D-D-A deadlock", ABCDBDDA);
2396         DO_TESTCASE_6R("A-B-C-D-B-C-D-A deadlock", ABCDBCDA);
2397         DO_TESTCASE_6("double unlock", double_unlock);
2398         DO_TESTCASE_6("initialize held", init_held);
2399
2400         printk("  --------------------------------------------------------------------------\n");
2401         print_testname("recursive read-lock");
2402         pr_cont("             |");
2403         dotest(rlock_AA1, SUCCESS, LOCKTYPE_RWLOCK);
2404         pr_cont("             |");
2405         dotest(rsem_AA1, FAILURE, LOCKTYPE_RWSEM);
2406         pr_cont("\n");
2407
2408         print_testname("recursive read-lock #2");
2409         pr_cont("             |");
2410         dotest(rlock_AA1B, SUCCESS, LOCKTYPE_RWLOCK);
2411         pr_cont("             |");
2412         dotest(rsem_AA1B, FAILURE, LOCKTYPE_RWSEM);
2413         pr_cont("\n");
2414
2415         print_testname("mixed read-write-lock");
2416         pr_cont("             |");
2417         dotest(rlock_AA2, FAILURE, LOCKTYPE_RWLOCK);
2418         pr_cont("             |");
2419         dotest(rsem_AA2, FAILURE, LOCKTYPE_RWSEM);
2420         pr_cont("\n");
2421
2422         print_testname("mixed write-read-lock");
2423         pr_cont("             |");
2424         dotest(rlock_AA3, FAILURE, LOCKTYPE_RWLOCK);
2425         pr_cont("             |");
2426         dotest(rsem_AA3, FAILURE, LOCKTYPE_RWSEM);
2427         pr_cont("\n");
2428
2429         print_testname("mixed read-lock/lock-write ABBA");
2430         pr_cont("             |");
2431         dotest(rlock_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
2432         pr_cont("             |");
2433         dotest(rwsem_ABBA1, FAILURE, LOCKTYPE_RWSEM);
2434
2435         print_testname("mixed read-lock/lock-read ABBA");
2436         pr_cont("             |");
2437         dotest(rlock_ABBA2, SUCCESS, LOCKTYPE_RWLOCK);
2438         pr_cont("             |");
2439         dotest(rwsem_ABBA2, FAILURE, LOCKTYPE_RWSEM);
2440
2441         print_testname("mixed write-lock/lock-write ABBA");
2442         pr_cont("             |");
2443         dotest(rlock_ABBA3, FAILURE, LOCKTYPE_RWLOCK);
2444         pr_cont("             |");
2445         dotest(rwsem_ABBA3, FAILURE, LOCKTYPE_RWSEM);
2446
2447         print_testname("chain cached mixed R-L/L-W ABBA");
2448         pr_cont("             |");
2449         dotest(rlock_chaincache_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
2450
2451         DO_TESTCASE_6x1RRB("rlock W1R2/W2R3/W3R1", W1R2_W2R3_W3R1);
2452         DO_TESTCASE_6x1RRB("rlock W1W2/R2R3/W3R1", W1W2_R2R3_W3R1);
2453         DO_TESTCASE_6x1RR("rlock W1W2/R2R3/R3W1", W1W2_R2R3_R3W1);
2454         DO_TESTCASE_6x1RR("rlock W1R2/R2R3/W3W1", W1R2_R2R3_W3W1);
2455
2456         printk("  --------------------------------------------------------------------------\n");
2457
2458         /*
2459          * irq-context testcases:
2460          */
2461         DO_TESTCASE_2x6("irqs-on + irq-safe-A", irqsafe1);
2462         DO_TESTCASE_2x3("sirq-safe-A => hirqs-on", irqsafe2A);
2463         DO_TESTCASE_2x6("safe-A + irqs-on", irqsafe2B);
2464         DO_TESTCASE_6x6("safe-A + unsafe-B #1", irqsafe3);
2465         DO_TESTCASE_6x6("safe-A + unsafe-B #2", irqsafe4);
2466         DO_TESTCASE_6x6RW("irq lock-inversion", irq_inversion);
2467
2468         DO_TESTCASE_6x2x2RW("irq read-recursion", irq_read_recursion);
2469         DO_TESTCASE_6x2x2RW("irq read-recursion #2", irq_read_recursion2);
2470         DO_TESTCASE_6x2x2RW("irq read-recursion #3", irq_read_recursion3);
2471
2472         ww_tests();
2473
2474         force_read_lock_recursive = 0;
2475         /*
2476          * queued_read_lock() specific test cases can be put here
2477          */
2478         if (IS_ENABLED(CONFIG_QUEUED_RWLOCKS))
2479                 queued_read_lock_tests();
2480
2481         if (unexpected_testcase_failures) {
2482                 printk("-----------------------------------------------------------------\n");
2483                 debug_locks = 0;
2484                 printk("BUG: %3d unexpected failures (out of %3d) - debugging disabled! |\n",
2485                         unexpected_testcase_failures, testcase_total);
2486                 printk("-----------------------------------------------------------------\n");
2487         } else if (expected_testcase_failures && testcase_successes) {
2488                 printk("--------------------------------------------------------\n");
2489                 printk("%3d out of %3d testcases failed, as expected. |\n",
2490                         expected_testcase_failures, testcase_total);
2491                 printk("----------------------------------------------------\n");
2492                 debug_locks = 1;
2493         } else if (expected_testcase_failures && !testcase_successes) {
2494                 printk("--------------------------------------------------------\n");
2495                 printk("All %3d testcases failed, as expected. |\n",
2496                         expected_testcase_failures);
2497                 printk("----------------------------------------\n");
2498                 debug_locks = 1;
2499         } else {
2500                 printk("-------------------------------------------------------\n");
2501                 printk("Good, all %3d testcases passed! |\n",
2502                         testcase_successes);
2503                 printk("---------------------------------\n");
2504                 debug_locks = 1;
2505         }
2506         lockdep_set_selftest_task(NULL);
2507         debug_locks_silent = 0;
2508 }