kernel/kcsan/kcsan-test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * KCSAN test with various race scenarious to test runtime behaviour. Since the
   4  * interface with which KCSAN's reports are obtained is via the console, this is
   5  * the output we should verify. For each test case checks the presence (or
   6  * absence) of generated reports. Relies on 'console' tracepoint to capture
   7  * reports as they appear in the kernel log.
   8  *
   9  * Makes use of KUnit for test organization, and the Torture framework for test
  10  * thread control.
  11  *
  12  * Copyright (C) 2020, Google LLC.
  13  * Author: Marco Elver <elver@google.com>
  14  */
  15
  16 #include <kunit/test.h>
  17 #include <linux/jiffies.h>
  18 #include <linux/kcsan-checks.h>
  19 #include <linux/kernel.h>
  20 #include <linux/sched.h>
  21 #include <linux/seqlock.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/string.h>
  24 #include <linux/timer.h>
  25 #include <linux/torture.h>
  26 #include <linux/tracepoint.h>
  27 #include <linux/types.h>
  28 #include <trace/events/printk.h>
  29
  30 /* Points to current test-case memory access "kernels". */
  31 static void (*access_kernels[2])(void);
  32
  33 static struct task_struct **threads; /* Lists of threads. */
  34 static unsigned long end_time;       /* End time of test. */
  35
  36 /* Report as observed from console. */
  37 static struct {
  38         spinlock_t lock;
  39         int nlines;
  40         char lines[3][512];
  41 } observed = {
  42         .lock = __SPIN_LOCK_UNLOCKED(observed.lock),
  43 };
  44
  45 /* Setup test checking loop. */
  46 static __no_kcsan inline void
  47 begin_test_checks(void (*func1)(void), void (*func2)(void))
  48 {
  49         kcsan_disable_current();
  50
  51         /*
  52          * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at
  53          * least one race is reported.
  54          */
  55         end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500);
  56
  57         /* Signal start; release potential initialization of shared data. */
  58         smp_store_release(&access_kernels[0], func1);
  59         smp_store_release(&access_kernels[1], func2);
  60 }
  61
  62 /* End test checking loop. */
  63 static __no_kcsan inline bool
  64 end_test_checks(bool stop)
  65 {
  66         if (!stop && time_before(jiffies, end_time)) {
  67                 /* Continue checking */
  68                 might_sleep();
  69                 return false;
  70         }
  71
  72         kcsan_enable_current();
  73         return true;
  74 }
  75
  76 /*
  77  * Probe for console output: checks if a race was reported, and obtains observed
  78  * lines of interest.
  79  */
  80 __no_kcsan
  81 static void probe_console(void *ignore, const char *buf, size_t len)
  82 {
  83         unsigned long flags;
  84         int nlines;
  85
  86         /*
  87          * Note that KCSAN reports under a global lock, so we do not risk the
  88          * possibility of having multiple reports interleaved. If that were the
  89          * case, we'd expect tests to fail.
  90          */
  91
  92         spin_lock_irqsave(&observed.lock, flags);
  93         nlines = observed.nlines;
  94
  95         if (strnstr(buf, "BUG: KCSAN: ", len) && strnstr(buf, "test_", len)) {
  96                 /*
  97                  * KCSAN report and related to the test.
  98                  *
  99                  * The provided @buf is not NUL-terminated; copy no more than
 100                  * @len bytes and let strscpy() add the missing NUL-terminator.
 101                  */
 102                 strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0])));
 103                 nlines = 1;
 104         } else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by", len)) {
 105                 strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0])));
 106
 107                 if (strnstr(buf, "race at unknown origin", len)) {
 108                         if (WARN_ON(nlines != 2))
 109                                 goto out;
 110
 111                         /* No second line of interest. */
 112                         strcpy(observed.lines[nlines++], "<none>");
 113                 }
 114         }
 115
 116 out:
 117         WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */
 118         spin_unlock_irqrestore(&observed.lock, flags);
 119 }
 120
 121 /* Check if a report related to the test exists. */
 122 __no_kcsan
 123 static bool report_available(void)
 124 {
 125         return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines);
 126 }
 127
 128 /* Report information we expect in a report. */
 129 struct expect_report {
 130         /* Access information of both accesses. */
 131         struct {
 132                 void *fn;    /* Function pointer to expected function of top frame. */
 133                 void *addr;  /* Address of access; unchecked if NULL. */
 134                 size_t size; /* Size of access; unchecked if @addr is NULL. */
 135                 int type;    /* Access type, see KCSAN_ACCESS definitions. */
 136         } access[2];
 137 };
 138
 139 /* Check observed report matches information in @r. */
 140 __no_kcsan
 141 static bool report_matches(const struct expect_report *r)
 142 {
 143         const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT;
 144         bool ret = false;
 145         unsigned long flags;
 146         typeof(observed.lines) expect;
 147         const char *end;
 148         char *cur;
 149         int i;
 150
 151         /* Doubled-checked locking. */
 152         if (!report_available())
 153                 return false;
 154
 155         /* Generate expected report contents. */
 156
 157         /* Title */
 158         cur = expect[0];
 159         end = &expect[0][sizeof(expect[0]) - 1];
 160         cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ",
 161                          is_assert ? "assert: race" : "data-race");
 162         if (r->access[1].fn) {
 163                 char tmp[2][64];
 164                 int cmp;
 165
 166                 /* Expect lexographically sorted function names in title. */
 167                 scnprintf(tmp[0], sizeof(tmp[0]), "%pS", r->access[0].fn);
 168                 scnprintf(tmp[1], sizeof(tmp[1]), "%pS", r->access[1].fn);
 169                 cmp = strcmp(tmp[0], tmp[1]);
 170                 cur += scnprintf(cur, end - cur, "%ps / %ps",
 171                                  cmp < 0 ? r->access[0].fn : r->access[1].fn,
 172                                  cmp < 0 ? r->access[1].fn : r->access[0].fn);
 173         } else {
 174                 scnprintf(cur, end - cur, "%pS", r->access[0].fn);
 175                 /* The exact offset won't match, remove it. */
 176                 cur = strchr(expect[0], '+');
 177                 if (cur)
 178                         *cur = '\0';
 179         }
 180
 181         /* Access 1 */
 182         cur = expect[1];
 183         end = &expect[1][sizeof(expect[1]) - 1];
 184         if (!r->access[1].fn)
 185                 cur += scnprintf(cur, end - cur, "race at unknown origin, with ");
 186
 187         /* Access 1 & 2 */
 188         for (i = 0; i < 2; ++i) {
 189                 const char *const access_type =
 190                         (r->access[i].type & KCSAN_ACCESS_ASSERT) ?
 191                                 ((r->access[i].type & KCSAN_ACCESS_WRITE) ?
 192                                          "assert no accesses" :
 193                                          "assert no writes") :
 194                                 ((r->access[i].type & KCSAN_ACCESS_WRITE) ?
 195                                          "write" :
 196                                          "read");
 197                 const char *const access_type_aux =
 198                         (r->access[i].type & KCSAN_ACCESS_ATOMIC) ?
 199                                 " (marked)" :
 200                                 ((r->access[i].type & KCSAN_ACCESS_SCOPED) ?
 201                                          " (scoped)" :
 202                                          "");
 203
 204                 if (i == 1) {
 205                         /* Access 2 */
 206                         cur = expect[2];
 207                         end = &expect[2][sizeof(expect[2]) - 1];
 208
 209                         if (!r->access[1].fn) {
 210                                 /* Dummy string if no second access is available. */
 211                                 strcpy(cur, "<none>");
 212                                 break;
 213                         }
 214                 }
 215
 216                 cur += scnprintf(cur, end - cur, "%s%s to ", access_type,
 217                                  access_type_aux);
 218
 219                 if (r->access[i].addr) /* Address is optional. */
 220                         cur += scnprintf(cur, end - cur, "0x%px of %zu bytes",
 221                                          r->access[i].addr, r->access[i].size);
 222         }
 223
 224         spin_lock_irqsave(&observed.lock, flags);
 225         if (!report_available())
 226                 goto out; /* A new report is being captured. */
 227
 228         /* Finally match expected output to what we actually observed. */
 229         ret = strstr(observed.lines[0], expect[0]) &&
 230               /* Access info may appear in any order. */
 231               ((strstr(observed.lines[1], expect[1]) &&
 232                 strstr(observed.lines[2], expect[2])) ||
 233                (strstr(observed.lines[1], expect[2]) &&
 234                 strstr(observed.lines[2], expect[1])));
 235 out:
 236         spin_unlock_irqrestore(&observed.lock, flags);
 237         return ret;
 238 }
 239
 240 /* ===== Test kernels ===== */
 241
 242 static long test_sink;
 243 static long test_var;
 244 /* @test_array should be large enough to fall into multiple watchpoint slots. */
 245 static long test_array[3 * PAGE_SIZE / sizeof(long)];
 246 static struct {
 247         long val[8];
 248 } test_struct;
 249 static DEFINE_SEQLOCK(test_seqlock);
 250
 251 /*
 252  * Helper to avoid compiler optimizing out reads, and to generate source values
 253  * for writes.
 254  */
 255 __no_kcsan
 256 static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); }
 257
 258 static noinline void test_kernel_read(void) { sink_value(test_var); }
 259
 260 static noinline void test_kernel_write(void)
 261 {
 262         test_var = READ_ONCE_NOCHECK(test_sink) + 1;
 263 }
 264
 265 static noinline void test_kernel_write_nochange(void) { test_var = 42; }
 266
 267 /* Suffixed by value-change exception filter. */
 268 static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; }
 269
 270 static noinline void test_kernel_read_atomic(void)
 271 {
 272         sink_value(READ_ONCE(test_var));
 273 }
 274
 275 static noinline void test_kernel_write_atomic(void)
 276 {
 277         WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1);
 278 }
 279
 280 __no_kcsan
 281 static noinline void test_kernel_write_uninstrumented(void) { test_var++; }
 282
 283 static noinline void test_kernel_data_race(void) { data_race(test_var++); }
 284
 285 static noinline void test_kernel_assert_writer(void)
 286 {
 287         ASSERT_EXCLUSIVE_WRITER(test_var);
 288 }
 289
 290 static noinline void test_kernel_assert_access(void)
 291 {
 292         ASSERT_EXCLUSIVE_ACCESS(test_var);
 293 }
 294
 295 #define TEST_CHANGE_BITS 0xff00ff00
 296
 297 static noinline void test_kernel_change_bits(void)
 298 {
 299         if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {
 300                 /*
 301                  * Avoid race of unknown origin for this test, just pretend they
 302                  * are atomic.
 303                  */
 304                 kcsan_nestable_atomic_begin();
 305                 test_var ^= TEST_CHANGE_BITS;
 306                 kcsan_nestable_atomic_end();
 307         } else
 308                 WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS);
 309 }
 310
 311 static noinline void test_kernel_assert_bits_change(void)
 312 {
 313         ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS);
 314 }
 315
 316 static noinline void test_kernel_assert_bits_nochange(void)
 317 {
 318         ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS);
 319 }
 320
 321 /* To check that scoped assertions do trigger anywhere in scope. */
 322 static noinline void test_enter_scope(void)
 323 {
 324         int x = 0;
 325
 326         /* Unrelated accesses to scoped assert. */
 327         READ_ONCE(test_sink);
 328         kcsan_check_read(&x, sizeof(x));
 329 }
 330
 331 static noinline void test_kernel_assert_writer_scoped(void)
 332 {
 333         ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var);
 334         test_enter_scope();
 335 }
 336
 337 static noinline void test_kernel_assert_access_scoped(void)
 338 {
 339         ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var);
 340         test_enter_scope();
 341 }
 342
 343 static noinline void test_kernel_rmw_array(void)
 344 {
 345         int i;
 346
 347         for (i = 0; i < ARRAY_SIZE(test_array); ++i)
 348                 test_array[i]++;
 349 }
 350
 351 static noinline void test_kernel_write_struct(void)
 352 {
 353         kcsan_check_write(&test_struct, sizeof(test_struct));
 354         kcsan_disable_current();
 355         test_struct.val[3]++; /* induce value change */
 356         kcsan_enable_current();
 357 }
 358
 359 static noinline void test_kernel_write_struct_part(void)
 360 {
 361         test_struct.val[3] = 42;
 362 }
 363
 364 static noinline void test_kernel_read_struct_zero_size(void)
 365 {
 366         kcsan_check_read(&test_struct.val[3], 0);
 367 }
 368
 369 static noinline void test_kernel_jiffies_reader(void)
 370 {
 371         sink_value((long)jiffies);
 372 }
 373
 374 static noinline void test_kernel_seqlock_reader(void)
 375 {
 376         unsigned int seq;
 377
 378         do {
 379                 seq = read_seqbegin(&test_seqlock);
 380                 sink_value(test_var);
 381         } while (read_seqretry(&test_seqlock, seq));
 382 }
 383
 384 static noinline void test_kernel_seqlock_writer(void)
 385 {
 386         unsigned long flags;
 387
 388         write_seqlock_irqsave(&test_seqlock, flags);
 389         test_var++;
 390         write_sequnlock_irqrestore(&test_seqlock, flags);
 391 }
 392
 393 /* ===== Test cases ===== */
 394
 395 /* Simple test with normal data race. */
 396 __no_kcsan
 397 static void test_basic(struct kunit *test)
 398 {
 399         const struct expect_report expect = {
 400                 .access = {
 401                         { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 402                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 403                 },
 404         };
 405         static const struct expect_report never = {
 406                 .access = {
 407                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 408                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 409                 },
 410         };
 411         bool match_expect = false;
 412         bool match_never = false;
 413
 414         begin_test_checks(test_kernel_write, test_kernel_read);
 415         do {
 416                 match_expect |= report_matches(&expect);
 417                 match_never = report_matches(&never);
 418         } while (!end_test_checks(match_never));
 419         KUNIT_EXPECT_TRUE(test, match_expect);
 420         KUNIT_EXPECT_FALSE(test, match_never);
 421 }
 422
 423 /*
 424  * Stress KCSAN with lots of concurrent races on different addresses until
 425  * timeout.
 426  */
 427 __no_kcsan
 428 static void test_concurrent_races(struct kunit *test)
 429 {
 430         const struct expect_report expect = {
 431                 .access = {
 432                         /* NULL will match any address. */
 433                         { test_kernel_rmw_array, NULL, 0, KCSAN_ACCESS_WRITE },
 434                         { test_kernel_rmw_array, NULL, 0, 0 },
 435                 },
 436         };
 437         static const struct expect_report never = {
 438                 .access = {
 439                         { test_kernel_rmw_array, NULL, 0, 0 },
 440                         { test_kernel_rmw_array, NULL, 0, 0 },
 441                 },
 442         };
 443         bool match_expect = false;
 444         bool match_never = false;
 445
 446         begin_test_checks(test_kernel_rmw_array, test_kernel_rmw_array);
 447         do {
 448                 match_expect |= report_matches(&expect);
 449                 match_never |= report_matches(&never);
 450         } while (!end_test_checks(false));
 451         KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */
 452         KUNIT_EXPECT_FALSE(test, match_never);
 453 }
 454
 455 /* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */
 456 __no_kcsan
 457 static void test_novalue_change(struct kunit *test)
 458 {
 459         const struct expect_report expect = {
 460                 .access = {
 461                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 462                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 463                 },
 464         };
 465         bool match_expect = false;
 466
 467         begin_test_checks(test_kernel_write_nochange, test_kernel_read);
 468         do {
 469                 match_expect = report_matches(&expect);
 470         } while (!end_test_checks(match_expect));
 471         if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY))
 472                 KUNIT_EXPECT_FALSE(test, match_expect);
 473         else
 474                 KUNIT_EXPECT_TRUE(test, match_expect);
 475 }
 476
 477 /*
 478  * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should
 479  * never apply work.
 480  */
 481 __no_kcsan
 482 static void test_novalue_change_exception(struct kunit *test)
 483 {
 484         const struct expect_report expect = {
 485                 .access = {
 486                         { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 487                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 488                 },
 489         };
 490         bool match_expect = false;
 491
 492         begin_test_checks(test_kernel_write_nochange_rcu, test_kernel_read);
 493         do {
 494                 match_expect = report_matches(&expect);
 495         } while (!end_test_checks(match_expect));
 496         KUNIT_EXPECT_TRUE(test, match_expect);
 497 }
 498
 499 /* Test that data races of unknown origin are reported. */
 500 __no_kcsan
 501 static void test_unknown_origin(struct kunit *test)
 502 {
 503         const struct expect_report expect = {
 504                 .access = {
 505                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 506                         { NULL },
 507                 },
 508         };
 509         bool match_expect = false;
 510
 511         begin_test_checks(test_kernel_write_uninstrumented, test_kernel_read);
 512         do {
 513                 match_expect = report_matches(&expect);
 514         } while (!end_test_checks(match_expect));
 515         if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN))
 516                 KUNIT_EXPECT_TRUE(test, match_expect);
 517         else
 518                 KUNIT_EXPECT_FALSE(test, match_expect);
 519 }
 520
 521 /* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */
 522 __no_kcsan
 523 static void test_write_write_assume_atomic(struct kunit *test)
 524 {
 525         const struct expect_report expect = {
 526                 .access = {
 527                         { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 528                         { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 529                 },
 530         };
 531         bool match_expect = false;
 532
 533         begin_test_checks(test_kernel_write, test_kernel_write);
 534         do {
 535                 sink_value(READ_ONCE(test_var)); /* induce value-change */
 536                 match_expect = report_matches(&expect);
 537         } while (!end_test_checks(match_expect));
 538         if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC))
 539                 KUNIT_EXPECT_FALSE(test, match_expect);
 540         else
 541                 KUNIT_EXPECT_TRUE(test, match_expect);
 542 }
 543
 544 /*
 545  * Test that data races with writes larger than word-size are always reported,
 546  * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
 547  */
 548 __no_kcsan
 549 static void test_write_write_struct(struct kunit *test)
 550 {
 551         const struct expect_report expect = {
 552                 .access = {
 553                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 554                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 555                 },
 556         };
 557         bool match_expect = false;
 558
 559         begin_test_checks(test_kernel_write_struct, test_kernel_write_struct);
 560         do {
 561                 match_expect = report_matches(&expect);
 562         } while (!end_test_checks(match_expect));
 563         KUNIT_EXPECT_TRUE(test, match_expect);
 564 }
 565
 566 /*
 567  * Test that data races where only one write is larger than word-size are always
 568  * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
 569  */
 570 __no_kcsan
 571 static void test_write_write_struct_part(struct kunit *test)
 572 {
 573         const struct expect_report expect = {
 574                 .access = {
 575                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 576                         { test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE },
 577                 },
 578         };
 579         bool match_expect = false;
 580
 581         begin_test_checks(test_kernel_write_struct, test_kernel_write_struct_part);
 582         do {
 583                 match_expect = report_matches(&expect);
 584         } while (!end_test_checks(match_expect));
 585         KUNIT_EXPECT_TRUE(test, match_expect);
 586 }
 587
 588 /* Test that races with atomic accesses never result in reports. */
 589 __no_kcsan
 590 static void test_read_atomic_write_atomic(struct kunit *test)
 591 {
 592         bool match_never = false;
 593
 594         begin_test_checks(test_kernel_read_atomic, test_kernel_write_atomic);
 595         do {
 596                 match_never = report_available();
 597         } while (!end_test_checks(match_never));
 598         KUNIT_EXPECT_FALSE(test, match_never);
 599 }
 600
 601 /* Test that a race with an atomic and plain access result in reports. */
 602 __no_kcsan
 603 static void test_read_plain_atomic_write(struct kunit *test)
 604 {
 605         const struct expect_report expect = {
 606                 .access = {
 607                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 608                         { test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
 609                 },
 610         };
 611         bool match_expect = false;
 612
 613         if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS))
 614                 return;
 615
 616         begin_test_checks(test_kernel_read, test_kernel_write_atomic);
 617         do {
 618                 match_expect = report_matches(&expect);
 619         } while (!end_test_checks(match_expect));
 620         KUNIT_EXPECT_TRUE(test, match_expect);
 621 }
 622
 623 /* Zero-sized accesses should never cause data race reports. */
 624 __no_kcsan
 625 static void test_zero_size_access(struct kunit *test)
 626 {
 627         const struct expect_report expect = {
 628                 .access = {
 629                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 630                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 631                 },
 632         };
 633         const struct expect_report never = {
 634                 .access = {
 635                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 636                         { test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 },
 637                 },
 638         };
 639         bool match_expect = false;
 640         bool match_never = false;
 641
 642         begin_test_checks(test_kernel_write_struct, test_kernel_read_struct_zero_size);
 643         do {
 644                 match_expect |= report_matches(&expect);
 645                 match_never = report_matches(&never);
 646         } while (!end_test_checks(match_never));
 647         KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */
 648         KUNIT_EXPECT_FALSE(test, match_never);
 649 }
 650
 651 /* Test the data_race() macro. */
 652 __no_kcsan
 653 static void test_data_race(struct kunit *test)
 654 {
 655         bool match_never = false;
 656
 657         begin_test_checks(test_kernel_data_race, test_kernel_data_race);
 658         do {
 659                 match_never = report_available();
 660         } while (!end_test_checks(match_never));
 661         KUNIT_EXPECT_FALSE(test, match_never);
 662 }
 663
 664 __no_kcsan
 665 static void test_assert_exclusive_writer(struct kunit *test)
 666 {
 667         const struct expect_report expect = {
 668                 .access = {
 669                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
 670                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 671                 },
 672         };
 673         bool match_expect = false;
 674
 675         begin_test_checks(test_kernel_assert_writer, test_kernel_write_nochange);
 676         do {
 677                 match_expect = report_matches(&expect);
 678         } while (!end_test_checks(match_expect));
 679         KUNIT_EXPECT_TRUE(test, match_expect);
 680 }
 681
 682 __no_kcsan
 683 static void test_assert_exclusive_access(struct kunit *test)
 684 {
 685         const struct expect_report expect = {
 686                 .access = {
 687                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
 688                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 689                 },
 690         };
 691         bool match_expect = false;
 692
 693         begin_test_checks(test_kernel_assert_access, test_kernel_read);
 694         do {
 695                 match_expect = report_matches(&expect);
 696         } while (!end_test_checks(match_expect));
 697         KUNIT_EXPECT_TRUE(test, match_expect);
 698 }
 699
 700 __no_kcsan
 701 static void test_assert_exclusive_access_writer(struct kunit *test)
 702 {
 703         const struct expect_report expect_access_writer = {
 704                 .access = {
 705                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
 706                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
 707                 },
 708         };
 709         const struct expect_report expect_access_access = {
 710                 .access = {
 711                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
 712                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
 713                 },
 714         };
 715         const struct expect_report never = {
 716                 .access = {
 717                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
 718                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
 719                 },
 720         };
 721         bool match_expect_access_writer = false;
 722         bool match_expect_access_access = false;
 723         bool match_never = false;
 724
 725         begin_test_checks(test_kernel_assert_access, test_kernel_assert_writer);
 726         do {
 727                 match_expect_access_writer |= report_matches(&expect_access_writer);
 728                 match_expect_access_access |= report_matches(&expect_access_access);
 729                 match_never |= report_matches(&never);
 730         } while (!end_test_checks(match_never));
 731         KUNIT_EXPECT_TRUE(test, match_expect_access_writer);
 732         KUNIT_EXPECT_TRUE(test, match_expect_access_access);
 733         KUNIT_EXPECT_FALSE(test, match_never);
 734 }
 735
 736 __no_kcsan
 737 static void test_assert_exclusive_bits_change(struct kunit *test)
 738 {
 739         const struct expect_report expect = {
 740                 .access = {
 741                         { test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
 742                         { test_kernel_change_bits, &test_var, sizeof(test_var),
 743                                 KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) },
 744                 },
 745         };
 746         bool match_expect = false;
 747
 748         begin_test_checks(test_kernel_assert_bits_change, test_kernel_change_bits);
 749         do {
 750                 match_expect = report_matches(&expect);
 751         } while (!end_test_checks(match_expect));
 752         KUNIT_EXPECT_TRUE(test, match_expect);
 753 }
 754
 755 __no_kcsan
 756 static void test_assert_exclusive_bits_nochange(struct kunit *test)
 757 {
 758         bool match_never = false;
 759
 760         begin_test_checks(test_kernel_assert_bits_nochange, test_kernel_change_bits);
 761         do {
 762                 match_never = report_available();
 763         } while (!end_test_checks(match_never));
 764         KUNIT_EXPECT_FALSE(test, match_never);
 765 }
 766
 767 __no_kcsan
 768 static void test_assert_exclusive_writer_scoped(struct kunit *test)
 769 {
 770         const struct expect_report expect_start = {
 771                 .access = {
 772                         { test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
 773                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 774                 },
 775         };
 776         const struct expect_report expect_anywhere = {
 777                 .access = {
 778                         { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
 779                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 780                 },
 781         };
 782         bool match_expect_start = false;
 783         bool match_expect_anywhere = false;
 784
 785         begin_test_checks(test_kernel_assert_writer_scoped, test_kernel_write_nochange);
 786         do {
 787                 match_expect_start |= report_matches(&expect_start);
 788                 match_expect_anywhere |= report_matches(&expect_anywhere);
 789         } while (!end_test_checks(match_expect_start && match_expect_anywhere));
 790         KUNIT_EXPECT_TRUE(test, match_expect_start);
 791         KUNIT_EXPECT_TRUE(test, match_expect_anywhere);
 792 }
 793
 794 __no_kcsan
 795 static void test_assert_exclusive_access_scoped(struct kunit *test)
 796 {
 797         const struct expect_report expect_start1 = {
 798                 .access = {
 799                         { test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
 800                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 801                 },
 802         };
 803         const struct expect_report expect_start2 = {
 804                 .access = { expect_start1.access[0], expect_start1.access[0] },
 805         };
 806         const struct expect_report expect_inscope = {
 807                 .access = {
 808                         { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
 809                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
 810                 },
 811         };
 812         bool match_expect_start = false;
 813         bool match_expect_inscope = false;
 814
 815         begin_test_checks(test_kernel_assert_access_scoped, test_kernel_read);
 816         end_time += msecs_to_jiffies(1000); /* This test requires a bit more time. */
 817         do {
 818                 match_expect_start |= report_matches(&expect_start1) || report_matches(&expect_start2);
 819                 match_expect_inscope |= report_matches(&expect_inscope);
 820         } while (!end_test_checks(match_expect_start && match_expect_inscope));
 821         KUNIT_EXPECT_TRUE(test, match_expect_start);
 822         KUNIT_EXPECT_TRUE(test, match_expect_inscope);
 823 }
 824
 825 /*
 826  * jiffies is special (declared to be volatile) and its accesses are typically
 827  * not marked; this test ensures that the compiler nor KCSAN gets confused about
 828  * jiffies's declaration on different architectures.
 829  */
 830 __no_kcsan
 831 static void test_jiffies_noreport(struct kunit *test)
 832 {
 833         bool match_never = false;
 834
 835         begin_test_checks(test_kernel_jiffies_reader, test_kernel_jiffies_reader);
 836         do {
 837                 match_never = report_available();
 838         } while (!end_test_checks(match_never));
 839         KUNIT_EXPECT_FALSE(test, match_never);
 840 }
 841
 842 /* Test that racing accesses in seqlock critical sections are not reported. */
 843 __no_kcsan
 844 static void test_seqlock_noreport(struct kunit *test)
 845 {
 846         bool match_never = false;
 847
 848         begin_test_checks(test_kernel_seqlock_reader, test_kernel_seqlock_writer);
 849         do {
 850                 match_never = report_available();
 851         } while (!end_test_checks(match_never));
 852         KUNIT_EXPECT_FALSE(test, match_never);
 853 }
 854
 855 /*
 856  * Each test case is run with different numbers of threads. Until KUnit supports
 857  * passing arguments for each test case, we encode #threads in the test case
 858  * name (read by get_num_threads()). [The '-' was chosen as a stylistic
 859  * preference to separate test name and #threads.]
 860  *
 861  * The thread counts are chosen to cover potentially interesting boundaries and
 862  * corner cases (range 2-5), and then stress the system with larger counts.
 863  */
 864 #define KCSAN_KUNIT_CASE(test_name)                                            \
 865         { .run_case = test_name, .name = #test_name "-02" },                   \
 866         { .run_case = test_name, .name = #test_name "-03" },                   \
 867         { .run_case = test_name, .name = #test_name "-04" },                   \
 868         { .run_case = test_name, .name = #test_name "-05" },                   \
 869         { .run_case = test_name, .name = #test_name "-08" },                   \
 870         { .run_case = test_name, .name = #test_name "-16" }
 871
 872 static struct kunit_case kcsan_test_cases[] = {
 873         KCSAN_KUNIT_CASE(test_basic),
 874         KCSAN_KUNIT_CASE(test_concurrent_races),
 875         KCSAN_KUNIT_CASE(test_novalue_change),
 876         KCSAN_KUNIT_CASE(test_novalue_change_exception),
 877         KCSAN_KUNIT_CASE(test_unknown_origin),
 878         KCSAN_KUNIT_CASE(test_write_write_assume_atomic),
 879         KCSAN_KUNIT_CASE(test_write_write_struct),
 880         KCSAN_KUNIT_CASE(test_write_write_struct_part),
 881         KCSAN_KUNIT_CASE(test_read_atomic_write_atomic),
 882         KCSAN_KUNIT_CASE(test_read_plain_atomic_write),
 883         KCSAN_KUNIT_CASE(test_zero_size_access),
 884         KCSAN_KUNIT_CASE(test_data_race),
 885         KCSAN_KUNIT_CASE(test_assert_exclusive_writer),
 886         KCSAN_KUNIT_CASE(test_assert_exclusive_access),
 887         KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer),
 888         KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change),
 889         KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange),
 890         KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped),
 891         KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped),
 892         KCSAN_KUNIT_CASE(test_jiffies_noreport),
 893         KCSAN_KUNIT_CASE(test_seqlock_noreport),
 894         {},
 895 };
 896
 897 /* ===== End test cases ===== */
 898
 899 /* Get number of threads encoded in test name. */
 900 static bool __no_kcsan
 901 get_num_threads(const char *test, int *nthreads)
 902 {
 903         int len = strlen(test);
 904
 905         if (WARN_ON(len < 3))
 906                 return false;
 907
 908         *nthreads = test[len - 1] - '0';
 909         *nthreads += (test[len - 2] - '0') * 10;
 910
 911         if (WARN_ON(*nthreads < 0))
 912                 return false;
 913
 914         return true;
 915 }
 916
 917 /* Concurrent accesses from interrupts. */
 918 __no_kcsan
 919 static void access_thread_timer(struct timer_list *timer)
 920 {
 921         static atomic_t cnt = ATOMIC_INIT(0);
 922         unsigned int idx;
 923         void (*func)(void);
 924
 925         idx = (unsigned int)atomic_inc_return(&cnt) % ARRAY_SIZE(access_kernels);
 926         /* Acquire potential initialization. */
 927         func = smp_load_acquire(&access_kernels[idx]);
 928         if (func)
 929                 func();
 930 }
 931
 932 /* The main loop for each thread. */
 933 __no_kcsan
 934 static int access_thread(void *arg)
 935 {
 936         struct timer_list timer;
 937         unsigned int cnt = 0;
 938         unsigned int idx;
 939         void (*func)(void);
 940
 941         timer_setup_on_stack(&timer, access_thread_timer, 0);
 942         do {
 943                 might_sleep();
 944
 945                 if (!timer_pending(&timer))
 946                         mod_timer(&timer, jiffies + 1);
 947                 else {
 948                         /* Iterate through all kernels. */
 949                         idx = cnt++ % ARRAY_SIZE(access_kernels);
 950                         /* Acquire potential initialization. */
 951                         func = smp_load_acquire(&access_kernels[idx]);
 952                         if (func)
 953                                 func();
 954                 }
 955         } while (!torture_must_stop());
 956         del_timer_sync(&timer);
 957         destroy_timer_on_stack(&timer);
 958
 959         torture_kthread_stopping("access_thread");
 960         return 0;
 961 }
 962
 963 __no_kcsan
 964 static int test_init(struct kunit *test)
 965 {
 966         unsigned long flags;
 967         int nthreads;
 968         int i;
 969
 970         spin_lock_irqsave(&observed.lock, flags);
 971         for (i = 0; i < ARRAY_SIZE(observed.lines); ++i)
 972                 observed.lines[i][0] = '\0';
 973         observed.nlines = 0;
 974         spin_unlock_irqrestore(&observed.lock, flags);
 975
 976         if (!torture_init_begin((char *)test->name, 1))
 977                 return -EBUSY;
 978
 979         if (!get_num_threads(test->name, &nthreads))
 980                 goto err;
 981
 982         if (WARN_ON(threads))
 983                 goto err;
 984
 985         for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) {
 986                 if (WARN_ON(access_kernels[i]))
 987                         goto err;
 988         }
 989
 990         if (!IS_ENABLED(CONFIG_PREEMPT) || !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
 991                 /*
 992                  * Without any preemption, keep 2 CPUs free for other tasks, one
 993                  * of which is the main test case function checking for
 994                  * completion or failure.
 995                  */
 996                 const int min_unused_cpus = IS_ENABLED(CONFIG_PREEMPT_NONE) ? 2 : 0;
 997                 const int min_required_cpus = 2 + min_unused_cpus;
 998
 999                 if (num_online_cpus() < min_required_cpus) {
1000                         pr_err("%s: too few online CPUs (%u < %d) for test",
1001                                test->name, num_online_cpus(), min_required_cpus);
1002                         goto err;
1003                 } else if (nthreads > num_online_cpus() - min_unused_cpus) {
1004                         nthreads = num_online_cpus() - min_unused_cpus;
1005                         pr_warn("%s: limiting number of threads to %d\n",
1006                                 test->name, nthreads);
1007                 }
1008         }
1009
1010         if (nthreads) {
1011                 threads = kcalloc(nthreads + 1, sizeof(struct task_struct *),
1012                                   GFP_KERNEL);
1013                 if (WARN_ON(!threads))
1014                         goto err;
1015
1016                 threads[nthreads] = NULL;
1017                 for (i = 0; i < nthreads; ++i) {
1018                         if (torture_create_kthread(access_thread, NULL,
1019                                                    threads[i]))
1020                                 goto err;
1021                 }
1022         }
1023
1024         torture_init_end();
1025
1026         return 0;
1027
1028 err:
1029         kfree(threads);
1030         threads = NULL;
1031         torture_init_end();
1032         return -EINVAL;
1033 }
1034
1035 __no_kcsan
1036 static void test_exit(struct kunit *test)
1037 {
1038         struct task_struct **stop_thread;
1039         int i;
1040
1041         if (torture_cleanup_begin())
1042                 return;
1043
1044         for (i = 0; i < ARRAY_SIZE(access_kernels); ++i)
1045                 WRITE_ONCE(access_kernels[i], NULL);
1046
1047         if (threads) {
1048                 for (stop_thread = threads; *stop_thread; stop_thread++)
1049                         torture_stop_kthread(reader_thread, *stop_thread);
1050
1051                 kfree(threads);
1052                 threads = NULL;
1053         }
1054
1055         torture_cleanup_end();
1056 }
1057
1058 static struct kunit_suite kcsan_test_suite = {
1059         .name = "kcsan-test",
1060         .test_cases = kcsan_test_cases,
1061         .init = test_init,
1062         .exit = test_exit,
1063 };
1064 static struct kunit_suite *kcsan_test_suites[] = { &kcsan_test_suite, NULL };
1065
1066 __no_kcsan
1067 static void register_tracepoints(struct tracepoint *tp, void *ignore)
1068 {
1069         check_trace_callback_type_console(probe_console);
1070         if (!strcmp(tp->name, "console"))
1071                 WARN_ON(tracepoint_probe_register(tp, probe_console, NULL));
1072 }
1073
1074 __no_kcsan
1075 static void unregister_tracepoints(struct tracepoint *tp, void *ignore)
1076 {
1077         if (!strcmp(tp->name, "console"))
1078                 tracepoint_probe_unregister(tp, probe_console, NULL);
1079 }
1080
1081 /*
1082  * We only want to do tracepoints setup and teardown once, therefore we have to
1083  * customize the init and exit functions and cannot rely on kunit_test_suite().
1084  */
1085 static int __init kcsan_test_init(void)
1086 {
1087         /*
1088          * Because we want to be able to build the test as a module, we need to
1089          * iterate through all known tracepoints, since the static registration
1090          * won't work here.
1091          */
1092         for_each_kernel_tracepoint(register_tracepoints, NULL);
1093         return __kunit_test_suites_init(kcsan_test_suites);
1094 }
1095
1096 static void kcsan_test_exit(void)
1097 {
1098         __kunit_test_suites_exit(kcsan_test_suites);
1099         for_each_kernel_tracepoint(unregister_tracepoints, NULL);
1100         tracepoint_synchronize_unregister();
1101 }
1102
1103 late_initcall(kcsan_test_init);
1104 module_exit(kcsan_test_exit);
1105
1106 MODULE_LICENSE("GPL v2");
1107 MODULE_AUTHOR("Marco Elver <elver@google.com>");