Merge tag 'io_uring-5.13-2021-06-12' of git://git.kernel.dk/linux-block
[linux-2.6-microblaze.git] / kernel / kcsan / kcsan_test.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * KCSAN test with various race scenarious to test runtime behaviour. Since the
4  * interface with which KCSAN's reports are obtained is via the console, this is
5  * the output we should verify. For each test case checks the presence (or
6  * absence) of generated reports. Relies on 'console' tracepoint to capture
7  * reports as they appear in the kernel log.
8  *
9  * Makes use of KUnit for test organization, and the Torture framework for test
10  * thread control.
11  *
12  * Copyright (C) 2020, Google LLC.
13  * Author: Marco Elver <elver@google.com>
14  */
15
16 #define pr_fmt(fmt) "kcsan_test: " fmt
17
18 #include <kunit/test.h>
19 #include <linux/jiffies.h>
20 #include <linux/kcsan-checks.h>
21 #include <linux/kernel.h>
22 #include <linux/sched.h>
23 #include <linux/seqlock.h>
24 #include <linux/spinlock.h>
25 #include <linux/string.h>
26 #include <linux/timer.h>
27 #include <linux/torture.h>
28 #include <linux/tracepoint.h>
29 #include <linux/types.h>
30 #include <trace/events/printk.h>
31
32 #ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE
33 #define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
34 #else
35 #define __KCSAN_ACCESS_RW(alt) (alt)
36 #endif
37
38 /* Points to current test-case memory access "kernels". */
39 static void (*access_kernels[2])(void);
40
41 static struct task_struct **threads; /* Lists of threads. */
42 static unsigned long end_time;       /* End time of test. */
43
44 /* Report as observed from console. */
45 static struct {
46         spinlock_t lock;
47         int nlines;
48         char lines[3][512];
49 } observed = {
50         .lock = __SPIN_LOCK_UNLOCKED(observed.lock),
51 };
52
53 /* Setup test checking loop. */
54 static __no_kcsan inline void
55 begin_test_checks(void (*func1)(void), void (*func2)(void))
56 {
57         kcsan_disable_current();
58
59         /*
60          * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at
61          * least one race is reported.
62          */
63         end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500);
64
65         /* Signal start; release potential initialization of shared data. */
66         smp_store_release(&access_kernels[0], func1);
67         smp_store_release(&access_kernels[1], func2);
68 }
69
70 /* End test checking loop. */
71 static __no_kcsan inline bool
72 end_test_checks(bool stop)
73 {
74         if (!stop && time_before(jiffies, end_time)) {
75                 /* Continue checking */
76                 might_sleep();
77                 return false;
78         }
79
80         kcsan_enable_current();
81         return true;
82 }
83
84 /*
85  * Probe for console output: checks if a race was reported, and obtains observed
86  * lines of interest.
87  */
88 __no_kcsan
89 static void probe_console(void *ignore, const char *buf, size_t len)
90 {
91         unsigned long flags;
92         int nlines;
93
94         /*
95          * Note that KCSAN reports under a global lock, so we do not risk the
96          * possibility of having multiple reports interleaved. If that were the
97          * case, we'd expect tests to fail.
98          */
99
100         spin_lock_irqsave(&observed.lock, flags);
101         nlines = observed.nlines;
102
103         if (strnstr(buf, "BUG: KCSAN: ", len) && strnstr(buf, "test_", len)) {
104                 /*
105                  * KCSAN report and related to the test.
106                  *
107                  * The provided @buf is not NUL-terminated; copy no more than
108                  * @len bytes and let strscpy() add the missing NUL-terminator.
109                  */
110                 strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0])));
111                 nlines = 1;
112         } else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by", len)) {
113                 strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0])));
114
115                 if (strnstr(buf, "race at unknown origin", len)) {
116                         if (WARN_ON(nlines != 2))
117                                 goto out;
118
119                         /* No second line of interest. */
120                         strcpy(observed.lines[nlines++], "<none>");
121                 }
122         }
123
124 out:
125         WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */
126         spin_unlock_irqrestore(&observed.lock, flags);
127 }
128
129 /* Check if a report related to the test exists. */
130 __no_kcsan
131 static bool report_available(void)
132 {
133         return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines);
134 }
135
136 /* Report information we expect in a report. */
137 struct expect_report {
138         /* Access information of both accesses. */
139         struct {
140                 void *fn;    /* Function pointer to expected function of top frame. */
141                 void *addr;  /* Address of access; unchecked if NULL. */
142                 size_t size; /* Size of access; unchecked if @addr is NULL. */
143                 int type;    /* Access type, see KCSAN_ACCESS definitions. */
144         } access[2];
145 };
146
147 /* Check observed report matches information in @r. */
148 __no_kcsan
149 static bool report_matches(const struct expect_report *r)
150 {
151         const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT;
152         bool ret = false;
153         unsigned long flags;
154         typeof(observed.lines) expect;
155         const char *end;
156         char *cur;
157         int i;
158
159         /* Doubled-checked locking. */
160         if (!report_available())
161                 return false;
162
163         /* Generate expected report contents. */
164
165         /* Title */
166         cur = expect[0];
167         end = &expect[0][sizeof(expect[0]) - 1];
168         cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ",
169                          is_assert ? "assert: race" : "data-race");
170         if (r->access[1].fn) {
171                 char tmp[2][64];
172                 int cmp;
173
174                 /* Expect lexographically sorted function names in title. */
175                 scnprintf(tmp[0], sizeof(tmp[0]), "%pS", r->access[0].fn);
176                 scnprintf(tmp[1], sizeof(tmp[1]), "%pS", r->access[1].fn);
177                 cmp = strcmp(tmp[0], tmp[1]);
178                 cur += scnprintf(cur, end - cur, "%ps / %ps",
179                                  cmp < 0 ? r->access[0].fn : r->access[1].fn,
180                                  cmp < 0 ? r->access[1].fn : r->access[0].fn);
181         } else {
182                 scnprintf(cur, end - cur, "%pS", r->access[0].fn);
183                 /* The exact offset won't match, remove it. */
184                 cur = strchr(expect[0], '+');
185                 if (cur)
186                         *cur = '\0';
187         }
188
189         /* Access 1 */
190         cur = expect[1];
191         end = &expect[1][sizeof(expect[1]) - 1];
192         if (!r->access[1].fn)
193                 cur += scnprintf(cur, end - cur, "race at unknown origin, with ");
194
195         /* Access 1 & 2 */
196         for (i = 0; i < 2; ++i) {
197                 const int ty = r->access[i].type;
198                 const char *const access_type =
199                         (ty & KCSAN_ACCESS_ASSERT) ?
200                                       ((ty & KCSAN_ACCESS_WRITE) ?
201                                                "assert no accesses" :
202                                                "assert no writes") :
203                                       ((ty & KCSAN_ACCESS_WRITE) ?
204                                                ((ty & KCSAN_ACCESS_COMPOUND) ?
205                                                         "read-write" :
206                                                         "write") :
207                                                "read");
208                 const char *const access_type_aux =
209                         (ty & KCSAN_ACCESS_ATOMIC) ?
210                                       " (marked)" :
211                                       ((ty & KCSAN_ACCESS_SCOPED) ? " (scoped)" : "");
212
213                 if (i == 1) {
214                         /* Access 2 */
215                         cur = expect[2];
216                         end = &expect[2][sizeof(expect[2]) - 1];
217
218                         if (!r->access[1].fn) {
219                                 /* Dummy string if no second access is available. */
220                                 strcpy(cur, "<none>");
221                                 break;
222                         }
223                 }
224
225                 cur += scnprintf(cur, end - cur, "%s%s to ", access_type,
226                                  access_type_aux);
227
228                 if (r->access[i].addr) /* Address is optional. */
229                         cur += scnprintf(cur, end - cur, "0x%px of %zu bytes",
230                                          r->access[i].addr, r->access[i].size);
231         }
232
233         spin_lock_irqsave(&observed.lock, flags);
234         if (!report_available())
235                 goto out; /* A new report is being captured. */
236
237         /* Finally match expected output to what we actually observed. */
238         ret = strstr(observed.lines[0], expect[0]) &&
239               /* Access info may appear in any order. */
240               ((strstr(observed.lines[1], expect[1]) &&
241                 strstr(observed.lines[2], expect[2])) ||
242                (strstr(observed.lines[1], expect[2]) &&
243                 strstr(observed.lines[2], expect[1])));
244 out:
245         spin_unlock_irqrestore(&observed.lock, flags);
246         return ret;
247 }
248
249 /* ===== Test kernels ===== */
250
251 static long test_sink;
252 static long test_var;
253 /* @test_array should be large enough to fall into multiple watchpoint slots. */
254 static long test_array[3 * PAGE_SIZE / sizeof(long)];
255 static struct {
256         long val[8];
257 } test_struct;
258 static DEFINE_SEQLOCK(test_seqlock);
259
260 /*
261  * Helper to avoid compiler optimizing out reads, and to generate source values
262  * for writes.
263  */
264 __no_kcsan
265 static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); }
266
267 static noinline void test_kernel_read(void) { sink_value(test_var); }
268
269 static noinline void test_kernel_write(void)
270 {
271         test_var = READ_ONCE_NOCHECK(test_sink) + 1;
272 }
273
274 static noinline void test_kernel_write_nochange(void) { test_var = 42; }
275
276 /* Suffixed by value-change exception filter. */
277 static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; }
278
279 static noinline void test_kernel_read_atomic(void)
280 {
281         sink_value(READ_ONCE(test_var));
282 }
283
284 static noinline void test_kernel_write_atomic(void)
285 {
286         WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1);
287 }
288
289 static noinline void test_kernel_atomic_rmw(void)
290 {
291         /* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */
292         __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED);
293 }
294
295 __no_kcsan
296 static noinline void test_kernel_write_uninstrumented(void) { test_var++; }
297
298 static noinline void test_kernel_data_race(void) { data_race(test_var++); }
299
300 static noinline void test_kernel_assert_writer(void)
301 {
302         ASSERT_EXCLUSIVE_WRITER(test_var);
303 }
304
305 static noinline void test_kernel_assert_access(void)
306 {
307         ASSERT_EXCLUSIVE_ACCESS(test_var);
308 }
309
310 #define TEST_CHANGE_BITS 0xff00ff00
311
312 static noinline void test_kernel_change_bits(void)
313 {
314         if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {
315                 /*
316                  * Avoid race of unknown origin for this test, just pretend they
317                  * are atomic.
318                  */
319                 kcsan_nestable_atomic_begin();
320                 test_var ^= TEST_CHANGE_BITS;
321                 kcsan_nestable_atomic_end();
322         } else
323                 WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS);
324 }
325
326 static noinline void test_kernel_assert_bits_change(void)
327 {
328         ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS);
329 }
330
331 static noinline void test_kernel_assert_bits_nochange(void)
332 {
333         ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS);
334 }
335
336 /* To check that scoped assertions do trigger anywhere in scope. */
337 static noinline void test_enter_scope(void)
338 {
339         int x = 0;
340
341         /* Unrelated accesses to scoped assert. */
342         READ_ONCE(test_sink);
343         kcsan_check_read(&x, sizeof(x));
344 }
345
346 static noinline void test_kernel_assert_writer_scoped(void)
347 {
348         ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var);
349         test_enter_scope();
350 }
351
352 static noinline void test_kernel_assert_access_scoped(void)
353 {
354         ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var);
355         test_enter_scope();
356 }
357
358 static noinline void test_kernel_rmw_array(void)
359 {
360         int i;
361
362         for (i = 0; i < ARRAY_SIZE(test_array); ++i)
363                 test_array[i]++;
364 }
365
366 static noinline void test_kernel_write_struct(void)
367 {
368         kcsan_check_write(&test_struct, sizeof(test_struct));
369         kcsan_disable_current();
370         test_struct.val[3]++; /* induce value change */
371         kcsan_enable_current();
372 }
373
374 static noinline void test_kernel_write_struct_part(void)
375 {
376         test_struct.val[3] = 42;
377 }
378
379 static noinline void test_kernel_read_struct_zero_size(void)
380 {
381         kcsan_check_read(&test_struct.val[3], 0);
382 }
383
384 static noinline void test_kernel_jiffies_reader(void)
385 {
386         sink_value((long)jiffies);
387 }
388
389 static noinline void test_kernel_seqlock_reader(void)
390 {
391         unsigned int seq;
392
393         do {
394                 seq = read_seqbegin(&test_seqlock);
395                 sink_value(test_var);
396         } while (read_seqretry(&test_seqlock, seq));
397 }
398
399 static noinline void test_kernel_seqlock_writer(void)
400 {
401         unsigned long flags;
402
403         write_seqlock_irqsave(&test_seqlock, flags);
404         test_var++;
405         write_sequnlock_irqrestore(&test_seqlock, flags);
406 }
407
408 static noinline void test_kernel_atomic_builtins(void)
409 {
410         /*
411          * Generate concurrent accesses, expecting no reports, ensuring KCSAN
412          * treats builtin atomics as actually atomic.
413          */
414         __atomic_load_n(&test_var, __ATOMIC_RELAXED);
415 }
416
417 /* ===== Test cases ===== */
418
419 /* Simple test with normal data race. */
420 __no_kcsan
421 static void test_basic(struct kunit *test)
422 {
423         const struct expect_report expect = {
424                 .access = {
425                         { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
426                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
427                 },
428         };
429         static const struct expect_report never = {
430                 .access = {
431                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
432                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
433                 },
434         };
435         bool match_expect = false;
436         bool match_never = false;
437
438         begin_test_checks(test_kernel_write, test_kernel_read);
439         do {
440                 match_expect |= report_matches(&expect);
441                 match_never = report_matches(&never);
442         } while (!end_test_checks(match_never));
443         KUNIT_EXPECT_TRUE(test, match_expect);
444         KUNIT_EXPECT_FALSE(test, match_never);
445 }
446
447 /*
448  * Stress KCSAN with lots of concurrent races on different addresses until
449  * timeout.
450  */
451 __no_kcsan
452 static void test_concurrent_races(struct kunit *test)
453 {
454         const struct expect_report expect = {
455                 .access = {
456                         /* NULL will match any address. */
457                         { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
458                         { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) },
459                 },
460         };
461         static const struct expect_report never = {
462                 .access = {
463                         { test_kernel_rmw_array, NULL, 0, 0 },
464                         { test_kernel_rmw_array, NULL, 0, 0 },
465                 },
466         };
467         bool match_expect = false;
468         bool match_never = false;
469
470         begin_test_checks(test_kernel_rmw_array, test_kernel_rmw_array);
471         do {
472                 match_expect |= report_matches(&expect);
473                 match_never |= report_matches(&never);
474         } while (!end_test_checks(false));
475         KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */
476         KUNIT_EXPECT_FALSE(test, match_never);
477 }
478
479 /* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */
480 __no_kcsan
481 static void test_novalue_change(struct kunit *test)
482 {
483         const struct expect_report expect = {
484                 .access = {
485                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
486                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
487                 },
488         };
489         bool match_expect = false;
490
491         begin_test_checks(test_kernel_write_nochange, test_kernel_read);
492         do {
493                 match_expect = report_matches(&expect);
494         } while (!end_test_checks(match_expect));
495         if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY))
496                 KUNIT_EXPECT_FALSE(test, match_expect);
497         else
498                 KUNIT_EXPECT_TRUE(test, match_expect);
499 }
500
501 /*
502  * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should
503  * never apply work.
504  */
505 __no_kcsan
506 static void test_novalue_change_exception(struct kunit *test)
507 {
508         const struct expect_report expect = {
509                 .access = {
510                         { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
511                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
512                 },
513         };
514         bool match_expect = false;
515
516         begin_test_checks(test_kernel_write_nochange_rcu, test_kernel_read);
517         do {
518                 match_expect = report_matches(&expect);
519         } while (!end_test_checks(match_expect));
520         KUNIT_EXPECT_TRUE(test, match_expect);
521 }
522
523 /* Test that data races of unknown origin are reported. */
524 __no_kcsan
525 static void test_unknown_origin(struct kunit *test)
526 {
527         const struct expect_report expect = {
528                 .access = {
529                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
530                         { NULL },
531                 },
532         };
533         bool match_expect = false;
534
535         begin_test_checks(test_kernel_write_uninstrumented, test_kernel_read);
536         do {
537                 match_expect = report_matches(&expect);
538         } while (!end_test_checks(match_expect));
539         if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN))
540                 KUNIT_EXPECT_TRUE(test, match_expect);
541         else
542                 KUNIT_EXPECT_FALSE(test, match_expect);
543 }
544
545 /* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */
546 __no_kcsan
547 static void test_write_write_assume_atomic(struct kunit *test)
548 {
549         const struct expect_report expect = {
550                 .access = {
551                         { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
552                         { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
553                 },
554         };
555         bool match_expect = false;
556
557         begin_test_checks(test_kernel_write, test_kernel_write);
558         do {
559                 sink_value(READ_ONCE(test_var)); /* induce value-change */
560                 match_expect = report_matches(&expect);
561         } while (!end_test_checks(match_expect));
562         if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC))
563                 KUNIT_EXPECT_FALSE(test, match_expect);
564         else
565                 KUNIT_EXPECT_TRUE(test, match_expect);
566 }
567
568 /*
569  * Test that data races with writes larger than word-size are always reported,
570  * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
571  */
572 __no_kcsan
573 static void test_write_write_struct(struct kunit *test)
574 {
575         const struct expect_report expect = {
576                 .access = {
577                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
578                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
579                 },
580         };
581         bool match_expect = false;
582
583         begin_test_checks(test_kernel_write_struct, test_kernel_write_struct);
584         do {
585                 match_expect = report_matches(&expect);
586         } while (!end_test_checks(match_expect));
587         KUNIT_EXPECT_TRUE(test, match_expect);
588 }
589
590 /*
591  * Test that data races where only one write is larger than word-size are always
592  * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
593  */
594 __no_kcsan
595 static void test_write_write_struct_part(struct kunit *test)
596 {
597         const struct expect_report expect = {
598                 .access = {
599                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
600                         { test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE },
601                 },
602         };
603         bool match_expect = false;
604
605         begin_test_checks(test_kernel_write_struct, test_kernel_write_struct_part);
606         do {
607                 match_expect = report_matches(&expect);
608         } while (!end_test_checks(match_expect));
609         KUNIT_EXPECT_TRUE(test, match_expect);
610 }
611
612 /* Test that races with atomic accesses never result in reports. */
613 __no_kcsan
614 static void test_read_atomic_write_atomic(struct kunit *test)
615 {
616         bool match_never = false;
617
618         begin_test_checks(test_kernel_read_atomic, test_kernel_write_atomic);
619         do {
620                 match_never = report_available();
621         } while (!end_test_checks(match_never));
622         KUNIT_EXPECT_FALSE(test, match_never);
623 }
624
625 /* Test that a race with an atomic and plain access result in reports. */
626 __no_kcsan
627 static void test_read_plain_atomic_write(struct kunit *test)
628 {
629         const struct expect_report expect = {
630                 .access = {
631                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
632                         { test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
633                 },
634         };
635         bool match_expect = false;
636
637         if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS))
638                 return;
639
640         begin_test_checks(test_kernel_read, test_kernel_write_atomic);
641         do {
642                 match_expect = report_matches(&expect);
643         } while (!end_test_checks(match_expect));
644         KUNIT_EXPECT_TRUE(test, match_expect);
645 }
646
647 /* Test that atomic RMWs generate correct report. */
648 __no_kcsan
649 static void test_read_plain_atomic_rmw(struct kunit *test)
650 {
651         const struct expect_report expect = {
652                 .access = {
653                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
654                         { test_kernel_atomic_rmw, &test_var, sizeof(test_var),
655                                 KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
656                 },
657         };
658         bool match_expect = false;
659
660         if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS))
661                 return;
662
663         begin_test_checks(test_kernel_read, test_kernel_atomic_rmw);
664         do {
665                 match_expect = report_matches(&expect);
666         } while (!end_test_checks(match_expect));
667         KUNIT_EXPECT_TRUE(test, match_expect);
668 }
669
670 /* Zero-sized accesses should never cause data race reports. */
671 __no_kcsan
672 static void test_zero_size_access(struct kunit *test)
673 {
674         const struct expect_report expect = {
675                 .access = {
676                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
677                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
678                 },
679         };
680         const struct expect_report never = {
681                 .access = {
682                         { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
683                         { test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 },
684                 },
685         };
686         bool match_expect = false;
687         bool match_never = false;
688
689         begin_test_checks(test_kernel_write_struct, test_kernel_read_struct_zero_size);
690         do {
691                 match_expect |= report_matches(&expect);
692                 match_never = report_matches(&never);
693         } while (!end_test_checks(match_never));
694         KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */
695         KUNIT_EXPECT_FALSE(test, match_never);
696 }
697
698 /* Test the data_race() macro. */
699 __no_kcsan
700 static void test_data_race(struct kunit *test)
701 {
702         bool match_never = false;
703
704         begin_test_checks(test_kernel_data_race, test_kernel_data_race);
705         do {
706                 match_never = report_available();
707         } while (!end_test_checks(match_never));
708         KUNIT_EXPECT_FALSE(test, match_never);
709 }
710
711 __no_kcsan
712 static void test_assert_exclusive_writer(struct kunit *test)
713 {
714         const struct expect_report expect = {
715                 .access = {
716                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
717                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
718                 },
719         };
720         bool match_expect = false;
721
722         begin_test_checks(test_kernel_assert_writer, test_kernel_write_nochange);
723         do {
724                 match_expect = report_matches(&expect);
725         } while (!end_test_checks(match_expect));
726         KUNIT_EXPECT_TRUE(test, match_expect);
727 }
728
729 __no_kcsan
730 static void test_assert_exclusive_access(struct kunit *test)
731 {
732         const struct expect_report expect = {
733                 .access = {
734                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
735                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
736                 },
737         };
738         bool match_expect = false;
739
740         begin_test_checks(test_kernel_assert_access, test_kernel_read);
741         do {
742                 match_expect = report_matches(&expect);
743         } while (!end_test_checks(match_expect));
744         KUNIT_EXPECT_TRUE(test, match_expect);
745 }
746
747 __no_kcsan
748 static void test_assert_exclusive_access_writer(struct kunit *test)
749 {
750         const struct expect_report expect_access_writer = {
751                 .access = {
752                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
753                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
754                 },
755         };
756         const struct expect_report expect_access_access = {
757                 .access = {
758                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
759                         { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
760                 },
761         };
762         const struct expect_report never = {
763                 .access = {
764                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
765                         { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
766                 },
767         };
768         bool match_expect_access_writer = false;
769         bool match_expect_access_access = false;
770         bool match_never = false;
771
772         begin_test_checks(test_kernel_assert_access, test_kernel_assert_writer);
773         do {
774                 match_expect_access_writer |= report_matches(&expect_access_writer);
775                 match_expect_access_access |= report_matches(&expect_access_access);
776                 match_never |= report_matches(&never);
777         } while (!end_test_checks(match_never));
778         KUNIT_EXPECT_TRUE(test, match_expect_access_writer);
779         KUNIT_EXPECT_TRUE(test, match_expect_access_access);
780         KUNIT_EXPECT_FALSE(test, match_never);
781 }
782
783 __no_kcsan
784 static void test_assert_exclusive_bits_change(struct kunit *test)
785 {
786         const struct expect_report expect = {
787                 .access = {
788                         { test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
789                         { test_kernel_change_bits, &test_var, sizeof(test_var),
790                                 KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) },
791                 },
792         };
793         bool match_expect = false;
794
795         begin_test_checks(test_kernel_assert_bits_change, test_kernel_change_bits);
796         do {
797                 match_expect = report_matches(&expect);
798         } while (!end_test_checks(match_expect));
799         KUNIT_EXPECT_TRUE(test, match_expect);
800 }
801
802 __no_kcsan
803 static void test_assert_exclusive_bits_nochange(struct kunit *test)
804 {
805         bool match_never = false;
806
807         begin_test_checks(test_kernel_assert_bits_nochange, test_kernel_change_bits);
808         do {
809                 match_never = report_available();
810         } while (!end_test_checks(match_never));
811         KUNIT_EXPECT_FALSE(test, match_never);
812 }
813
814 __no_kcsan
815 static void test_assert_exclusive_writer_scoped(struct kunit *test)
816 {
817         const struct expect_report expect_start = {
818                 .access = {
819                         { test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
820                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
821                 },
822         };
823         const struct expect_report expect_anywhere = {
824                 .access = {
825                         { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
826                         { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
827                 },
828         };
829         bool match_expect_start = false;
830         bool match_expect_anywhere = false;
831
832         begin_test_checks(test_kernel_assert_writer_scoped, test_kernel_write_nochange);
833         do {
834                 match_expect_start |= report_matches(&expect_start);
835                 match_expect_anywhere |= report_matches(&expect_anywhere);
836         } while (!end_test_checks(match_expect_start && match_expect_anywhere));
837         KUNIT_EXPECT_TRUE(test, match_expect_start);
838         KUNIT_EXPECT_TRUE(test, match_expect_anywhere);
839 }
840
841 __no_kcsan
842 static void test_assert_exclusive_access_scoped(struct kunit *test)
843 {
844         const struct expect_report expect_start1 = {
845                 .access = {
846                         { test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
847                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
848                 },
849         };
850         const struct expect_report expect_start2 = {
851                 .access = { expect_start1.access[0], expect_start1.access[0] },
852         };
853         const struct expect_report expect_inscope = {
854                 .access = {
855                         { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
856                         { test_kernel_read, &test_var, sizeof(test_var), 0 },
857                 },
858         };
859         bool match_expect_start = false;
860         bool match_expect_inscope = false;
861
862         begin_test_checks(test_kernel_assert_access_scoped, test_kernel_read);
863         end_time += msecs_to_jiffies(1000); /* This test requires a bit more time. */
864         do {
865                 match_expect_start |= report_matches(&expect_start1) || report_matches(&expect_start2);
866                 match_expect_inscope |= report_matches(&expect_inscope);
867         } while (!end_test_checks(match_expect_start && match_expect_inscope));
868         KUNIT_EXPECT_TRUE(test, match_expect_start);
869         KUNIT_EXPECT_TRUE(test, match_expect_inscope);
870 }
871
872 /*
873  * jiffies is special (declared to be volatile) and its accesses are typically
874  * not marked; this test ensures that the compiler nor KCSAN gets confused about
875  * jiffies's declaration on different architectures.
876  */
877 __no_kcsan
878 static void test_jiffies_noreport(struct kunit *test)
879 {
880         bool match_never = false;
881
882         begin_test_checks(test_kernel_jiffies_reader, test_kernel_jiffies_reader);
883         do {
884                 match_never = report_available();
885         } while (!end_test_checks(match_never));
886         KUNIT_EXPECT_FALSE(test, match_never);
887 }
888
889 /* Test that racing accesses in seqlock critical sections are not reported. */
890 __no_kcsan
891 static void test_seqlock_noreport(struct kunit *test)
892 {
893         bool match_never = false;
894
895         begin_test_checks(test_kernel_seqlock_reader, test_kernel_seqlock_writer);
896         do {
897                 match_never = report_available();
898         } while (!end_test_checks(match_never));
899         KUNIT_EXPECT_FALSE(test, match_never);
900 }
901
902 /*
903  * Test atomic builtins work and required instrumentation functions exist. We
904  * also test that KCSAN understands they're atomic by racing with them via
905  * test_kernel_atomic_builtins(), and expect no reports.
906  *
907  * The atomic builtins _SHOULD NOT_ be used in normal kernel code!
908  */
909 static void test_atomic_builtins(struct kunit *test)
910 {
911         bool match_never = false;
912
913         begin_test_checks(test_kernel_atomic_builtins, test_kernel_atomic_builtins);
914         do {
915                 long tmp;
916
917                 kcsan_enable_current();
918
919                 __atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED);
920                 KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED));
921
922                 KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED));
923                 KUNIT_EXPECT_EQ(test, 20L, test_var);
924
925                 tmp = 20L;
926                 KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L,
927                                                                     0, __ATOMIC_RELAXED,
928                                                                     __ATOMIC_RELAXED));
929                 KUNIT_EXPECT_EQ(test, tmp, 20L);
930                 KUNIT_EXPECT_EQ(test, test_var, 30L);
931                 KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L,
932                                                                      1, __ATOMIC_RELAXED,
933                                                                      __ATOMIC_RELAXED));
934                 KUNIT_EXPECT_EQ(test, tmp, 30L);
935                 KUNIT_EXPECT_EQ(test, test_var, 30L);
936
937                 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED));
938                 KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED));
939                 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED));
940                 KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED));
941                 KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED));
942                 KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED));
943                 KUNIT_EXPECT_EQ(test, -2L, test_var);
944
945                 __atomic_thread_fence(__ATOMIC_SEQ_CST);
946                 __atomic_signal_fence(__ATOMIC_SEQ_CST);
947
948                 kcsan_disable_current();
949
950                 match_never = report_available();
951         } while (!end_test_checks(match_never));
952         KUNIT_EXPECT_FALSE(test, match_never);
953 }
954
955 /*
956  * Generate thread counts for all test cases. Values generated are in interval
957  * [2, 5] followed by exponentially increasing thread counts from 8 to 32.
958  *
959  * The thread counts are chosen to cover potentially interesting boundaries and
960  * corner cases (2 to 5), and then stress the system with larger counts.
961  */
962 static const void *nthreads_gen_params(const void *prev, char *desc)
963 {
964         long nthreads = (long)prev;
965
966         if (nthreads < 0 || nthreads >= 32)
967                 nthreads = 0; /* stop */
968         else if (!nthreads)
969                 nthreads = 2; /* initial value */
970         else if (nthreads < 5)
971                 nthreads++;
972         else if (nthreads == 5)
973                 nthreads = 8;
974         else
975                 nthreads *= 2;
976
977         if (!IS_ENABLED(CONFIG_PREEMPT) || !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
978                 /*
979                  * Without any preemption, keep 2 CPUs free for other tasks, one
980                  * of which is the main test case function checking for
981                  * completion or failure.
982                  */
983                 const long min_unused_cpus = IS_ENABLED(CONFIG_PREEMPT_NONE) ? 2 : 0;
984                 const long min_required_cpus = 2 + min_unused_cpus;
985
986                 if (num_online_cpus() < min_required_cpus) {
987                         pr_err_once("Too few online CPUs (%u < %ld) for test\n",
988                                     num_online_cpus(), min_required_cpus);
989                         nthreads = 0;
990                 } else if (nthreads >= num_online_cpus() - min_unused_cpus) {
991                         /* Use negative value to indicate last param. */
992                         nthreads = -(num_online_cpus() - min_unused_cpus);
993                         pr_warn_once("Limiting number of threads to %ld (only %d online CPUs)\n",
994                                      -nthreads, num_online_cpus());
995                 }
996         }
997
998         snprintf(desc, KUNIT_PARAM_DESC_SIZE, "threads=%ld", abs(nthreads));
999         return (void *)nthreads;
1000 }
1001
1002 #define KCSAN_KUNIT_CASE(test_name) KUNIT_CASE_PARAM(test_name, nthreads_gen_params)
1003 static struct kunit_case kcsan_test_cases[] = {
1004         KCSAN_KUNIT_CASE(test_basic),
1005         KCSAN_KUNIT_CASE(test_concurrent_races),
1006         KCSAN_KUNIT_CASE(test_novalue_change),
1007         KCSAN_KUNIT_CASE(test_novalue_change_exception),
1008         KCSAN_KUNIT_CASE(test_unknown_origin),
1009         KCSAN_KUNIT_CASE(test_write_write_assume_atomic),
1010         KCSAN_KUNIT_CASE(test_write_write_struct),
1011         KCSAN_KUNIT_CASE(test_write_write_struct_part),
1012         KCSAN_KUNIT_CASE(test_read_atomic_write_atomic),
1013         KCSAN_KUNIT_CASE(test_read_plain_atomic_write),
1014         KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw),
1015         KCSAN_KUNIT_CASE(test_zero_size_access),
1016         KCSAN_KUNIT_CASE(test_data_race),
1017         KCSAN_KUNIT_CASE(test_assert_exclusive_writer),
1018         KCSAN_KUNIT_CASE(test_assert_exclusive_access),
1019         KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer),
1020         KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change),
1021         KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange),
1022         KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped),
1023         KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped),
1024         KCSAN_KUNIT_CASE(test_jiffies_noreport),
1025         KCSAN_KUNIT_CASE(test_seqlock_noreport),
1026         KCSAN_KUNIT_CASE(test_atomic_builtins),
1027         {},
1028 };
1029
1030 /* ===== End test cases ===== */
1031
1032 /* Concurrent accesses from interrupts. */
1033 __no_kcsan
1034 static void access_thread_timer(struct timer_list *timer)
1035 {
1036         static atomic_t cnt = ATOMIC_INIT(0);
1037         unsigned int idx;
1038         void (*func)(void);
1039
1040         idx = (unsigned int)atomic_inc_return(&cnt) % ARRAY_SIZE(access_kernels);
1041         /* Acquire potential initialization. */
1042         func = smp_load_acquire(&access_kernels[idx]);
1043         if (func)
1044                 func();
1045 }
1046
1047 /* The main loop for each thread. */
1048 __no_kcsan
1049 static int access_thread(void *arg)
1050 {
1051         struct timer_list timer;
1052         unsigned int cnt = 0;
1053         unsigned int idx;
1054         void (*func)(void);
1055
1056         timer_setup_on_stack(&timer, access_thread_timer, 0);
1057         do {
1058                 might_sleep();
1059
1060                 if (!timer_pending(&timer))
1061                         mod_timer(&timer, jiffies + 1);
1062                 else {
1063                         /* Iterate through all kernels. */
1064                         idx = cnt++ % ARRAY_SIZE(access_kernels);
1065                         /* Acquire potential initialization. */
1066                         func = smp_load_acquire(&access_kernels[idx]);
1067                         if (func)
1068                                 func();
1069                 }
1070         } while (!torture_must_stop());
1071         del_timer_sync(&timer);
1072         destroy_timer_on_stack(&timer);
1073
1074         torture_kthread_stopping("access_thread");
1075         return 0;
1076 }
1077
1078 __no_kcsan
1079 static int test_init(struct kunit *test)
1080 {
1081         unsigned long flags;
1082         int nthreads;
1083         int i;
1084
1085         spin_lock_irqsave(&observed.lock, flags);
1086         for (i = 0; i < ARRAY_SIZE(observed.lines); ++i)
1087                 observed.lines[i][0] = '\0';
1088         observed.nlines = 0;
1089         spin_unlock_irqrestore(&observed.lock, flags);
1090
1091         if (!torture_init_begin((char *)test->name, 1))
1092                 return -EBUSY;
1093
1094         if (WARN_ON(threads))
1095                 goto err;
1096
1097         for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) {
1098                 if (WARN_ON(access_kernels[i]))
1099                         goto err;
1100         }
1101
1102         nthreads = abs((long)test->param_value);
1103         if (WARN_ON(!nthreads))
1104                 goto err;
1105
1106         threads = kcalloc(nthreads + 1, sizeof(struct task_struct *), GFP_KERNEL);
1107         if (WARN_ON(!threads))
1108                 goto err;
1109
1110         threads[nthreads] = NULL;
1111         for (i = 0; i < nthreads; ++i) {
1112                 if (torture_create_kthread(access_thread, NULL, threads[i]))
1113                         goto err;
1114         }
1115
1116         torture_init_end();
1117
1118         return 0;
1119
1120 err:
1121         kfree(threads);
1122         threads = NULL;
1123         torture_init_end();
1124         return -EINVAL;
1125 }
1126
1127 __no_kcsan
1128 static void test_exit(struct kunit *test)
1129 {
1130         struct task_struct **stop_thread;
1131         int i;
1132
1133         if (torture_cleanup_begin())
1134                 return;
1135
1136         for (i = 0; i < ARRAY_SIZE(access_kernels); ++i)
1137                 WRITE_ONCE(access_kernels[i], NULL);
1138
1139         if (threads) {
1140                 for (stop_thread = threads; *stop_thread; stop_thread++)
1141                         torture_stop_kthread(reader_thread, *stop_thread);
1142
1143                 kfree(threads);
1144                 threads = NULL;
1145         }
1146
1147         torture_cleanup_end();
1148 }
1149
1150 static struct kunit_suite kcsan_test_suite = {
1151         .name = "kcsan",
1152         .test_cases = kcsan_test_cases,
1153         .init = test_init,
1154         .exit = test_exit,
1155 };
1156 static struct kunit_suite *kcsan_test_suites[] = { &kcsan_test_suite, NULL };
1157
1158 __no_kcsan
1159 static void register_tracepoints(struct tracepoint *tp, void *ignore)
1160 {
1161         check_trace_callback_type_console(probe_console);
1162         if (!strcmp(tp->name, "console"))
1163                 WARN_ON(tracepoint_probe_register(tp, probe_console, NULL));
1164 }
1165
1166 __no_kcsan
1167 static void unregister_tracepoints(struct tracepoint *tp, void *ignore)
1168 {
1169         if (!strcmp(tp->name, "console"))
1170                 tracepoint_probe_unregister(tp, probe_console, NULL);
1171 }
1172
1173 /*
1174  * We only want to do tracepoints setup and teardown once, therefore we have to
1175  * customize the init and exit functions and cannot rely on kunit_test_suite().
1176  */
1177 static int __init kcsan_test_init(void)
1178 {
1179         /*
1180          * Because we want to be able to build the test as a module, we need to
1181          * iterate through all known tracepoints, since the static registration
1182          * won't work here.
1183          */
1184         for_each_kernel_tracepoint(register_tracepoints, NULL);
1185         return __kunit_test_suites_init(kcsan_test_suites);
1186 }
1187
1188 static void kcsan_test_exit(void)
1189 {
1190         __kunit_test_suites_exit(kcsan_test_suites);
1191         for_each_kernel_tracepoint(unregister_tracepoints, NULL);
1192         tracepoint_synchronize_unregister();
1193 }
1194
1195 late_initcall(kcsan_test_init);
1196 module_exit(kcsan_test_exit);
1197
1198 MODULE_LICENSE("GPL v2");
1199 MODULE_AUTHOR("Marco Elver <elver@google.com>");