kernel/kcsan/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/atomic.h>
   4 #include <linux/bug.h>
   5 #include <linux/delay.h>
   6 #include <linux/export.h>
   7 #include <linux/init.h>
   8 #include <linux/kernel.h>
   9 #include <linux/list.h>
  10 #include <linux/moduleparam.h>
  11 #include <linux/percpu.h>
  12 #include <linux/preempt.h>
  13 #include <linux/random.h>
  14 #include <linux/sched.h>
  15 #include <linux/uaccess.h>
  16
  17 #include "atomic.h"
  18 #include "encoding.h"
  19 #include "kcsan.h"
  20
  21 static bool kcsan_early_enable = IS_ENABLED(CONFIG_KCSAN_EARLY_ENABLE);
  22 unsigned int kcsan_udelay_task = CONFIG_KCSAN_UDELAY_TASK;
  23 unsigned int kcsan_udelay_interrupt = CONFIG_KCSAN_UDELAY_INTERRUPT;
  24 static long kcsan_skip_watch = CONFIG_KCSAN_SKIP_WATCH;
  25 static bool kcsan_interrupt_watcher = IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER);
  26
  27 #ifdef MODULE_PARAM_PREFIX
  28 #undef MODULE_PARAM_PREFIX
  29 #endif
  30 #define MODULE_PARAM_PREFIX "kcsan."
  31 module_param_named(early_enable, kcsan_early_enable, bool, 0);
  32 module_param_named(udelay_task, kcsan_udelay_task, uint, 0644);
  33 module_param_named(udelay_interrupt, kcsan_udelay_interrupt, uint, 0644);
  34 module_param_named(skip_watch, kcsan_skip_watch, long, 0644);
  35 module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444);
  36
  37 bool kcsan_enabled;
  38
  39 /* Per-CPU kcsan_ctx for interrupts */
  40 static DEFINE_PER_CPU(struct kcsan_ctx, kcsan_cpu_ctx) = {
  41         .disable_count          = 0,
  42         .atomic_next            = 0,
  43         .atomic_nest_count      = 0,
  44         .in_flat_atomic         = false,
  45         .access_mask            = 0,
  46         .scoped_accesses        = {LIST_POISON1, NULL},
  47 };
  48
  49 /*
  50  * Helper macros to index into adjacent slots, starting from address slot
  51  * itself, followed by the right and left slots.
  52  *
  53  * The purpose is 2-fold:
  54  *
  55  *      1. if during insertion the address slot is already occupied, check if
  56  *         any adjacent slots are free;
  57  *      2. accesses that straddle a slot boundary due to size that exceeds a
  58  *         slot's range may check adjacent slots if any watchpoint matches.
  59  *
  60  * Note that accesses with very large size may still miss a watchpoint; however,
  61  * given this should be rare, this is a reasonable trade-off to make, since this
  62  * will avoid:
  63  *
  64  *      1. excessive contention between watchpoint checks and setup;
  65  *      2. larger number of simultaneous watchpoints without sacrificing
  66  *         performance.
  67  *
  68  * Example: SLOT_IDX values for KCSAN_CHECK_ADJACENT=1, where i is [0, 1, 2]:
  69  *
  70  *   slot=0:  [ 1,  2,  0]
  71  *   slot=9:  [10, 11,  9]
  72  *   slot=63: [64, 65, 63]
  73  */
  74 #define SLOT_IDX(slot, i) (slot + ((i + KCSAN_CHECK_ADJACENT) % NUM_SLOTS))
  75
  76 /*
  77  * SLOT_IDX_FAST is used in the fast-path. Not first checking the address's primary
  78  * slot (middle) is fine if we assume that races occur rarely. The set of
  79  * indices {SLOT_IDX(slot, i) | i in [0, NUM_SLOTS)} is equivalent to
  80  * {SLOT_IDX_FAST(slot, i) | i in [0, NUM_SLOTS)}.
  81  */
  82 #define SLOT_IDX_FAST(slot, i) (slot + i)
  83
  84 /*
  85  * Watchpoints, with each entry encoded as defined in encoding.h: in order to be
  86  * able to safely update and access a watchpoint without introducing locking
  87  * overhead, we encode each watchpoint as a single atomic long. The initial
  88  * zero-initialized state matches INVALID_WATCHPOINT.
  89  *
  90  * Add NUM_SLOTS-1 entries to account for overflow; this helps avoid having to
  91  * use more complicated SLOT_IDX_FAST calculation with modulo in the fast-path.
  92  */
  93 static atomic_long_t watchpoints[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS-1];
  94
  95 /*
  96  * Instructions to skip watching counter, used in should_watch(). We use a
  97  * per-CPU counter to avoid excessive contention.
  98  */
  99 static DEFINE_PER_CPU(long, kcsan_skip);
 100
 101 static __always_inline atomic_long_t *find_watchpoint(unsigned long addr,
 102                                                       size_t size,
 103                                                       bool expect_write,
 104                                                       long *encoded_watchpoint)
 105 {
 106         const int slot = watchpoint_slot(addr);
 107         const unsigned long addr_masked = addr & WATCHPOINT_ADDR_MASK;
 108         atomic_long_t *watchpoint;
 109         unsigned long wp_addr_masked;
 110         size_t wp_size;
 111         bool is_write;
 112         int i;
 113
 114         BUILD_BUG_ON(CONFIG_KCSAN_NUM_WATCHPOINTS < NUM_SLOTS);
 115
 116         for (i = 0; i < NUM_SLOTS; ++i) {
 117                 watchpoint = &watchpoints[SLOT_IDX_FAST(slot, i)];
 118                 *encoded_watchpoint = atomic_long_read(watchpoint);
 119                 if (!decode_watchpoint(*encoded_watchpoint, &wp_addr_masked,
 120                                        &wp_size, &is_write))
 121                         continue;
 122
 123                 if (expect_write && !is_write)
 124                         continue;
 125
 126                 /* Check if the watchpoint matches the access. */
 127                 if (matching_access(wp_addr_masked, wp_size, addr_masked, size))
 128                         return watchpoint;
 129         }
 130
 131         return NULL;
 132 }
 133
 134 static inline atomic_long_t *
 135 insert_watchpoint(unsigned long addr, size_t size, bool is_write)
 136 {
 137         const int slot = watchpoint_slot(addr);
 138         const long encoded_watchpoint = encode_watchpoint(addr, size, is_write);
 139         atomic_long_t *watchpoint;
 140         int i;
 141
 142         /* Check slot index logic, ensuring we stay within array bounds. */
 143         BUILD_BUG_ON(SLOT_IDX(0, 0) != KCSAN_CHECK_ADJACENT);
 144         BUILD_BUG_ON(SLOT_IDX(0, KCSAN_CHECK_ADJACENT+1) != 0);
 145         BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS-1, KCSAN_CHECK_ADJACENT) != ARRAY_SIZE(watchpoints)-1);
 146         BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS-1, KCSAN_CHECK_ADJACENT+1) != ARRAY_SIZE(watchpoints) - NUM_SLOTS);
 147
 148         for (i = 0; i < NUM_SLOTS; ++i) {
 149                 long expect_val = INVALID_WATCHPOINT;
 150
 151                 /* Try to acquire this slot. */
 152                 watchpoint = &watchpoints[SLOT_IDX(slot, i)];
 153                 if (atomic_long_try_cmpxchg_relaxed(watchpoint, &expect_val, encoded_watchpoint))
 154                         return watchpoint;
 155         }
 156
 157         return NULL;
 158 }
 159
 160 /*
 161  * Return true if watchpoint was successfully consumed, false otherwise.
 162  *
 163  * This may return false if:
 164  *
 165  *      1. another thread already consumed the watchpoint;
 166  *      2. the thread that set up the watchpoint already removed it;
 167  *      3. the watchpoint was removed and then re-used.
 168  */
 169 static __always_inline bool
 170 try_consume_watchpoint(atomic_long_t *watchpoint, long encoded_watchpoint)
 171 {
 172         return atomic_long_try_cmpxchg_relaxed(watchpoint, &encoded_watchpoint, CONSUMED_WATCHPOINT);
 173 }
 174
 175 /* Return true if watchpoint was not touched, false if already consumed. */
 176 static inline bool consume_watchpoint(atomic_long_t *watchpoint)
 177 {
 178         return atomic_long_xchg_relaxed(watchpoint, CONSUMED_WATCHPOINT) != CONSUMED_WATCHPOINT;
 179 }
 180
 181 /* Remove the watchpoint -- its slot may be reused after. */
 182 static inline void remove_watchpoint(atomic_long_t *watchpoint)
 183 {
 184         atomic_long_set(watchpoint, INVALID_WATCHPOINT);
 185 }
 186
 187 static __always_inline struct kcsan_ctx *get_ctx(void)
 188 {
 189         /*
 190          * In interrupts, use raw_cpu_ptr to avoid unnecessary checks, that would
 191          * also result in calls that generate warnings in uaccess regions.
 192          */
 193         return in_task() ? &current->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);
 194 }
 195
 196 /* Check scoped accesses; never inline because this is a slow-path! */
 197 static noinline void kcsan_check_scoped_accesses(void)
 198 {
 199         struct kcsan_ctx *ctx = get_ctx();
 200         struct list_head *prev_save = ctx->scoped_accesses.prev;
 201         struct kcsan_scoped_access *scoped_access;
 202
 203         ctx->scoped_accesses.prev = NULL;  /* Avoid recursion. */
 204         list_for_each_entry(scoped_access, &ctx->scoped_accesses, list)
 205                 __kcsan_check_access(scoped_access->ptr, scoped_access->size, scoped_access->type);
 206         ctx->scoped_accesses.prev = prev_save;
 207 }
 208
 209 /* Rules for generic atomic accesses. Called from fast-path. */
 210 static __always_inline bool
 211 is_atomic(const volatile void *ptr, size_t size, int type, struct kcsan_ctx *ctx)
 212 {
 213         if (type & KCSAN_ACCESS_ATOMIC)
 214                 return true;
 215
 216         /*
 217          * Unless explicitly declared atomic, never consider an assertion access
 218          * as atomic. This allows using them also in atomic regions, such as
 219          * seqlocks, without implicitly changing their semantics.
 220          */
 221         if (type & KCSAN_ACCESS_ASSERT)
 222                 return false;
 223
 224         if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) &&
 225             (type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) &&
 226             IS_ALIGNED((unsigned long)ptr, size))
 227                 return true; /* Assume aligned writes up to word size are atomic. */
 228
 229         if (ctx->atomic_next > 0) {
 230                 /*
 231                  * Because we do not have separate contexts for nested
 232                  * interrupts, in case atomic_next is set, we simply assume that
 233                  * the outer interrupt set atomic_next. In the worst case, we
 234                  * will conservatively consider operations as atomic. This is a
 235                  * reasonable trade-off to make, since this case should be
 236                  * extremely rare; however, even if extremely rare, it could
 237                  * lead to false positives otherwise.
 238                  */
 239                 if ((hardirq_count() >> HARDIRQ_SHIFT) < 2)
 240                         --ctx->atomic_next; /* in task, or outer interrupt */
 241                 return true;
 242         }
 243
 244         return ctx->atomic_nest_count > 0 || ctx->in_flat_atomic;
 245 }
 246
 247 static __always_inline bool
 248 should_watch(const volatile void *ptr, size_t size, int type, struct kcsan_ctx *ctx)
 249 {
 250         /*
 251          * Never set up watchpoints when memory operations are atomic.
 252          *
 253          * Need to check this first, before kcsan_skip check below: (1) atomics
 254          * should not count towards skipped instructions, and (2) to actually
 255          * decrement kcsan_atomic_next for consecutive instruction stream.
 256          */
 257         if (is_atomic(ptr, size, type, ctx))
 258                 return false;
 259
 260         if (this_cpu_dec_return(kcsan_skip) >= 0)
 261                 return false;
 262
 263         /*
 264          * NOTE: If we get here, kcsan_skip must always be reset in slow path
 265          * via reset_kcsan_skip() to avoid underflow.
 266          */
 267
 268         /* this operation should be watched */
 269         return true;
 270 }
 271
 272 static inline void reset_kcsan_skip(void)
 273 {
 274         long skip_count = kcsan_skip_watch -
 275                           (IS_ENABLED(CONFIG_KCSAN_SKIP_WATCH_RANDOMIZE) ?
 276                                    prandom_u32_max(kcsan_skip_watch) :
 277                                    0);
 278         this_cpu_write(kcsan_skip, skip_count);
 279 }
 280
 281 static __always_inline bool kcsan_is_enabled(void)
 282 {
 283         return READ_ONCE(kcsan_enabled) && get_ctx()->disable_count == 0;
 284 }
 285
 286 static inline unsigned int get_delay(void)
 287 {
 288         unsigned int delay = in_task() ? kcsan_udelay_task : kcsan_udelay_interrupt;
 289         return delay - (IS_ENABLED(CONFIG_KCSAN_DELAY_RANDOMIZE) ?
 290                                 prandom_u32_max(delay) :
 291                                 0);
 292 }
 293
 294 /*
 295  * Pull everything together: check_access() below contains the performance
 296  * critical operations; the fast-path (including check_access) functions should
 297  * all be inlinable by the instrumentation functions.
 298  *
 299  * The slow-path (kcsan_found_watchpoint, kcsan_setup_watchpoint) are
 300  * non-inlinable -- note that, we prefix these with "kcsan_" to ensure they can
 301  * be filtered from the stacktrace, as well as give them unique names for the
 302  * UACCESS whitelist of objtool. Each function uses user_access_save/restore(),
 303  * since they do not access any user memory, but instrumentation is still
 304  * emitted in UACCESS regions.
 305  */
 306
 307 static noinline void kcsan_found_watchpoint(const volatile void *ptr,
 308                                             size_t size,
 309                                             int type,
 310                                             atomic_long_t *watchpoint,
 311                                             long encoded_watchpoint)
 312 {
 313         unsigned long flags;
 314         bool consumed;
 315
 316         if (!kcsan_is_enabled())
 317                 return;
 318
 319         /*
 320          * The access_mask check relies on value-change comparison. To avoid
 321          * reporting a race where e.g. the writer set up the watchpoint, but the
 322          * reader has access_mask!=0, we have to ignore the found watchpoint.
 323          */
 324         if (get_ctx()->access_mask != 0)
 325                 return;
 326
 327         /*
 328          * Consume the watchpoint as soon as possible, to minimize the chances
 329          * of !consumed. Consuming the watchpoint must always be guarded by
 330          * kcsan_is_enabled() check, as otherwise we might erroneously
 331          * triggering reports when disabled.
 332          */
 333         consumed = try_consume_watchpoint(watchpoint, encoded_watchpoint);
 334
 335         /* keep this after try_consume_watchpoint */
 336         flags = user_access_save();
 337
 338         if (consumed) {
 339                 kcsan_report(ptr, size, type, KCSAN_VALUE_CHANGE_MAYBE,
 340                              KCSAN_REPORT_CONSUMED_WATCHPOINT,
 341                              watchpoint - watchpoints);
 342         } else {
 343                 /*
 344                  * The other thread may not print any diagnostics, as it has
 345                  * already removed the watchpoint, or another thread consumed
 346                  * the watchpoint before this thread.
 347                  */
 348                 kcsan_counter_inc(KCSAN_COUNTER_REPORT_RACES);
 349         }
 350
 351         if ((type & KCSAN_ACCESS_ASSERT) != 0)
 352                 kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES);
 353         else
 354                 kcsan_counter_inc(KCSAN_COUNTER_DATA_RACES);
 355
 356         user_access_restore(flags);
 357 }
 358
 359 static noinline void
 360 kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type)
 361 {
 362         const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0;
 363         const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0;
 364         atomic_long_t *watchpoint;
 365         union {
 366                 u8 _1;
 367                 u16 _2;
 368                 u32 _4;
 369                 u64 _8;
 370         } expect_value;
 371         unsigned long access_mask;
 372         enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE;
 373         unsigned long ua_flags = user_access_save();
 374         unsigned long irq_flags = 0;
 375
 376         /*
 377          * Always reset kcsan_skip counter in slow-path to avoid underflow; see
 378          * should_watch().
 379          */
 380         reset_kcsan_skip();
 381
 382         if (!kcsan_is_enabled())
 383                 goto out;
 384
 385         /*
 386          * Special atomic rules: unlikely to be true, so we check them here in
 387          * the slow-path, and not in the fast-path in is_atomic(). Call after
 388          * kcsan_is_enabled(), as we may access memory that is not yet
 389          * initialized during early boot.
 390          */
 391         if (!is_assert && kcsan_is_atomic_special(ptr))
 392                 goto out;
 393
 394         if (!check_encodable((unsigned long)ptr, size)) {
 395                 kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES);
 396                 goto out;
 397         }
 398
 399         if (!kcsan_interrupt_watcher)
 400                 /* Use raw to avoid lockdep recursion via IRQ flags tracing. */
 401                 raw_local_irq_save(irq_flags);
 402
 403         watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write);
 404         if (watchpoint == NULL) {
 405                 /*
 406                  * Out of capacity: the size of 'watchpoints', and the frequency
 407                  * with which should_watch() returns true should be tweaked so
 408                  * that this case happens very rarely.
 409                  */
 410                 kcsan_counter_inc(KCSAN_COUNTER_NO_CAPACITY);
 411                 goto out_unlock;
 412         }
 413
 414         kcsan_counter_inc(KCSAN_COUNTER_SETUP_WATCHPOINTS);
 415         kcsan_counter_inc(KCSAN_COUNTER_USED_WATCHPOINTS);
 416
 417         /*
 418          * Read the current value, to later check and infer a race if the data
 419          * was modified via a non-instrumented access, e.g. from a device.
 420          */
 421         expect_value._8 = 0;
 422         switch (size) {
 423         case 1:
 424                 expect_value._1 = READ_ONCE(*(const u8 *)ptr);
 425                 break;
 426         case 2:
 427                 expect_value._2 = READ_ONCE(*(const u16 *)ptr);
 428                 break;
 429         case 4:
 430                 expect_value._4 = READ_ONCE(*(const u32 *)ptr);
 431                 break;
 432         case 8:
 433                 expect_value._8 = READ_ONCE(*(const u64 *)ptr);
 434                 break;
 435         default:
 436                 break; /* ignore; we do not diff the values */
 437         }
 438
 439         if (IS_ENABLED(CONFIG_KCSAN_DEBUG)) {
 440                 kcsan_disable_current();
 441                 pr_err("KCSAN: watching %s, size: %zu, addr: %px [slot: %d, encoded: %lx]\n",
 442                        is_write ? "write" : "read", size, ptr,
 443                        watchpoint_slot((unsigned long)ptr),
 444                        encode_watchpoint((unsigned long)ptr, size, is_write));
 445                 kcsan_enable_current();
 446         }
 447
 448         /*
 449          * Delay this thread, to increase probability of observing a racy
 450          * conflicting access.
 451          */
 452         udelay(get_delay());
 453
 454         /*
 455          * Re-read value, and check if it is as expected; if not, we infer a
 456          * racy access.
 457          */
 458         access_mask = get_ctx()->access_mask;
 459         switch (size) {
 460         case 1:
 461                 expect_value._1 ^= READ_ONCE(*(const u8 *)ptr);
 462                 if (access_mask)
 463                         expect_value._1 &= (u8)access_mask;
 464                 break;
 465         case 2:
 466                 expect_value._2 ^= READ_ONCE(*(const u16 *)ptr);
 467                 if (access_mask)
 468                         expect_value._2 &= (u16)access_mask;
 469                 break;
 470         case 4:
 471                 expect_value._4 ^= READ_ONCE(*(const u32 *)ptr);
 472                 if (access_mask)
 473                         expect_value._4 &= (u32)access_mask;
 474                 break;
 475         case 8:
 476                 expect_value._8 ^= READ_ONCE(*(const u64 *)ptr);
 477                 if (access_mask)
 478                         expect_value._8 &= (u64)access_mask;
 479                 break;
 480         default:
 481                 break; /* ignore; we do not diff the values */
 482         }
 483
 484         /* Were we able to observe a value-change? */
 485         if (expect_value._8 != 0)
 486                 value_change = KCSAN_VALUE_CHANGE_TRUE;
 487
 488         /* Check if this access raced with another. */
 489         if (!consume_watchpoint(watchpoint)) {
 490                 /*
 491                  * Depending on the access type, map a value_change of MAYBE to
 492                  * TRUE (always report) or FALSE (never report).
 493                  */
 494                 if (value_change == KCSAN_VALUE_CHANGE_MAYBE) {
 495                         if (access_mask != 0) {
 496                                 /*
 497                                  * For access with access_mask, we require a
 498                                  * value-change, as it is likely that races on
 499                                  * ~access_mask bits are expected.
 500                                  */
 501                                 value_change = KCSAN_VALUE_CHANGE_FALSE;
 502                         } else if (size > 8 || is_assert) {
 503                                 /* Always assume a value-change. */
 504                                 value_change = KCSAN_VALUE_CHANGE_TRUE;
 505                         }
 506                 }
 507
 508                 /*
 509                  * No need to increment 'data_races' counter, as the racing
 510                  * thread already did.
 511                  *
 512                  * Count 'assert_failures' for each failed ASSERT access,
 513                  * therefore both this thread and the racing thread may
 514                  * increment this counter.
 515                  */
 516                 if (is_assert && value_change == KCSAN_VALUE_CHANGE_TRUE)
 517                         kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES);
 518
 519                 kcsan_report(ptr, size, type, value_change, KCSAN_REPORT_RACE_SIGNAL,
 520                              watchpoint - watchpoints);
 521         } else if (value_change == KCSAN_VALUE_CHANGE_TRUE) {
 522                 /* Inferring a race, since the value should not have changed. */
 523
 524                 kcsan_counter_inc(KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN);
 525                 if (is_assert)
 526                         kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES);
 527
 528                 if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN) || is_assert)
 529                         kcsan_report(ptr, size, type, KCSAN_VALUE_CHANGE_TRUE,
 530                                      KCSAN_REPORT_RACE_UNKNOWN_ORIGIN,
 531                                      watchpoint - watchpoints);
 532         }
 533
 534         /*
 535          * Remove watchpoint; must be after reporting, since the slot may be
 536          * reused after this point.
 537          */
 538         remove_watchpoint(watchpoint);
 539         kcsan_counter_dec(KCSAN_COUNTER_USED_WATCHPOINTS);
 540 out_unlock:
 541         if (!kcsan_interrupt_watcher)
 542                 raw_local_irq_restore(irq_flags);
 543 out:
 544         user_access_restore(ua_flags);
 545 }
 546
 547 static __always_inline void check_access(const volatile void *ptr, size_t size,
 548                                          int type)
 549 {
 550         const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0;
 551         atomic_long_t *watchpoint;
 552         long encoded_watchpoint;
 553
 554         /*
 555          * Do nothing for 0 sized check; this comparison will be optimized out
 556          * for constant sized instrumentation (__tsan_{read,write}N).
 557          */
 558         if (unlikely(size == 0))
 559                 return;
 560
 561         /*
 562          * Avoid user_access_save in fast-path: find_watchpoint is safe without
 563          * user_access_save, as the address that ptr points to is only used to
 564          * check if a watchpoint exists; ptr is never dereferenced.
 565          */
 566         watchpoint = find_watchpoint((unsigned long)ptr, size, !is_write,
 567                                      &encoded_watchpoint);
 568         /*
 569          * It is safe to check kcsan_is_enabled() after find_watchpoint in the
 570          * slow-path, as long as no state changes that cause a race to be
 571          * detected and reported have occurred until kcsan_is_enabled() is
 572          * checked.
 573          */
 574
 575         if (unlikely(watchpoint != NULL))
 576                 kcsan_found_watchpoint(ptr, size, type, watchpoint,
 577                                        encoded_watchpoint);
 578         else {
 579                 struct kcsan_ctx *ctx = get_ctx(); /* Call only once in fast-path. */
 580
 581                 if (unlikely(should_watch(ptr, size, type, ctx)))
 582                         kcsan_setup_watchpoint(ptr, size, type);
 583                 else if (unlikely(ctx->scoped_accesses.prev))
 584                         kcsan_check_scoped_accesses();
 585         }
 586 }
 587
 588 /* === Public interface ===================================================== */
 589
 590 void __init kcsan_init(void)
 591 {
 592         BUG_ON(!in_task());
 593
 594         kcsan_debugfs_init();
 595
 596         /*
 597          * We are in the init task, and no other tasks should be running;
 598          * WRITE_ONCE without memory barrier is sufficient.
 599          */
 600         if (kcsan_early_enable)
 601                 WRITE_ONCE(kcsan_enabled, true);
 602 }
 603
 604 /* === Exported interface =================================================== */
 605
 606 void kcsan_disable_current(void)
 607 {
 608         ++get_ctx()->disable_count;
 609 }
 610 EXPORT_SYMBOL(kcsan_disable_current);
 611
 612 void kcsan_enable_current(void)
 613 {
 614         if (get_ctx()->disable_count-- == 0) {
 615                 /*
 616                  * Warn if kcsan_enable_current() calls are unbalanced with
 617                  * kcsan_disable_current() calls, which causes disable_count to
 618                  * become negative and should not happen.
 619                  */
 620                 kcsan_disable_current(); /* restore to 0, KCSAN still enabled */
 621                 kcsan_disable_current(); /* disable to generate warning */
 622                 WARN(1, "Unbalanced %s()", __func__);
 623                 kcsan_enable_current();
 624         }
 625 }
 626 EXPORT_SYMBOL(kcsan_enable_current);
 627
 628 void kcsan_enable_current_nowarn(void)
 629 {
 630         if (get_ctx()->disable_count-- == 0)
 631                 kcsan_disable_current();
 632 }
 633 EXPORT_SYMBOL(kcsan_enable_current_nowarn);
 634
 635 void kcsan_nestable_atomic_begin(void)
 636 {
 637         /*
 638          * Do *not* check and warn if we are in a flat atomic region: nestable
 639          * and flat atomic regions are independent from each other.
 640          * See include/linux/kcsan.h: struct kcsan_ctx comments for more
 641          * comments.
 642          */
 643
 644         ++get_ctx()->atomic_nest_count;
 645 }
 646 EXPORT_SYMBOL(kcsan_nestable_atomic_begin);
 647
 648 void kcsan_nestable_atomic_end(void)
 649 {
 650         if (get_ctx()->atomic_nest_count-- == 0) {
 651                 /*
 652                  * Warn if kcsan_nestable_atomic_end() calls are unbalanced with
 653                  * kcsan_nestable_atomic_begin() calls, which causes
 654                  * atomic_nest_count to become negative and should not happen.
 655                  */
 656                 kcsan_nestable_atomic_begin(); /* restore to 0 */
 657                 kcsan_disable_current(); /* disable to generate warning */
 658                 WARN(1, "Unbalanced %s()", __func__);
 659                 kcsan_enable_current();
 660         }
 661 }
 662 EXPORT_SYMBOL(kcsan_nestable_atomic_end);
 663
 664 void kcsan_flat_atomic_begin(void)
 665 {
 666         get_ctx()->in_flat_atomic = true;
 667 }
 668 EXPORT_SYMBOL(kcsan_flat_atomic_begin);
 669
 670 void kcsan_flat_atomic_end(void)
 671 {
 672         get_ctx()->in_flat_atomic = false;
 673 }
 674 EXPORT_SYMBOL(kcsan_flat_atomic_end);
 675
 676 void kcsan_atomic_next(int n)
 677 {
 678         get_ctx()->atomic_next = n;
 679 }
 680 EXPORT_SYMBOL(kcsan_atomic_next);
 681
 682 void kcsan_set_access_mask(unsigned long mask)
 683 {
 684         get_ctx()->access_mask = mask;
 685 }
 686 EXPORT_SYMBOL(kcsan_set_access_mask);
 687
 688 struct kcsan_scoped_access *
 689 kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
 690                           struct kcsan_scoped_access *sa)
 691 {
 692         struct kcsan_ctx *ctx = get_ctx();
 693
 694         __kcsan_check_access(ptr, size, type);
 695
 696         ctx->disable_count++; /* Disable KCSAN, in case list debugging is on. */
 697
 698         INIT_LIST_HEAD(&sa->list);
 699         sa->ptr = ptr;
 700         sa->size = size;
 701         sa->type = type;
 702
 703         if (!ctx->scoped_accesses.prev) /* Lazy initialize list head. */
 704                 INIT_LIST_HEAD(&ctx->scoped_accesses);
 705         list_add(&sa->list, &ctx->scoped_accesses);
 706
 707         ctx->disable_count--;
 708         return sa;
 709 }
 710 EXPORT_SYMBOL(kcsan_begin_scoped_access);
 711
 712 void kcsan_end_scoped_access(struct kcsan_scoped_access *sa)
 713 {
 714         struct kcsan_ctx *ctx = get_ctx();
 715
 716         if (WARN(!ctx->scoped_accesses.prev, "Unbalanced %s()?", __func__))
 717                 return;
 718
 719         ctx->disable_count++; /* Disable KCSAN, in case list debugging is on. */
 720
 721         list_del(&sa->list);
 722         if (list_empty(&ctx->scoped_accesses))
 723                 /*
 724                  * Ensure we do not enter kcsan_check_scoped_accesses()
 725                  * slow-path if unnecessary, and avoids requiring list_empty()
 726                  * in the fast-path (to avoid a READ_ONCE() and potential
 727                  * uaccess warning).
 728                  */
 729                 ctx->scoped_accesses.prev = NULL;
 730
 731         ctx->disable_count--;
 732
 733         __kcsan_check_access(sa->ptr, sa->size, sa->type);
 734 }
 735 EXPORT_SYMBOL(kcsan_end_scoped_access);
 736
 737 void __kcsan_check_access(const volatile void *ptr, size_t size, int type)
 738 {
 739         check_access(ptr, size, type);
 740 }
 741 EXPORT_SYMBOL(__kcsan_check_access);
 742
 743 /*
 744  * KCSAN uses the same instrumentation that is emitted by supported compilers
 745  * for ThreadSanitizer (TSAN).
 746  *
 747  * When enabled, the compiler emits instrumentation calls (the functions
 748  * prefixed with "__tsan" below) for all loads and stores that it generated;
 749  * inline asm is not instrumented.
 750  *
 751  * Note that, not all supported compiler versions distinguish aligned/unaligned
 752  * accesses, but e.g. recent versions of Clang do. We simply alias the unaligned
 753  * version to the generic version, which can handle both.
 754  */
 755
 756 #define DEFINE_TSAN_READ_WRITE(size)                                           \
 757         void __tsan_read##size(void *ptr)                                      \
 758         {                                                                      \
 759                 check_access(ptr, size, 0);                                    \
 760         }                                                                      \
 761         EXPORT_SYMBOL(__tsan_read##size);                                      \
 762         void __tsan_unaligned_read##size(void *ptr)                            \
 763                 __alias(__tsan_read##size);                                    \
 764         EXPORT_SYMBOL(__tsan_unaligned_read##size);                            \
 765         void __tsan_write##size(void *ptr)                                     \
 766         {                                                                      \
 767                 check_access(ptr, size, KCSAN_ACCESS_WRITE);                   \
 768         }                                                                      \
 769         EXPORT_SYMBOL(__tsan_write##size);                                     \
 770         void __tsan_unaligned_write##size(void *ptr)                           \
 771                 __alias(__tsan_write##size);                                   \
 772         EXPORT_SYMBOL(__tsan_unaligned_write##size)
 773
 774 DEFINE_TSAN_READ_WRITE(1);
 775 DEFINE_TSAN_READ_WRITE(2);
 776 DEFINE_TSAN_READ_WRITE(4);
 777 DEFINE_TSAN_READ_WRITE(8);
 778 DEFINE_TSAN_READ_WRITE(16);
 779
 780 void __tsan_read_range(void *ptr, size_t size)
 781 {
 782         check_access(ptr, size, 0);
 783 }
 784 EXPORT_SYMBOL(__tsan_read_range);
 785
 786 void __tsan_write_range(void *ptr, size_t size)
 787 {
 788         check_access(ptr, size, KCSAN_ACCESS_WRITE);
 789 }
 790 EXPORT_SYMBOL(__tsan_write_range);
 791
 792 /*
 793  * The below are not required by KCSAN, but can still be emitted by the
 794  * compiler.
 795  */
 796 void __tsan_func_entry(void *call_pc)
 797 {
 798 }
 799 EXPORT_SYMBOL(__tsan_func_entry);
 800 void __tsan_func_exit(void)
 801 {
 802 }
 803 EXPORT_SYMBOL(__tsan_func_exit);
 804 void __tsan_init(void)
 805 {
 806 }
 807 EXPORT_SYMBOL(__tsan_init);