kernel/kcsan/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/atomic.h>
   4 #include <linux/bug.h>
   5 #include <linux/delay.h>
   6 #include <linux/export.h>
   7 #include <linux/init.h>
   8 #include <linux/kernel.h>
   9 #include <linux/list.h>
  10 #include <linux/moduleparam.h>
  11 #include <linux/percpu.h>
  12 #include <linux/preempt.h>
  13 #include <linux/random.h>
  14 #include <linux/sched.h>
  15 #include <linux/uaccess.h>
  16
  17 #include "atomic.h"
  18 #include "encoding.h"
  19 #include "kcsan.h"
  20
  21 static bool kcsan_early_enable = IS_ENABLED(CONFIG_KCSAN_EARLY_ENABLE);
  22 unsigned int kcsan_udelay_task = CONFIG_KCSAN_UDELAY_TASK;
  23 unsigned int kcsan_udelay_interrupt = CONFIG_KCSAN_UDELAY_INTERRUPT;
  24 static long kcsan_skip_watch = CONFIG_KCSAN_SKIP_WATCH;
  25 static bool kcsan_interrupt_watcher = IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER);
  26
  27 #ifdef MODULE_PARAM_PREFIX
  28 #undef MODULE_PARAM_PREFIX
  29 #endif
  30 #define MODULE_PARAM_PREFIX "kcsan."
  31 module_param_named(early_enable, kcsan_early_enable, bool, 0);
  32 module_param_named(udelay_task, kcsan_udelay_task, uint, 0644);
  33 module_param_named(udelay_interrupt, kcsan_udelay_interrupt, uint, 0644);
  34 module_param_named(skip_watch, kcsan_skip_watch, long, 0644);
  35 module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444);
  36
  37 bool kcsan_enabled;
  38
  39 /* Per-CPU kcsan_ctx for interrupts */
  40 static DEFINE_PER_CPU(struct kcsan_ctx, kcsan_cpu_ctx) = {
  41         .disable_count          = 0,
  42         .atomic_next            = 0,
  43         .atomic_nest_count      = 0,
  44         .in_flat_atomic         = false,
  45         .access_mask            = 0,
  46         .scoped_accesses        = {LIST_POISON1, NULL},
  47 };
  48
  49 /*
  50  * Helper macros to index into adjacent slots, starting from address slot
  51  * itself, followed by the right and left slots.
  52  *
  53  * The purpose is 2-fold:
  54  *
  55  *      1. if during insertion the address slot is already occupied, check if
  56  *         any adjacent slots are free;
  57  *      2. accesses that straddle a slot boundary due to size that exceeds a
  58  *         slot's range may check adjacent slots if any watchpoint matches.
  59  *
  60  * Note that accesses with very large size may still miss a watchpoint; however,
  61  * given this should be rare, this is a reasonable trade-off to make, since this
  62  * will avoid:
  63  *
  64  *      1. excessive contention between watchpoint checks and setup;
  65  *      2. larger number of simultaneous watchpoints without sacrificing
  66  *         performance.
  67  *
  68  * Example: SLOT_IDX values for KCSAN_CHECK_ADJACENT=1, where i is [0, 1, 2]:
  69  *
  70  *   slot=0:  [ 1,  2,  0]
  71  *   slot=9:  [10, 11,  9]
  72  *   slot=63: [64, 65, 63]
  73  */
  74 #define SLOT_IDX(slot, i) (slot + ((i + KCSAN_CHECK_ADJACENT) % NUM_SLOTS))
  75
  76 /*
  77  * SLOT_IDX_FAST is used in the fast-path. Not first checking the address's primary
  78  * slot (middle) is fine if we assume that races occur rarely. The set of
  79  * indices {SLOT_IDX(slot, i) | i in [0, NUM_SLOTS)} is equivalent to
  80  * {SLOT_IDX_FAST(slot, i) | i in [0, NUM_SLOTS)}.
  81  */
  82 #define SLOT_IDX_FAST(slot, i) (slot + i)
  83
  84 /*
  85  * Watchpoints, with each entry encoded as defined in encoding.h: in order to be
  86  * able to safely update and access a watchpoint without introducing locking
  87  * overhead, we encode each watchpoint as a single atomic long. The initial
  88  * zero-initialized state matches INVALID_WATCHPOINT.
  89  *
  90  * Add NUM_SLOTS-1 entries to account for overflow; this helps avoid having to
  91  * use more complicated SLOT_IDX_FAST calculation with modulo in the fast-path.
  92  */
  93 static atomic_long_t watchpoints[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS-1];
  94
  95 /*
  96  * Instructions to skip watching counter, used in should_watch(). We use a
  97  * per-CPU counter to avoid excessive contention.
  98  */
  99 static DEFINE_PER_CPU(long, kcsan_skip);
 100
 101 static __always_inline atomic_long_t *find_watchpoint(unsigned long addr,
 102                                                       size_t size,
 103                                                       bool expect_write,
 104                                                       long *encoded_watchpoint)
 105 {
 106         const int slot = watchpoint_slot(addr);
 107         const unsigned long addr_masked = addr & WATCHPOINT_ADDR_MASK;
 108         atomic_long_t *watchpoint;
 109         unsigned long wp_addr_masked;
 110         size_t wp_size;
 111         bool is_write;
 112         int i;
 113
 114         BUILD_BUG_ON(CONFIG_KCSAN_NUM_WATCHPOINTS < NUM_SLOTS);
 115
 116         for (i = 0; i < NUM_SLOTS; ++i) {
 117                 watchpoint = &watchpoints[SLOT_IDX_FAST(slot, i)];
 118                 *encoded_watchpoint = atomic_long_read(watchpoint);
 119                 if (!decode_watchpoint(*encoded_watchpoint, &wp_addr_masked,
 120                                        &wp_size, &is_write))
 121                         continue;
 122
 123                 if (expect_write && !is_write)
 124                         continue;
 125
 126                 /* Check if the watchpoint matches the access. */
 127                 if (matching_access(wp_addr_masked, wp_size, addr_masked, size))
 128                         return watchpoint;
 129         }
 130
 131         return NULL;
 132 }
 133
 134 static inline atomic_long_t *
 135 insert_watchpoint(unsigned long addr, size_t size, bool is_write)
 136 {
 137         const int slot = watchpoint_slot(addr);
 138         const long encoded_watchpoint = encode_watchpoint(addr, size, is_write);
 139         atomic_long_t *watchpoint;
 140         int i;
 141
 142         /* Check slot index logic, ensuring we stay within array bounds. */
 143         BUILD_BUG_ON(SLOT_IDX(0, 0) != KCSAN_CHECK_ADJACENT);
 144         BUILD_BUG_ON(SLOT_IDX(0, KCSAN_CHECK_ADJACENT+1) != 0);
 145         BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS-1, KCSAN_CHECK_ADJACENT) != ARRAY_SIZE(watchpoints)-1);
 146         BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS-1, KCSAN_CHECK_ADJACENT+1) != ARRAY_SIZE(watchpoints) - NUM_SLOTS);
 147
 148         for (i = 0; i < NUM_SLOTS; ++i) {
 149                 long expect_val = INVALID_WATCHPOINT;
 150
 151                 /* Try to acquire this slot. */
 152                 watchpoint = &watchpoints[SLOT_IDX(slot, i)];
 153                 if (atomic_long_try_cmpxchg_relaxed(watchpoint, &expect_val, encoded_watchpoint))
 154                         return watchpoint;
 155         }
 156
 157         return NULL;
 158 }
 159
 160 /*
 161  * Return true if watchpoint was successfully consumed, false otherwise.
 162  *
 163  * This may return false if:
 164  *
 165  *      1. another thread already consumed the watchpoint;
 166  *      2. the thread that set up the watchpoint already removed it;
 167  *      3. the watchpoint was removed and then re-used.
 168  */
 169 static __always_inline bool
 170 try_consume_watchpoint(atomic_long_t *watchpoint, long encoded_watchpoint)
 171 {
 172         return atomic_long_try_cmpxchg_relaxed(watchpoint, &encoded_watchpoint, CONSUMED_WATCHPOINT);
 173 }
 174
 175 /* Return true if watchpoint was not touched, false if already consumed. */
 176 static inline bool consume_watchpoint(atomic_long_t *watchpoint)
 177 {
 178         return atomic_long_xchg_relaxed(watchpoint, CONSUMED_WATCHPOINT) != CONSUMED_WATCHPOINT;
 179 }
 180
 181 /* Remove the watchpoint -- its slot may be reused after. */
 182 static inline void remove_watchpoint(atomic_long_t *watchpoint)
 183 {
 184         atomic_long_set(watchpoint, INVALID_WATCHPOINT);
 185 }
 186
 187 static __always_inline struct kcsan_ctx *get_ctx(void)
 188 {
 189         /*
 190          * In interrupts, use raw_cpu_ptr to avoid unnecessary checks, that would
 191          * also result in calls that generate warnings in uaccess regions.
 192          */
 193         return in_task() ? &current->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);
 194 }
 195
 196 /* Check scoped accesses; never inline because this is a slow-path! */
 197 static noinline void kcsan_check_scoped_accesses(void)
 198 {
 199         struct kcsan_ctx *ctx = get_ctx();
 200         struct list_head *prev_save = ctx->scoped_accesses.prev;
 201         struct kcsan_scoped_access *scoped_access;
 202
 203         ctx->scoped_accesses.prev = NULL;  /* Avoid recursion. */
 204         list_for_each_entry(scoped_access, &ctx->scoped_accesses, list)
 205                 __kcsan_check_access(scoped_access->ptr, scoped_access->size, scoped_access->type);
 206         ctx->scoped_accesses.prev = prev_save;
 207 }
 208
 209 /* Rules for generic atomic accesses. Called from fast-path. */
 210 static __always_inline bool
 211 is_atomic(const volatile void *ptr, size_t size, int type, struct kcsan_ctx *ctx)
 212 {
 213         if (type & KCSAN_ACCESS_ATOMIC)
 214                 return true;
 215
 216         /*
 217          * Unless explicitly declared atomic, never consider an assertion access
 218          * as atomic. This allows using them also in atomic regions, such as
 219          * seqlocks, without implicitly changing their semantics.
 220          */
 221         if (type & KCSAN_ACCESS_ASSERT)
 222                 return false;
 223
 224         if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) &&
 225             (type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) &&
 226             IS_ALIGNED((unsigned long)ptr, size))
 227                 return true; /* Assume aligned writes up to word size are atomic. */
 228
 229         if (ctx->atomic_next > 0) {
 230                 /*
 231                  * Because we do not have separate contexts for nested
 232                  * interrupts, in case atomic_next is set, we simply assume that
 233                  * the outer interrupt set atomic_next. In the worst case, we
 234                  * will conservatively consider operations as atomic. This is a
 235                  * reasonable trade-off to make, since this case should be
 236                  * extremely rare; however, even if extremely rare, it could
 237                  * lead to false positives otherwise.
 238                  */
 239                 if ((hardirq_count() >> HARDIRQ_SHIFT) < 2)
 240                         --ctx->atomic_next; /* in task, or outer interrupt */
 241                 return true;
 242         }
 243
 244         return ctx->atomic_nest_count > 0 || ctx->in_flat_atomic;
 245 }
 246
 247 static __always_inline bool
 248 should_watch(const volatile void *ptr, size_t size, int type, struct kcsan_ctx *ctx)
 249 {
 250         /*
 251          * Never set up watchpoints when memory operations are atomic.
 252          *
 253          * Need to check this first, before kcsan_skip check below: (1) atomics
 254          * should not count towards skipped instructions, and (2) to actually
 255          * decrement kcsan_atomic_next for consecutive instruction stream.
 256          */
 257         if (is_atomic(ptr, size, type, ctx))
 258                 return false;
 259
 260         if (this_cpu_dec_return(kcsan_skip) >= 0)
 261                 return false;
 262
 263         /*
 264          * NOTE: If we get here, kcsan_skip must always be reset in slow path
 265          * via reset_kcsan_skip() to avoid underflow.
 266          */
 267
 268         /* this operation should be watched */
 269         return true;
 270 }
 271
 272 static inline void reset_kcsan_skip(void)
 273 {
 274         long skip_count = kcsan_skip_watch -
 275                           (IS_ENABLED(CONFIG_KCSAN_SKIP_WATCH_RANDOMIZE) ?
 276                                    prandom_u32_max(kcsan_skip_watch) :
 277                                    0);
 278         this_cpu_write(kcsan_skip, skip_count);
 279 }
 280
 281 static __always_inline bool kcsan_is_enabled(void)
 282 {
 283         return READ_ONCE(kcsan_enabled) && get_ctx()->disable_count == 0;
 284 }
 285
 286 static inline unsigned int get_delay(void)
 287 {
 288         unsigned int delay = in_task() ? kcsan_udelay_task : kcsan_udelay_interrupt;
 289         return delay - (IS_ENABLED(CONFIG_KCSAN_DELAY_RANDOMIZE) ?
 290                                 prandom_u32_max(delay) :
 291                                 0);
 292 }
 293
 294 void kcsan_save_irqtrace(struct task_struct *task)
 295 {
 296 #ifdef CONFIG_TRACE_IRQFLAGS
 297         task->kcsan_save_irqtrace = task->irqtrace;
 298 #endif
 299 }
 300
 301 void kcsan_restore_irqtrace(struct task_struct *task)
 302 {
 303 #ifdef CONFIG_TRACE_IRQFLAGS
 304         task->irqtrace = task->kcsan_save_irqtrace;
 305 #endif
 306 }
 307
 308 /*
 309  * Pull everything together: check_access() below contains the performance
 310  * critical operations; the fast-path (including check_access) functions should
 311  * all be inlinable by the instrumentation functions.
 312  *
 313  * The slow-path (kcsan_found_watchpoint, kcsan_setup_watchpoint) are
 314  * non-inlinable -- note that, we prefix these with "kcsan_" to ensure they can
 315  * be filtered from the stacktrace, as well as give them unique names for the
 316  * UACCESS whitelist of objtool. Each function uses user_access_save/restore(),
 317  * since they do not access any user memory, but instrumentation is still
 318  * emitted in UACCESS regions.
 319  */
 320
 321 static noinline void kcsan_found_watchpoint(const volatile void *ptr,
 322                                             size_t size,
 323                                             int type,
 324                                             atomic_long_t *watchpoint,
 325                                             long encoded_watchpoint)
 326 {
 327         unsigned long flags;
 328         bool consumed;
 329
 330         if (!kcsan_is_enabled())
 331                 return;
 332
 333         /*
 334          * The access_mask check relies on value-change comparison. To avoid
 335          * reporting a race where e.g. the writer set up the watchpoint, but the
 336          * reader has access_mask!=0, we have to ignore the found watchpoint.
 337          */
 338         if (get_ctx()->access_mask != 0)
 339                 return;
 340
 341         /*
 342          * Consume the watchpoint as soon as possible, to minimize the chances
 343          * of !consumed. Consuming the watchpoint must always be guarded by
 344          * kcsan_is_enabled() check, as otherwise we might erroneously
 345          * triggering reports when disabled.
 346          */
 347         consumed = try_consume_watchpoint(watchpoint, encoded_watchpoint);
 348
 349         /* keep this after try_consume_watchpoint */
 350         flags = user_access_save();
 351
 352         if (consumed) {
 353                 kcsan_save_irqtrace(current);
 354                 kcsan_report(ptr, size, type, KCSAN_VALUE_CHANGE_MAYBE,
 355                              KCSAN_REPORT_CONSUMED_WATCHPOINT,
 356                              watchpoint - watchpoints);
 357                 kcsan_restore_irqtrace(current);
 358         } else {
 359                 /*
 360                  * The other thread may not print any diagnostics, as it has
 361                  * already removed the watchpoint, or another thread consumed
 362                  * the watchpoint before this thread.
 363                  */
 364                 kcsan_counter_inc(KCSAN_COUNTER_REPORT_RACES);
 365         }
 366
 367         if ((type & KCSAN_ACCESS_ASSERT) != 0)
 368                 kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES);
 369         else
 370                 kcsan_counter_inc(KCSAN_COUNTER_DATA_RACES);
 371
 372         user_access_restore(flags);
 373 }
 374
 375 static noinline void
 376 kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type)
 377 {
 378         const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0;
 379         const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0;
 380         atomic_long_t *watchpoint;
 381         union {
 382                 u8 _1;
 383                 u16 _2;
 384                 u32 _4;
 385                 u64 _8;
 386         } expect_value;
 387         unsigned long access_mask;
 388         enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE;
 389         unsigned long ua_flags = user_access_save();
 390         unsigned long irq_flags = 0;
 391
 392         /*
 393          * Always reset kcsan_skip counter in slow-path to avoid underflow; see
 394          * should_watch().
 395          */
 396         reset_kcsan_skip();
 397
 398         if (!kcsan_is_enabled())
 399                 goto out;
 400
 401         /*
 402          * Special atomic rules: unlikely to be true, so we check them here in
 403          * the slow-path, and not in the fast-path in is_atomic(). Call after
 404          * kcsan_is_enabled(), as we may access memory that is not yet
 405          * initialized during early boot.
 406          */
 407         if (!is_assert && kcsan_is_atomic_special(ptr))
 408                 goto out;
 409
 410         if (!check_encodable((unsigned long)ptr, size)) {
 411                 kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES);
 412                 goto out;
 413         }
 414
 415         /*
 416          * Save and restore the IRQ state trace touched by KCSAN, since KCSAN's
 417          * runtime is entered for every memory access, and potentially useful
 418          * information is lost if dirtied by KCSAN.
 419          */
 420         kcsan_save_irqtrace(current);
 421         if (!kcsan_interrupt_watcher)
 422                 local_irq_save(irq_flags);
 423
 424         watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write);
 425         if (watchpoint == NULL) {
 426                 /*
 427                  * Out of capacity: the size of 'watchpoints', and the frequency
 428                  * with which should_watch() returns true should be tweaked so
 429                  * that this case happens very rarely.
 430                  */
 431                 kcsan_counter_inc(KCSAN_COUNTER_NO_CAPACITY);
 432                 goto out_unlock;
 433         }
 434
 435         kcsan_counter_inc(KCSAN_COUNTER_SETUP_WATCHPOINTS);
 436         kcsan_counter_inc(KCSAN_COUNTER_USED_WATCHPOINTS);
 437
 438         /*
 439          * Read the current value, to later check and infer a race if the data
 440          * was modified via a non-instrumented access, e.g. from a device.
 441          */
 442         expect_value._8 = 0;
 443         switch (size) {
 444         case 1:
 445                 expect_value._1 = READ_ONCE(*(const u8 *)ptr);
 446                 break;
 447         case 2:
 448                 expect_value._2 = READ_ONCE(*(const u16 *)ptr);
 449                 break;
 450         case 4:
 451                 expect_value._4 = READ_ONCE(*(const u32 *)ptr);
 452                 break;
 453         case 8:
 454                 expect_value._8 = READ_ONCE(*(const u64 *)ptr);
 455                 break;
 456         default:
 457                 break; /* ignore; we do not diff the values */
 458         }
 459
 460         if (IS_ENABLED(CONFIG_KCSAN_DEBUG)) {
 461                 kcsan_disable_current();
 462                 pr_err("KCSAN: watching %s, size: %zu, addr: %px [slot: %d, encoded: %lx]\n",
 463                        is_write ? "write" : "read", size, ptr,
 464                        watchpoint_slot((unsigned long)ptr),
 465                        encode_watchpoint((unsigned long)ptr, size, is_write));
 466                 kcsan_enable_current();
 467         }
 468
 469         /*
 470          * Delay this thread, to increase probability of observing a racy
 471          * conflicting access.
 472          */
 473         udelay(get_delay());
 474
 475         /*
 476          * Re-read value, and check if it is as expected; if not, we infer a
 477          * racy access.
 478          */
 479         access_mask = get_ctx()->access_mask;
 480         switch (size) {
 481         case 1:
 482                 expect_value._1 ^= READ_ONCE(*(const u8 *)ptr);
 483                 if (access_mask)
 484                         expect_value._1 &= (u8)access_mask;
 485                 break;
 486         case 2:
 487                 expect_value._2 ^= READ_ONCE(*(const u16 *)ptr);
 488                 if (access_mask)
 489                         expect_value._2 &= (u16)access_mask;
 490                 break;
 491         case 4:
 492                 expect_value._4 ^= READ_ONCE(*(const u32 *)ptr);
 493                 if (access_mask)
 494                         expect_value._4 &= (u32)access_mask;
 495                 break;
 496         case 8:
 497                 expect_value._8 ^= READ_ONCE(*(const u64 *)ptr);
 498                 if (access_mask)
 499                         expect_value._8 &= (u64)access_mask;
 500                 break;
 501         default:
 502                 break; /* ignore; we do not diff the values */
 503         }
 504
 505         /* Were we able to observe a value-change? */
 506         if (expect_value._8 != 0)
 507                 value_change = KCSAN_VALUE_CHANGE_TRUE;
 508
 509         /* Check if this access raced with another. */
 510         if (!consume_watchpoint(watchpoint)) {
 511                 /*
 512                  * Depending on the access type, map a value_change of MAYBE to
 513                  * TRUE (always report) or FALSE (never report).
 514                  */
 515                 if (value_change == KCSAN_VALUE_CHANGE_MAYBE) {
 516                         if (access_mask != 0) {
 517                                 /*
 518                                  * For access with access_mask, we require a
 519                                  * value-change, as it is likely that races on
 520                                  * ~access_mask bits are expected.
 521                                  */
 522                                 value_change = KCSAN_VALUE_CHANGE_FALSE;
 523                         } else if (size > 8 || is_assert) {
 524                                 /* Always assume a value-change. */
 525                                 value_change = KCSAN_VALUE_CHANGE_TRUE;
 526                         }
 527                 }
 528
 529                 /*
 530                  * No need to increment 'data_races' counter, as the racing
 531                  * thread already did.
 532                  *
 533                  * Count 'assert_failures' for each failed ASSERT access,
 534                  * therefore both this thread and the racing thread may
 535                  * increment this counter.
 536                  */
 537                 if (is_assert && value_change == KCSAN_VALUE_CHANGE_TRUE)
 538                         kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES);
 539
 540                 kcsan_report(ptr, size, type, value_change, KCSAN_REPORT_RACE_SIGNAL,
 541                              watchpoint - watchpoints);
 542         } else if (value_change == KCSAN_VALUE_CHANGE_TRUE) {
 543                 /* Inferring a race, since the value should not have changed. */
 544
 545                 kcsan_counter_inc(KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN);
 546                 if (is_assert)
 547                         kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES);
 548
 549                 if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN) || is_assert)
 550                         kcsan_report(ptr, size, type, KCSAN_VALUE_CHANGE_TRUE,
 551                                      KCSAN_REPORT_RACE_UNKNOWN_ORIGIN,
 552                                      watchpoint - watchpoints);
 553         }
 554
 555         /*
 556          * Remove watchpoint; must be after reporting, since the slot may be
 557          * reused after this point.
 558          */
 559         remove_watchpoint(watchpoint);
 560         kcsan_counter_dec(KCSAN_COUNTER_USED_WATCHPOINTS);
 561 out_unlock:
 562         if (!kcsan_interrupt_watcher)
 563                 local_irq_restore(irq_flags);
 564         kcsan_restore_irqtrace(current);
 565 out:
 566         user_access_restore(ua_flags);
 567 }
 568
 569 static __always_inline void check_access(const volatile void *ptr, size_t size,
 570                                          int type)
 571 {
 572         const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0;
 573         atomic_long_t *watchpoint;
 574         long encoded_watchpoint;
 575
 576         /*
 577          * Do nothing for 0 sized check; this comparison will be optimized out
 578          * for constant sized instrumentation (__tsan_{read,write}N).
 579          */
 580         if (unlikely(size == 0))
 581                 return;
 582
 583         /*
 584          * Avoid user_access_save in fast-path: find_watchpoint is safe without
 585          * user_access_save, as the address that ptr points to is only used to
 586          * check if a watchpoint exists; ptr is never dereferenced.
 587          */
 588         watchpoint = find_watchpoint((unsigned long)ptr, size, !is_write,
 589                                      &encoded_watchpoint);
 590         /*
 591          * It is safe to check kcsan_is_enabled() after find_watchpoint in the
 592          * slow-path, as long as no state changes that cause a race to be
 593          * detected and reported have occurred until kcsan_is_enabled() is
 594          * checked.
 595          */
 596
 597         if (unlikely(watchpoint != NULL))
 598                 kcsan_found_watchpoint(ptr, size, type, watchpoint,
 599                                        encoded_watchpoint);
 600         else {
 601                 struct kcsan_ctx *ctx = get_ctx(); /* Call only once in fast-path. */
 602
 603                 if (unlikely(should_watch(ptr, size, type, ctx)))
 604                         kcsan_setup_watchpoint(ptr, size, type);
 605                 else if (unlikely(ctx->scoped_accesses.prev))
 606                         kcsan_check_scoped_accesses();
 607         }
 608 }
 609
 610 /* === Public interface ===================================================== */
 611
 612 void __init kcsan_init(void)
 613 {
 614         BUG_ON(!in_task());
 615
 616         kcsan_debugfs_init();
 617
 618         /*
 619          * We are in the init task, and no other tasks should be running;
 620          * WRITE_ONCE without memory barrier is sufficient.
 621          */
 622         if (kcsan_early_enable)
 623                 WRITE_ONCE(kcsan_enabled, true);
 624 }
 625
 626 /* === Exported interface =================================================== */
 627
 628 void kcsan_disable_current(void)
 629 {
 630         ++get_ctx()->disable_count;
 631 }
 632 EXPORT_SYMBOL(kcsan_disable_current);
 633
 634 void kcsan_enable_current(void)
 635 {
 636         if (get_ctx()->disable_count-- == 0) {
 637                 /*
 638                  * Warn if kcsan_enable_current() calls are unbalanced with
 639                  * kcsan_disable_current() calls, which causes disable_count to
 640                  * become negative and should not happen.
 641                  */
 642                 kcsan_disable_current(); /* restore to 0, KCSAN still enabled */
 643                 kcsan_disable_current(); /* disable to generate warning */
 644                 WARN(1, "Unbalanced %s()", __func__);
 645                 kcsan_enable_current();
 646         }
 647 }
 648 EXPORT_SYMBOL(kcsan_enable_current);
 649
 650 void kcsan_enable_current_nowarn(void)
 651 {
 652         if (get_ctx()->disable_count-- == 0)
 653                 kcsan_disable_current();
 654 }
 655 EXPORT_SYMBOL(kcsan_enable_current_nowarn);
 656
 657 void kcsan_nestable_atomic_begin(void)
 658 {
 659         /*
 660          * Do *not* check and warn if we are in a flat atomic region: nestable
 661          * and flat atomic regions are independent from each other.
 662          * See include/linux/kcsan.h: struct kcsan_ctx comments for more
 663          * comments.
 664          */
 665
 666         ++get_ctx()->atomic_nest_count;
 667 }
 668 EXPORT_SYMBOL(kcsan_nestable_atomic_begin);
 669
 670 void kcsan_nestable_atomic_end(void)
 671 {
 672         if (get_ctx()->atomic_nest_count-- == 0) {
 673                 /*
 674                  * Warn if kcsan_nestable_atomic_end() calls are unbalanced with
 675                  * kcsan_nestable_atomic_begin() calls, which causes
 676                  * atomic_nest_count to become negative and should not happen.
 677                  */
 678                 kcsan_nestable_atomic_begin(); /* restore to 0 */
 679                 kcsan_disable_current(); /* disable to generate warning */
 680                 WARN(1, "Unbalanced %s()", __func__);
 681                 kcsan_enable_current();
 682         }
 683 }
 684 EXPORT_SYMBOL(kcsan_nestable_atomic_end);
 685
 686 void kcsan_flat_atomic_begin(void)
 687 {
 688         get_ctx()->in_flat_atomic = true;
 689 }
 690 EXPORT_SYMBOL(kcsan_flat_atomic_begin);
 691
 692 void kcsan_flat_atomic_end(void)
 693 {
 694         get_ctx()->in_flat_atomic = false;
 695 }
 696 EXPORT_SYMBOL(kcsan_flat_atomic_end);
 697
 698 void kcsan_atomic_next(int n)
 699 {
 700         get_ctx()->atomic_next = n;
 701 }
 702 EXPORT_SYMBOL(kcsan_atomic_next);
 703
 704 void kcsan_set_access_mask(unsigned long mask)
 705 {
 706         get_ctx()->access_mask = mask;
 707 }
 708 EXPORT_SYMBOL(kcsan_set_access_mask);
 709
 710 struct kcsan_scoped_access *
 711 kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
 712                           struct kcsan_scoped_access *sa)
 713 {
 714         struct kcsan_ctx *ctx = get_ctx();
 715
 716         __kcsan_check_access(ptr, size, type);
 717
 718         ctx->disable_count++; /* Disable KCSAN, in case list debugging is on. */
 719
 720         INIT_LIST_HEAD(&sa->list);
 721         sa->ptr = ptr;
 722         sa->size = size;
 723         sa->type = type;
 724
 725         if (!ctx->scoped_accesses.prev) /* Lazy initialize list head. */
 726                 INIT_LIST_HEAD(&ctx->scoped_accesses);
 727         list_add(&sa->list, &ctx->scoped_accesses);
 728
 729         ctx->disable_count--;
 730         return sa;
 731 }
 732 EXPORT_SYMBOL(kcsan_begin_scoped_access);
 733
 734 void kcsan_end_scoped_access(struct kcsan_scoped_access *sa)
 735 {
 736         struct kcsan_ctx *ctx = get_ctx();
 737
 738         if (WARN(!ctx->scoped_accesses.prev, "Unbalanced %s()?", __func__))
 739                 return;
 740
 741         ctx->disable_count++; /* Disable KCSAN, in case list debugging is on. */
 742
 743         list_del(&sa->list);
 744         if (list_empty(&ctx->scoped_accesses))
 745                 /*
 746                  * Ensure we do not enter kcsan_check_scoped_accesses()
 747                  * slow-path if unnecessary, and avoids requiring list_empty()
 748                  * in the fast-path (to avoid a READ_ONCE() and potential
 749                  * uaccess warning).
 750                  */
 751                 ctx->scoped_accesses.prev = NULL;
 752
 753         ctx->disable_count--;
 754
 755         __kcsan_check_access(sa->ptr, sa->size, sa->type);
 756 }
 757 EXPORT_SYMBOL(kcsan_end_scoped_access);
 758
 759 void __kcsan_check_access(const volatile void *ptr, size_t size, int type)
 760 {
 761         check_access(ptr, size, type);
 762 }
 763 EXPORT_SYMBOL(__kcsan_check_access);
 764
 765 /*
 766  * KCSAN uses the same instrumentation that is emitted by supported compilers
 767  * for ThreadSanitizer (TSAN).
 768  *
 769  * When enabled, the compiler emits instrumentation calls (the functions
 770  * prefixed with "__tsan" below) for all loads and stores that it generated;
 771  * inline asm is not instrumented.
 772  *
 773  * Note that, not all supported compiler versions distinguish aligned/unaligned
 774  * accesses, but e.g. recent versions of Clang do. We simply alias the unaligned
 775  * version to the generic version, which can handle both.
 776  */
 777
 778 #define DEFINE_TSAN_READ_WRITE(size)                                           \
 779         void __tsan_read##size(void *ptr);                                     \
 780         void __tsan_read##size(void *ptr)                                      \
 781         {                                                                      \
 782                 check_access(ptr, size, 0);                                    \
 783         }                                                                      \
 784         EXPORT_SYMBOL(__tsan_read##size);                                      \
 785         void __tsan_unaligned_read##size(void *ptr)                            \
 786                 __alias(__tsan_read##size);                                    \
 787         EXPORT_SYMBOL(__tsan_unaligned_read##size);                            \
 788         void __tsan_write##size(void *ptr);                                    \
 789         void __tsan_write##size(void *ptr)                                     \
 790         {                                                                      \
 791                 check_access(ptr, size, KCSAN_ACCESS_WRITE);                   \
 792         }                                                                      \
 793         EXPORT_SYMBOL(__tsan_write##size);                                     \
 794         void __tsan_unaligned_write##size(void *ptr)                           \
 795                 __alias(__tsan_write##size);                                   \
 796         EXPORT_SYMBOL(__tsan_unaligned_write##size)
 797
 798 DEFINE_TSAN_READ_WRITE(1);
 799 DEFINE_TSAN_READ_WRITE(2);
 800 DEFINE_TSAN_READ_WRITE(4);
 801 DEFINE_TSAN_READ_WRITE(8);
 802 DEFINE_TSAN_READ_WRITE(16);
 803
 804 void __tsan_read_range(void *ptr, size_t size);
 805 void __tsan_read_range(void *ptr, size_t size)
 806 {
 807         check_access(ptr, size, 0);
 808 }
 809 EXPORT_SYMBOL(__tsan_read_range);
 810
 811 void __tsan_write_range(void *ptr, size_t size);
 812 void __tsan_write_range(void *ptr, size_t size)
 813 {
 814         check_access(ptr, size, KCSAN_ACCESS_WRITE);
 815 }
 816 EXPORT_SYMBOL(__tsan_write_range);
 817
 818 /*
 819  * Use of explicit volatile is generally disallowed [1], however, volatile is
 820  * still used in various concurrent context, whether in low-level
 821  * synchronization primitives or for legacy reasons.
 822  * [1] https://lwn.net/Articles/233479/
 823  *
 824  * We only consider volatile accesses atomic if they are aligned and would pass
 825  * the size-check of compiletime_assert_rwonce_type().
 826  */
 827 #define DEFINE_TSAN_VOLATILE_READ_WRITE(size)                                  \
 828         void __tsan_volatile_read##size(void *ptr);                            \
 829         void __tsan_volatile_read##size(void *ptr)                             \
 830         {                                                                      \
 831                 const bool is_atomic = size <= sizeof(long long) &&            \
 832                                        IS_ALIGNED((unsigned long)ptr, size);   \
 833                 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) && is_atomic)      \
 834                         return;                                                \
 835                 check_access(ptr, size, is_atomic ? KCSAN_ACCESS_ATOMIC : 0);  \
 836         }                                                                      \
 837         EXPORT_SYMBOL(__tsan_volatile_read##size);                             \
 838         void __tsan_unaligned_volatile_read##size(void *ptr)                   \
 839                 __alias(__tsan_volatile_read##size);                           \
 840         EXPORT_SYMBOL(__tsan_unaligned_volatile_read##size);                   \
 841         void __tsan_volatile_write##size(void *ptr);                           \
 842         void __tsan_volatile_write##size(void *ptr)                            \
 843         {                                                                      \
 844                 const bool is_atomic = size <= sizeof(long long) &&            \
 845                                        IS_ALIGNED((unsigned long)ptr, size);   \
 846                 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) && is_atomic)      \
 847                         return;                                                \
 848                 check_access(ptr, size,                                        \
 849                              KCSAN_ACCESS_WRITE |                              \
 850                                      (is_atomic ? KCSAN_ACCESS_ATOMIC : 0));   \
 851         }                                                                      \
 852         EXPORT_SYMBOL(__tsan_volatile_write##size);                            \
 853         void __tsan_unaligned_volatile_write##size(void *ptr)                  \
 854                 __alias(__tsan_volatile_write##size);                          \
 855         EXPORT_SYMBOL(__tsan_unaligned_volatile_write##size)
 856
 857 DEFINE_TSAN_VOLATILE_READ_WRITE(1);
 858 DEFINE_TSAN_VOLATILE_READ_WRITE(2);
 859 DEFINE_TSAN_VOLATILE_READ_WRITE(4);
 860 DEFINE_TSAN_VOLATILE_READ_WRITE(8);
 861 DEFINE_TSAN_VOLATILE_READ_WRITE(16);
 862
 863 /*
 864  * The below are not required by KCSAN, but can still be emitted by the
 865  * compiler.
 866  */
 867 void __tsan_func_entry(void *call_pc);
 868 void __tsan_func_entry(void *call_pc)
 869 {
 870 }
 871 EXPORT_SYMBOL(__tsan_func_entry);
 872 void __tsan_func_exit(void);
 873 void __tsan_func_exit(void)
 874 {
 875 }
 876 EXPORT_SYMBOL(__tsan_func_exit);
 877 void __tsan_init(void);
 878 void __tsan_init(void)
 879 {
 880 }
 881 EXPORT_SYMBOL(__tsan_init);