include/linux/seqlock.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __LINUX_SEQLOCK_H
   3 #define __LINUX_SEQLOCK_H
   4
   5 /*
   6  * seqcount_t / seqlock_t - a reader-writer consistency mechanism with
   7  * lockless readers (read-only retry loops), and no writer starvation.
   8  *
   9  * See Documentation/locking/seqlock.rst
  10  *
  11  * Copyrights:
  12  * - Based on x86_64 vsyscall gettimeofday: Keith Owens, Andrea Arcangeli
  13  * - Sequence counters with associated locks, (C) 2020 Linutronix GmbH
  14  */
  15
  16 #include <linux/compiler.h>
  17 #include <linux/kcsan-checks.h>
  18 #include <linux/lockdep.h>
  19 #include <linux/mutex.h>
  20 #include <linux/preempt.h>
  21 #include <linux/spinlock.h>
  22
  23 #include <asm/processor.h>
  24
  25 /*
  26  * The seqlock seqcount_t interface does not prescribe a precise sequence of
  27  * read begin/retry/end. For readers, typically there is a call to
  28  * read_seqcount_begin() and read_seqcount_retry(), however, there are more
  29  * esoteric cases which do not follow this pattern.
  30  *
  31  * As a consequence, we take the following best-effort approach for raw usage
  32  * via seqcount_t under KCSAN: upon beginning a seq-reader critical section,
  33  * pessimistically mark the next KCSAN_SEQLOCK_REGION_MAX memory accesses as
  34  * atomics; if there is a matching read_seqcount_retry() call, no following
  35  * memory operations are considered atomic. Usage of the seqlock_t interface
  36  * is not affected.
  37  */
  38 #define KCSAN_SEQLOCK_REGION_MAX 1000
  39
  40 /*
  41  * Sequence counters (seqcount_t)
  42  *
  43  * This is the raw counting mechanism, without any writer protection.
  44  *
  45  * Write side critical sections must be serialized and non-preemptible.
  46  *
  47  * If readers can be invoked from hardirq or softirq contexts,
  48  * interrupts or bottom halves must also be respectively disabled before
  49  * entering the write section.
  50  *
  51  * This mechanism can't be used if the protected data contains pointers,
  52  * as the writer can invalidate a pointer that a reader is following.
  53  *
  54  * If the write serialization mechanism is one of the common kernel
  55  * locking primitives, use a sequence counter with associated lock
  56  * (seqcount_LOCKNAME_t) instead.
  57  *
  58  * If it's desired to automatically handle the sequence counter writer
  59  * serialization and non-preemptibility requirements, use a sequential
  60  * lock (seqlock_t) instead.
  61  *
  62  * See Documentation/locking/seqlock.rst
  63  */
  64 typedef struct seqcount {
  65         unsigned sequence;
  66 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  67         struct lockdep_map dep_map;
  68 #endif
  69 } seqcount_t;
  70
  71 static inline void __seqcount_init(seqcount_t *s, const char *name,
  72                                           struct lock_class_key *key)
  73 {
  74         /*
  75          * Make sure we are not reinitializing a held lock:
  76          */
  77         lockdep_init_map(&s->dep_map, name, key, 0);
  78         s->sequence = 0;
  79 }
  80
  81 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  82
  83 # define SEQCOUNT_DEP_MAP_INIT(lockname)                                \
  84                 .dep_map = { .name = #lockname }
  85
  86 /**
  87  * seqcount_init() - runtime initializer for seqcount_t
  88  * @s: Pointer to the seqcount_t instance
  89  */
  90 # define seqcount_init(s)                                               \
  91         do {                                                            \
  92                 static struct lock_class_key __key;                     \
  93                 __seqcount_init((s), #s, &__key);                       \
  94         } while (0)
  95
  96 static inline void seqcount_lockdep_reader_access(const seqcount_t *s)
  97 {
  98         seqcount_t *l = (seqcount_t *)s;
  99         unsigned long flags;
 100
 101         local_irq_save(flags);
 102         seqcount_acquire_read(&l->dep_map, 0, 0, _RET_IP_);
 103         seqcount_release(&l->dep_map, _RET_IP_);
 104         local_irq_restore(flags);
 105 }
 106
 107 #else
 108 # define SEQCOUNT_DEP_MAP_INIT(lockname)
 109 # define seqcount_init(s) __seqcount_init(s, NULL, NULL)
 110 # define seqcount_lockdep_reader_access(x)
 111 #endif
 112
 113 /**
 114  * SEQCNT_ZERO() - static initializer for seqcount_t
 115  * @name: Name of the seqcount_t instance
 116  */
 117 #define SEQCNT_ZERO(name) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(name) }
 118
 119 /*
 120  * Sequence counters with associated locks (seqcount_LOCKNAME_t)
 121  *
 122  * A sequence counter which associates the lock used for writer
 123  * serialization at initialization time. This enables lockdep to validate
 124  * that the write side critical section is properly serialized.
 125  *
 126  * For associated locks which do not implicitly disable preemption,
 127  * preemption protection is enforced in the write side function.
 128  *
 129  * Lockdep is never used in any for the raw write variants.
 130  *
 131  * See Documentation/locking/seqlock.rst
 132  */
 133
 134 #ifdef CONFIG_LOCKDEP
 135 #define __SEQ_LOCK(expr)        expr
 136 #else
 137 #define __SEQ_LOCK(expr)
 138 #endif
 139
 140 /**
 141  * typedef seqcount_LOCKNAME_t - sequence counter with LOCKNAME associated
 142  * @seqcount:   The real sequence counter
 143  * @lock:       Pointer to the associated lock
 144  *
 145  * A plain sequence counter with external writer synchronization by
 146  * LOCKNAME @lock. The lock is associated to the sequence counter in the
 147  * static initializer or init function. This enables lockdep to validate
 148  * that the write side critical section is properly serialized.
 149  *
 150  * LOCKNAME:    raw_spinlock, spinlock, rwlock, mutex, or ww_mutex.
 151  */
 152
 153 /*
 154  * seqcount_LOCKNAME_init() - runtime initializer for seqcount_LOCKNAME_t
 155  * @s:          Pointer to the seqcount_LOCKNAME_t instance
 156  * @lock:       Pointer to the associated lock
 157  */
 158
 159 /*
 160  * SEQCOUNT_LOCKNAME()  - Instantiate seqcount_LOCKNAME_t and helpers
 161  * seqprop_LOCKNAME_*() - Property accessors for seqcount_LOCKNAME_t
 162  *
 163  * @lockname:           "LOCKNAME" part of seqcount_LOCKNAME_t
 164  * @locktype:           LOCKNAME canonical C data type
 165  * @preemptible:        preemptibility of above lockname
 166  * @lockmember:         argument for lockdep_assert_held()
 167  */
 168 #define SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockmember)  \
 169 typedef struct seqcount_##lockname {                                    \
 170         seqcount_t              seqcount;                               \
 171         __SEQ_LOCK(locktype     *lock);                                 \
 172 } seqcount_##lockname##_t;                                              \
 173                                                                         \
 174 static __always_inline void                                             \
 175 seqcount_##lockname##_init(seqcount_##lockname##_t *s, locktype *lock)  \
 176 {                                                                       \
 177         seqcount_init(&s->seqcount);                                    \
 178         __SEQ_LOCK(s->lock = lock);                                     \
 179 }                                                                       \
 180                                                                         \
 181 static __always_inline seqcount_t *                                     \
 182 __seqprop_##lockname##_ptr(seqcount_##lockname##_t *s)                  \
 183 {                                                                       \
 184         return &s->seqcount;                                            \
 185 }                                                                       \
 186                                                                         \
 187 static __always_inline unsigned                                         \
 188 __seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s)       \
 189 {                                                                       \
 190         return READ_ONCE(s->seqcount.sequence);                         \
 191 }                                                                       \
 192                                                                         \
 193 static __always_inline bool                                             \
 194 __seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s)    \
 195 {                                                                       \
 196         return preemptible;                                             \
 197 }                                                                       \
 198                                                                         \
 199 static __always_inline void                                             \
 200 __seqprop_##lockname##_assert(const seqcount_##lockname##_t *s)         \
 201 {                                                                       \
 202         __SEQ_LOCK(lockdep_assert_held(lockmember));                    \
 203 }
 204
 205 /*
 206  * __seqprop() for seqcount_t
 207  */
 208
 209 static inline seqcount_t *__seqprop_ptr(seqcount_t *s)
 210 {
 211         return s;
 212 }
 213
 214 static inline unsigned __seqprop_sequence(const seqcount_t *s)
 215 {
 216         return READ_ONCE(s->sequence);
 217 }
 218
 219 static inline bool __seqprop_preemptible(const seqcount_t *s)
 220 {
 221         return false;
 222 }
 223
 224 static inline void __seqprop_assert(const seqcount_t *s)
 225 {
 226         lockdep_assert_preemption_disabled();
 227 }
 228
 229 SEQCOUNT_LOCKNAME(raw_spinlock, raw_spinlock_t,         false,  s->lock)
 230 SEQCOUNT_LOCKNAME(spinlock,     spinlock_t,             false,  s->lock)
 231 SEQCOUNT_LOCKNAME(rwlock,       rwlock_t,               false,  s->lock)
 232 SEQCOUNT_LOCKNAME(mutex,        struct mutex,           true,   s->lock)
 233 SEQCOUNT_LOCKNAME(ww_mutex,     struct ww_mutex,        true,   &s->lock->base)
 234
 235 /*
 236  * SEQCNT_LOCKNAME_ZERO - static initializer for seqcount_LOCKNAME_t
 237  * @name:       Name of the seqcount_LOCKNAME_t instance
 238  * @lock:       Pointer to the associated LOCKNAME
 239  */
 240
 241 #define SEQCOUNT_LOCKNAME_ZERO(seq_name, assoc_lock) {                  \
 242         .seqcount               = SEQCNT_ZERO(seq_name.seqcount),       \
 243         __SEQ_LOCK(.lock        = (assoc_lock))                         \
 244 }
 245
 246 #define SEQCNT_SPINLOCK_ZERO(name, lock)        SEQCOUNT_LOCKNAME_ZERO(name, lock)
 247 #define SEQCNT_RAW_SPINLOCK_ZERO(name, lock)    SEQCOUNT_LOCKNAME_ZERO(name, lock)
 248 #define SEQCNT_RWLOCK_ZERO(name, lock)          SEQCOUNT_LOCKNAME_ZERO(name, lock)
 249 #define SEQCNT_MUTEX_ZERO(name, lock)           SEQCOUNT_LOCKNAME_ZERO(name, lock)
 250 #define SEQCNT_WW_MUTEX_ZERO(name, lock)        SEQCOUNT_LOCKNAME_ZERO(name, lock)
 251
 252 #define __seqprop_case(s, lockname, prop)                               \
 253         seqcount_##lockname##_t: __seqprop_##lockname##_##prop((void *)(s))
 254
 255 #define __seqprop(s, prop) _Generic(*(s),                               \
 256         seqcount_t:             __seqprop_##prop((void *)(s)),          \
 257         __seqprop_case((s),     raw_spinlock,   prop),                  \
 258         __seqprop_case((s),     spinlock,       prop),                  \
 259         __seqprop_case((s),     rwlock,         prop),                  \
 260         __seqprop_case((s),     mutex,          prop),                  \
 261         __seqprop_case((s),     ww_mutex,       prop))
 262
 263 #define __seqcount_ptr(s)               __seqprop(s, ptr)
 264 #define __seqcount_sequence(s)          __seqprop(s, sequence)
 265 #define __seqcount_lock_preemptible(s)  __seqprop(s, preemptible)
 266 #define __seqcount_assert_lock_held(s)  __seqprop(s, assert)
 267
 268 /**
 269  * __read_seqcount_begin() - begin a seqcount_t read section w/o barrier
 270  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 271  *
 272  * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb()
 273  * barrier. Callers should ensure that smp_rmb() or equivalent ordering is
 274  * provided before actually loading any of the variables that are to be
 275  * protected in this critical section.
 276  *
 277  * Use carefully, only in critical code, and comment how the barrier is
 278  * provided.
 279  *
 280  * Return: count to be passed to read_seqcount_retry()
 281  */
 282 #define __read_seqcount_begin(s)                                        \
 283 ({                                                                      \
 284         unsigned seq;                                                   \
 285                                                                         \
 286         while ((seq = __seqcount_sequence(s)) & 1)                      \
 287                 cpu_relax();                                            \
 288                                                                         \
 289         kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX);                    \
 290         seq;                                                            \
 291 })
 292
 293 /**
 294  * raw_read_seqcount_begin() - begin a seqcount_t read section w/o lockdep
 295  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 296  *
 297  * Return: count to be passed to read_seqcount_retry()
 298  */
 299 #define raw_read_seqcount_begin(s)                                      \
 300 ({                                                                      \
 301         unsigned seq = __read_seqcount_begin(s);                        \
 302                                                                         \
 303         smp_rmb();                                                      \
 304         seq;                                                            \
 305 })
 306
 307 /**
 308  * read_seqcount_begin() - begin a seqcount_t read critical section
 309  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 310  *
 311  * Return: count to be passed to read_seqcount_retry()
 312  */
 313 #define read_seqcount_begin(s)                                          \
 314 ({                                                                      \
 315         seqcount_lockdep_reader_access(__seqcount_ptr(s));              \
 316         raw_read_seqcount_begin(s);                                     \
 317 })
 318
 319 /**
 320  * raw_read_seqcount() - read the raw seqcount_t counter value
 321  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 322  *
 323  * raw_read_seqcount opens a read critical section of the given
 324  * seqcount_t, without any lockdep checking, and without checking or
 325  * masking the sequence counter LSB. Calling code is responsible for
 326  * handling that.
 327  *
 328  * Return: count to be passed to read_seqcount_retry()
 329  */
 330 #define raw_read_seqcount(s)                                            \
 331 ({                                                                      \
 332         unsigned seq = __seqcount_sequence(s);                          \
 333                                                                         \
 334         smp_rmb();                                                      \
 335         kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX);                    \
 336         seq;                                                            \
 337 })
 338
 339 /**
 340  * raw_seqcount_begin() - begin a seqcount_t read critical section w/o
 341  *                        lockdep and w/o counter stabilization
 342  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 343  *
 344  * raw_seqcount_begin opens a read critical section of the given
 345  * seqcount_t. Unlike read_seqcount_begin(), this function will not wait
 346  * for the count to stabilize. If a writer is active when it begins, it
 347  * will fail the read_seqcount_retry() at the end of the read critical
 348  * section instead of stabilizing at the beginning of it.
 349  *
 350  * Use this only in special kernel hot paths where the read section is
 351  * small and has a high probability of success through other external
 352  * means. It will save a single branching instruction.
 353  *
 354  * Return: count to be passed to read_seqcount_retry()
 355  */
 356 #define raw_seqcount_begin(s)                                           \
 357 ({                                                                      \
 358         /*                                                              \
 359          * If the counter is odd, let read_seqcount_retry() fail        \
 360          * by decrementing the counter.                                 \
 361          */                                                             \
 362         raw_read_seqcount(s) & ~1;                                      \
 363 })
 364
 365 /**
 366  * __read_seqcount_retry() - end a seqcount_t read section w/o barrier
 367  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 368  * @start: count, from read_seqcount_begin()
 369  *
 370  * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb()
 371  * barrier. Callers should ensure that smp_rmb() or equivalent ordering is
 372  * provided before actually loading any of the variables that are to be
 373  * protected in this critical section.
 374  *
 375  * Use carefully, only in critical code, and comment how the barrier is
 376  * provided.
 377  *
 378  * Return: true if a read section retry is required, else false
 379  */
 380 #define __read_seqcount_retry(s, start)                                 \
 381         __read_seqcount_t_retry(__seqcount_ptr(s), start)
 382
 383 static inline int __read_seqcount_t_retry(const seqcount_t *s, unsigned start)
 384 {
 385         kcsan_atomic_next(0);
 386         return unlikely(READ_ONCE(s->sequence) != start);
 387 }
 388
 389 /**
 390  * read_seqcount_retry() - end a seqcount_t read critical section
 391  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 392  * @start: count, from read_seqcount_begin()
 393  *
 394  * read_seqcount_retry closes the read critical section of given
 395  * seqcount_t.  If the critical section was invalid, it must be ignored
 396  * (and typically retried).
 397  *
 398  * Return: true if a read section retry is required, else false
 399  */
 400 #define read_seqcount_retry(s, start)                                   \
 401         read_seqcount_t_retry(__seqcount_ptr(s), start)
 402
 403 static inline int read_seqcount_t_retry(const seqcount_t *s, unsigned start)
 404 {
 405         smp_rmb();
 406         return __read_seqcount_t_retry(s, start);
 407 }
 408
 409 /**
 410  * raw_write_seqcount_begin() - start a seqcount_t write section w/o lockdep
 411  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 412  */
 413 #define raw_write_seqcount_begin(s)                                     \
 414 do {                                                                    \
 415         if (__seqcount_lock_preemptible(s))                             \
 416                 preempt_disable();                                      \
 417                                                                         \
 418         raw_write_seqcount_t_begin(__seqcount_ptr(s));                  \
 419 } while (0)
 420
 421 static inline void raw_write_seqcount_t_begin(seqcount_t *s)
 422 {
 423         kcsan_nestable_atomic_begin();
 424         s->sequence++;
 425         smp_wmb();
 426 }
 427
 428 /**
 429  * raw_write_seqcount_end() - end a seqcount_t write section w/o lockdep
 430  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 431  */
 432 #define raw_write_seqcount_end(s)                                       \
 433 do {                                                                    \
 434         raw_write_seqcount_t_end(__seqcount_ptr(s));                    \
 435                                                                         \
 436         if (__seqcount_lock_preemptible(s))                             \
 437                 preempt_enable();                                       \
 438 } while (0)
 439
 440 static inline void raw_write_seqcount_t_end(seqcount_t *s)
 441 {
 442         smp_wmb();
 443         s->sequence++;
 444         kcsan_nestable_atomic_end();
 445 }
 446
 447 /**
 448  * write_seqcount_begin_nested() - start a seqcount_t write section with
 449  *                                 custom lockdep nesting level
 450  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 451  * @subclass: lockdep nesting level
 452  *
 453  * See Documentation/locking/lockdep-design.rst
 454  */
 455 #define write_seqcount_begin_nested(s, subclass)                        \
 456 do {                                                                    \
 457         __seqcount_assert_lock_held(s);                                 \
 458                                                                         \
 459         if (__seqcount_lock_preemptible(s))                             \
 460                 preempt_disable();                                      \
 461                                                                         \
 462         write_seqcount_t_begin_nested(__seqcount_ptr(s), subclass);     \
 463 } while (0)
 464
 465 static inline void write_seqcount_t_begin_nested(seqcount_t *s, int subclass)
 466 {
 467         raw_write_seqcount_t_begin(s);
 468         seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_);
 469 }
 470
 471 /**
 472  * write_seqcount_begin() - start a seqcount_t write side critical section
 473  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 474  *
 475  * write_seqcount_begin opens a write side critical section of the given
 476  * seqcount_t.
 477  *
 478  * Context: seqcount_t write side critical sections must be serialized and
 479  * non-preemptible. If readers can be invoked from hardirq or softirq
 480  * context, interrupts or bottom halves must be respectively disabled.
 481  */
 482 #define write_seqcount_begin(s)                                         \
 483 do {                                                                    \
 484         __seqcount_assert_lock_held(s);                                 \
 485                                                                         \
 486         if (__seqcount_lock_preemptible(s))                             \
 487                 preempt_disable();                                      \
 488                                                                         \
 489         write_seqcount_t_begin(__seqcount_ptr(s));                      \
 490 } while (0)
 491
 492 static inline void write_seqcount_t_begin(seqcount_t *s)
 493 {
 494         write_seqcount_t_begin_nested(s, 0);
 495 }
 496
 497 /**
 498  * write_seqcount_end() - end a seqcount_t write side critical section
 499  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 500  *
 501  * The write section must've been opened with write_seqcount_begin().
 502  */
 503 #define write_seqcount_end(s)                                           \
 504 do {                                                                    \
 505         write_seqcount_t_end(__seqcount_ptr(s));                        \
 506                                                                         \
 507         if (__seqcount_lock_preemptible(s))                             \
 508                 preempt_enable();                                       \
 509 } while (0)
 510
 511 static inline void write_seqcount_t_end(seqcount_t *s)
 512 {
 513         seqcount_release(&s->dep_map, _RET_IP_);
 514         raw_write_seqcount_t_end(s);
 515 }
 516
 517 /**
 518  * raw_write_seqcount_barrier() - do a seqcount_t write barrier
 519  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 520  *
 521  * This can be used to provide an ordering guarantee instead of the usual
 522  * consistency guarantee. It is one wmb cheaper, because it can collapse
 523  * the two back-to-back wmb()s.
 524  *
 525  * Note that writes surrounding the barrier should be declared atomic (e.g.
 526  * via WRITE_ONCE): a) to ensure the writes become visible to other threads
 527  * atomically, avoiding compiler optimizations; b) to document which writes are
 528  * meant to propagate to the reader critical section. This is necessary because
 529  * neither writes before and after the barrier are enclosed in a seq-writer
 530  * critical section that would ensure readers are aware of ongoing writes::
 531  *
 532  *      seqcount_t seq;
 533  *      bool X = true, Y = false;
 534  *
 535  *      void read(void)
 536  *      {
 537  *              bool x, y;
 538  *
 539  *              do {
 540  *                      int s = read_seqcount_begin(&seq);
 541  *
 542  *                      x = X; y = Y;
 543  *
 544  *              } while (read_seqcount_retry(&seq, s));
 545  *
 546  *              BUG_ON(!x && !y);
 547  *      }
 548  *
 549  *      void write(void)
 550  *      {
 551  *              WRITE_ONCE(Y, true);
 552  *
 553  *              raw_write_seqcount_barrier(seq);
 554  *
 555  *              WRITE_ONCE(X, false);
 556  *      }
 557  */
 558 #define raw_write_seqcount_barrier(s)                                   \
 559         raw_write_seqcount_t_barrier(__seqcount_ptr(s))
 560
 561 static inline void raw_write_seqcount_t_barrier(seqcount_t *s)
 562 {
 563         kcsan_nestable_atomic_begin();
 564         s->sequence++;
 565         smp_wmb();
 566         s->sequence++;
 567         kcsan_nestable_atomic_end();
 568 }
 569
 570 /**
 571  * write_seqcount_invalidate() - invalidate in-progress seqcount_t read
 572  *                               side operations
 573  * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
 574  *
 575  * After write_seqcount_invalidate, no seqcount_t read side operations
 576  * will complete successfully and see data older than this.
 577  */
 578 #define write_seqcount_invalidate(s)                                    \
 579         write_seqcount_t_invalidate(__seqcount_ptr(s))
 580
 581 static inline void write_seqcount_t_invalidate(seqcount_t *s)
 582 {
 583         smp_wmb();
 584         kcsan_nestable_atomic_begin();
 585         s->sequence+=2;
 586         kcsan_nestable_atomic_end();
 587 }
 588
 589 /*
 590  * Latch sequence counters (seqcount_latch_t)
 591  *
 592  * A sequence counter variant where the counter even/odd value is used to
 593  * switch between two copies of protected data. This allows the read path,
 594  * typically NMIs, to safely interrupt the write side critical section.
 595  *
 596  * As the write sections are fully preemptible, no special handling for
 597  * PREEMPT_RT is needed.
 598  */
 599 typedef struct {
 600         seqcount_t seqcount;
 601 } seqcount_latch_t;
 602
 603 /**
 604  * SEQCNT_LATCH_ZERO() - static initializer for seqcount_latch_t
 605  * @seq_name: Name of the seqcount_latch_t instance
 606  */
 607 #define SEQCNT_LATCH_ZERO(seq_name) {                                   \
 608         .seqcount               = SEQCNT_ZERO(seq_name.seqcount),       \
 609 }
 610
 611 /**
 612  * seqcount_latch_init() - runtime initializer for seqcount_latch_t
 613  * @s: Pointer to the seqcount_latch_t instance
 614  */
 615 static inline void seqcount_latch_init(seqcount_latch_t *s)
 616 {
 617         seqcount_init(&s->seqcount);
 618 }
 619
 620 /**
 621  * raw_read_seqcount_latch() - pick even/odd latch data copy
 622  * @s: Pointer to seqcount_latch_t
 623  *
 624  * See raw_write_seqcount_latch() for details and a full reader/writer
 625  * usage example.
 626  *
 627  * Return: sequence counter raw value. Use the lowest bit as an index for
 628  * picking which data copy to read. The full counter must then be checked
 629  * with read_seqcount_latch_retry().
 630  */
 631 static inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s)
 632 {
 633         /*
 634          * Pairs with the first smp_wmb() in raw_write_seqcount_latch().
 635          * Due to the dependent load, a full smp_rmb() is not needed.
 636          */
 637         return READ_ONCE(s->seqcount.sequence);
 638 }
 639
 640 /**
 641  * read_seqcount_latch_retry() - end a seqcount_latch_t read section
 642  * @s:          Pointer to seqcount_latch_t
 643  * @start:      count, from raw_read_seqcount_latch()
 644  *
 645  * Return: true if a read section retry is required, else false
 646  */
 647 static inline int
 648 read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start)
 649 {
 650         return read_seqcount_retry(&s->seqcount, start);
 651 }
 652
 653 /**
 654  * raw_write_seqcount_latch() - redirect latch readers to even/odd copy
 655  * @s: Pointer to seqcount_latch_t
 656  *
 657  * The latch technique is a multiversion concurrency control method that allows
 658  * queries during non-atomic modifications. If you can guarantee queries never
 659  * interrupt the modification -- e.g. the concurrency is strictly between CPUs
 660  * -- you most likely do not need this.
 661  *
 662  * Where the traditional RCU/lockless data structures rely on atomic
 663  * modifications to ensure queries observe either the old or the new state the
 664  * latch allows the same for non-atomic updates. The trade-off is doubling the
 665  * cost of storage; we have to maintain two copies of the entire data
 666  * structure.
 667  *
 668  * Very simply put: we first modify one copy and then the other. This ensures
 669  * there is always one copy in a stable state, ready to give us an answer.
 670  *
 671  * The basic form is a data structure like::
 672  *
 673  *      struct latch_struct {
 674  *              seqcount_latch_t        seq;
 675  *              struct data_struct      data[2];
 676  *      };
 677  *
 678  * Where a modification, which is assumed to be externally serialized, does the
 679  * following::
 680  *
 681  *      void latch_modify(struct latch_struct *latch, ...)
 682  *      {
 683  *              smp_wmb();      // Ensure that the last data[1] update is visible
 684  *              latch->seq.sequence++;
 685  *              smp_wmb();      // Ensure that the seqcount update is visible
 686  *
 687  *              modify(latch->data[0], ...);
 688  *
 689  *              smp_wmb();      // Ensure that the data[0] update is visible
 690  *              latch->seq.sequence++;
 691  *              smp_wmb();      // Ensure that the seqcount update is visible
 692  *
 693  *              modify(latch->data[1], ...);
 694  *      }
 695  *
 696  * The query will have a form like::
 697  *
 698  *      struct entry *latch_query(struct latch_struct *latch, ...)
 699  *      {
 700  *              struct entry *entry;
 701  *              unsigned seq, idx;
 702  *
 703  *              do {
 704  *                      seq = raw_read_seqcount_latch(&latch->seq);
 705  *
 706  *                      idx = seq & 0x01;
 707  *                      entry = data_query(latch->data[idx], ...);
 708  *
 709  *              // This includes needed smp_rmb()
 710  *              } while (read_seqcount_latch_retry(&latch->seq, seq));
 711  *
 712  *              return entry;
 713  *      }
 714  *
 715  * So during the modification, queries are first redirected to data[1]. Then we
 716  * modify data[0]. When that is complete, we redirect queries back to data[0]
 717  * and we can modify data[1].
 718  *
 719  * NOTE:
 720  *
 721  *      The non-requirement for atomic modifications does _NOT_ include
 722  *      the publishing of new entries in the case where data is a dynamic
 723  *      data structure.
 724  *
 725  *      An iteration might start in data[0] and get suspended long enough
 726  *      to miss an entire modification sequence, once it resumes it might
 727  *      observe the new entry.
 728  *
 729  * NOTE2:
 730  *
 731  *      When data is a dynamic data structure; one should use regular RCU
 732  *      patterns to manage the lifetimes of the objects within.
 733  */
 734 static inline void raw_write_seqcount_latch(seqcount_latch_t *s)
 735 {
 736         smp_wmb();      /* prior stores before incrementing "sequence" */
 737         s->seqcount.sequence++;
 738         smp_wmb();      /* increment "sequence" before following stores */
 739 }
 740
 741 /*
 742  * Sequential locks (seqlock_t)
 743  *
 744  * Sequence counters with an embedded spinlock for writer serialization
 745  * and non-preemptibility.
 746  *
 747  * For more info, see:
 748  *    - Comments on top of seqcount_t
 749  *    - Documentation/locking/seqlock.rst
 750  */
 751 typedef struct {
 752         struct seqcount seqcount;
 753         spinlock_t lock;
 754 } seqlock_t;
 755
 756 #define __SEQLOCK_UNLOCKED(lockname)                                    \
 757         {                                                               \
 758                 .seqcount = SEQCNT_ZERO(lockname),                      \
 759                 .lock = __SPIN_LOCK_UNLOCKED(lockname)                  \
 760         }
 761
 762 /**
 763  * seqlock_init() - dynamic initializer for seqlock_t
 764  * @sl: Pointer to the seqlock_t instance
 765  */
 766 #define seqlock_init(sl)                                                \
 767         do {                                                            \
 768                 seqcount_init(&(sl)->seqcount);                         \
 769                 spin_lock_init(&(sl)->lock);                            \
 770         } while (0)
 771
 772 /**
 773  * DEFINE_SEQLOCK() - Define a statically allocated seqlock_t
 774  * @sl: Name of the seqlock_t instance
 775  */
 776 #define DEFINE_SEQLOCK(sl) \
 777                 seqlock_t sl = __SEQLOCK_UNLOCKED(sl)
 778
 779 /**
 780  * read_seqbegin() - start a seqlock_t read side critical section
 781  * @sl: Pointer to seqlock_t
 782  *
 783  * Return: count, to be passed to read_seqretry()
 784  */
 785 static inline unsigned read_seqbegin(const seqlock_t *sl)
 786 {
 787         unsigned ret = read_seqcount_begin(&sl->seqcount);
 788
 789         kcsan_atomic_next(0);  /* non-raw usage, assume closing read_seqretry() */
 790         kcsan_flat_atomic_begin();
 791         return ret;
 792 }
 793
 794 /**
 795  * read_seqretry() - end a seqlock_t read side section
 796  * @sl: Pointer to seqlock_t
 797  * @start: count, from read_seqbegin()
 798  *
 799  * read_seqretry closes the read side critical section of given seqlock_t.
 800  * If the critical section was invalid, it must be ignored (and typically
 801  * retried).
 802  *
 803  * Return: true if a read section retry is required, else false
 804  */
 805 static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start)
 806 {
 807         /*
 808          * Assume not nested: read_seqretry() may be called multiple times when
 809          * completing read critical section.
 810          */
 811         kcsan_flat_atomic_end();
 812
 813         return read_seqcount_retry(&sl->seqcount, start);
 814 }
 815
 816 /**
 817  * write_seqlock() - start a seqlock_t write side critical section
 818  * @sl: Pointer to seqlock_t
 819  *
 820  * write_seqlock opens a write side critical section for the given
 821  * seqlock_t.  It also implicitly acquires the spinlock_t embedded inside
 822  * that sequential lock. All seqlock_t write side sections are thus
 823  * automatically serialized and non-preemptible.
 824  *
 825  * Context: if the seqlock_t read section, or other write side critical
 826  * sections, can be invoked from hardirq or softirq contexts, use the
 827  * _irqsave or _bh variants of this function instead.
 828  */
 829 static inline void write_seqlock(seqlock_t *sl)
 830 {
 831         spin_lock(&sl->lock);
 832         write_seqcount_t_begin(&sl->seqcount);
 833 }
 834
 835 /**
 836  * write_sequnlock() - end a seqlock_t write side critical section
 837  * @sl: Pointer to seqlock_t
 838  *
 839  * write_sequnlock closes the (serialized and non-preemptible) write side
 840  * critical section of given seqlock_t.
 841  */
 842 static inline void write_sequnlock(seqlock_t *sl)
 843 {
 844         write_seqcount_t_end(&sl->seqcount);
 845         spin_unlock(&sl->lock);
 846 }
 847
 848 /**
 849  * write_seqlock_bh() - start a softirqs-disabled seqlock_t write section
 850  * @sl: Pointer to seqlock_t
 851  *
 852  * _bh variant of write_seqlock(). Use only if the read side section, or
 853  * other write side sections, can be invoked from softirq contexts.
 854  */
 855 static inline void write_seqlock_bh(seqlock_t *sl)
 856 {
 857         spin_lock_bh(&sl->lock);
 858         write_seqcount_t_begin(&sl->seqcount);
 859 }
 860
 861 /**
 862  * write_sequnlock_bh() - end a softirqs-disabled seqlock_t write section
 863  * @sl: Pointer to seqlock_t
 864  *
 865  * write_sequnlock_bh closes the serialized, non-preemptible, and
 866  * softirqs-disabled, seqlock_t write side critical section opened with
 867  * write_seqlock_bh().
 868  */
 869 static inline void write_sequnlock_bh(seqlock_t *sl)
 870 {
 871         write_seqcount_t_end(&sl->seqcount);
 872         spin_unlock_bh(&sl->lock);
 873 }
 874
 875 /**
 876  * write_seqlock_irq() - start a non-interruptible seqlock_t write section
 877  * @sl: Pointer to seqlock_t
 878  *
 879  * _irq variant of write_seqlock(). Use only if the read side section, or
 880  * other write sections, can be invoked from hardirq contexts.
 881  */
 882 static inline void write_seqlock_irq(seqlock_t *sl)
 883 {
 884         spin_lock_irq(&sl->lock);
 885         write_seqcount_t_begin(&sl->seqcount);
 886 }
 887
 888 /**
 889  * write_sequnlock_irq() - end a non-interruptible seqlock_t write section
 890  * @sl: Pointer to seqlock_t
 891  *
 892  * write_sequnlock_irq closes the serialized and non-interruptible
 893  * seqlock_t write side section opened with write_seqlock_irq().
 894  */
 895 static inline void write_sequnlock_irq(seqlock_t *sl)
 896 {
 897         write_seqcount_t_end(&sl->seqcount);
 898         spin_unlock_irq(&sl->lock);
 899 }
 900
 901 static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl)
 902 {
 903         unsigned long flags;
 904
 905         spin_lock_irqsave(&sl->lock, flags);
 906         write_seqcount_t_begin(&sl->seqcount);
 907         return flags;
 908 }
 909
 910 /**
 911  * write_seqlock_irqsave() - start a non-interruptible seqlock_t write
 912  *                           section
 913  * @lock:  Pointer to seqlock_t
 914  * @flags: Stack-allocated storage for saving caller's local interrupt
 915  *         state, to be passed to write_sequnlock_irqrestore().
 916  *
 917  * _irqsave variant of write_seqlock(). Use it only if the read side
 918  * section, or other write sections, can be invoked from hardirq context.
 919  */
 920 #define write_seqlock_irqsave(lock, flags)                              \
 921         do { flags = __write_seqlock_irqsave(lock); } while (0)
 922
 923 /**
 924  * write_sequnlock_irqrestore() - end non-interruptible seqlock_t write
 925  *                                section
 926  * @sl:    Pointer to seqlock_t
 927  * @flags: Caller's saved interrupt state, from write_seqlock_irqsave()
 928  *
 929  * write_sequnlock_irqrestore closes the serialized and non-interruptible
 930  * seqlock_t write section previously opened with write_seqlock_irqsave().
 931  */
 932 static inline void
 933 write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags)
 934 {
 935         write_seqcount_t_end(&sl->seqcount);
 936         spin_unlock_irqrestore(&sl->lock, flags);
 937 }
 938
 939 /**
 940  * read_seqlock_excl() - begin a seqlock_t locking reader section
 941  * @sl: Pointer to seqlock_t
 942  *
 943  * read_seqlock_excl opens a seqlock_t locking reader critical section.  A
 944  * locking reader exclusively locks out *both* other writers *and* other
 945  * locking readers, but it does not update the embedded sequence number.
 946  *
 947  * Locking readers act like a normal spin_lock()/spin_unlock().
 948  *
 949  * Context: if the seqlock_t write section, *or other read sections*, can
 950  * be invoked from hardirq or softirq contexts, use the _irqsave or _bh
 951  * variant of this function instead.
 952  *
 953  * The opened read section must be closed with read_sequnlock_excl().
 954  */
 955 static inline void read_seqlock_excl(seqlock_t *sl)
 956 {
 957         spin_lock(&sl->lock);
 958 }
 959
 960 /**
 961  * read_sequnlock_excl() - end a seqlock_t locking reader critical section
 962  * @sl: Pointer to seqlock_t
 963  */
 964 static inline void read_sequnlock_excl(seqlock_t *sl)
 965 {
 966         spin_unlock(&sl->lock);
 967 }
 968
 969 /**
 970  * read_seqlock_excl_bh() - start a seqlock_t locking reader section with
 971  *                          softirqs disabled
 972  * @sl: Pointer to seqlock_t
 973  *
 974  * _bh variant of read_seqlock_excl(). Use this variant only if the
 975  * seqlock_t write side section, *or other read sections*, can be invoked
 976  * from softirq contexts.
 977  */
 978 static inline void read_seqlock_excl_bh(seqlock_t *sl)
 979 {
 980         spin_lock_bh(&sl->lock);
 981 }
 982
 983 /**
 984  * read_sequnlock_excl_bh() - stop a seqlock_t softirq-disabled locking
 985  *                            reader section
 986  * @sl: Pointer to seqlock_t
 987  */
 988 static inline void read_sequnlock_excl_bh(seqlock_t *sl)
 989 {
 990         spin_unlock_bh(&sl->lock);
 991 }
 992
 993 /**
 994  * read_seqlock_excl_irq() - start a non-interruptible seqlock_t locking
 995  *                           reader section
 996  * @sl: Pointer to seqlock_t
 997  *
 998  * _irq variant of read_seqlock_excl(). Use this only if the seqlock_t
 999  * write side section, *or other read sections*, can be invoked from a
1000  * hardirq context.
1001  */
1002 static inline void read_seqlock_excl_irq(seqlock_t *sl)
1003 {
1004         spin_lock_irq(&sl->lock);
1005 }
1006
1007 /**
1008  * read_sequnlock_excl_irq() - end an interrupts-disabled seqlock_t
1009  *                             locking reader section
1010  * @sl: Pointer to seqlock_t
1011  */
1012 static inline void read_sequnlock_excl_irq(seqlock_t *sl)
1013 {
1014         spin_unlock_irq(&sl->lock);
1015 }
1016
1017 static inline unsigned long __read_seqlock_excl_irqsave(seqlock_t *sl)
1018 {
1019         unsigned long flags;
1020
1021         spin_lock_irqsave(&sl->lock, flags);
1022         return flags;
1023 }
1024
1025 /**
1026  * read_seqlock_excl_irqsave() - start a non-interruptible seqlock_t
1027  *                               locking reader section
1028  * @lock:  Pointer to seqlock_t
1029  * @flags: Stack-allocated storage for saving caller's local interrupt
1030  *         state, to be passed to read_sequnlock_excl_irqrestore().
1031  *
1032  * _irqsave variant of read_seqlock_excl(). Use this only if the seqlock_t
1033  * write side section, *or other read sections*, can be invoked from a
1034  * hardirq context.
1035  */
1036 #define read_seqlock_excl_irqsave(lock, flags)                          \
1037         do { flags = __read_seqlock_excl_irqsave(lock); } while (0)
1038
1039 /**
1040  * read_sequnlock_excl_irqrestore() - end non-interruptible seqlock_t
1041  *                                    locking reader section
1042  * @sl:    Pointer to seqlock_t
1043  * @flags: Caller saved interrupt state, from read_seqlock_excl_irqsave()
1044  */
1045 static inline void
1046 read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags)
1047 {
1048         spin_unlock_irqrestore(&sl->lock, flags);
1049 }
1050
1051 /**
1052  * read_seqbegin_or_lock() - begin a seqlock_t lockless or locking reader
1053  * @lock: Pointer to seqlock_t
1054  * @seq : Marker and return parameter. If the passed value is even, the
1055  * reader will become a *lockless* seqlock_t reader as in read_seqbegin().
1056  * If the passed value is odd, the reader will become a *locking* reader
1057  * as in read_seqlock_excl().  In the first call to this function, the
1058  * caller *must* initialize and pass an even value to @seq; this way, a
1059  * lockless read can be optimistically tried first.
1060  *
1061  * read_seqbegin_or_lock is an API designed to optimistically try a normal
1062  * lockless seqlock_t read section first.  If an odd counter is found, the
1063  * lockless read trial has failed, and the next read iteration transforms
1064  * itself into a full seqlock_t locking reader.
1065  *
1066  * This is typically used to avoid seqlock_t lockless readers starvation
1067  * (too much retry loops) in the case of a sharp spike in write side
1068  * activity.
1069  *
1070  * Context: if the seqlock_t write section, *or other read sections*, can
1071  * be invoked from hardirq or softirq contexts, use the _irqsave or _bh
1072  * variant of this function instead.
1073  *
1074  * Check Documentation/locking/seqlock.rst for template example code.
1075  *
1076  * Return: the encountered sequence counter value, through the @seq
1077  * parameter, which is overloaded as a return parameter. This returned
1078  * value must be checked with need_seqretry(). If the read section need to
1079  * be retried, this returned value must also be passed as the @seq
1080  * parameter of the next read_seqbegin_or_lock() iteration.
1081  */
1082 static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq)
1083 {
1084         if (!(*seq & 1))        /* Even */
1085                 *seq = read_seqbegin(lock);
1086         else                    /* Odd */
1087                 read_seqlock_excl(lock);
1088 }
1089
1090 /**
1091  * need_seqretry() - validate seqlock_t "locking or lockless" read section
1092  * @lock: Pointer to seqlock_t
1093  * @seq: sequence count, from read_seqbegin_or_lock()
1094  *
1095  * Return: true if a read section retry is required, false otherwise
1096  */
1097 static inline int need_seqretry(seqlock_t *lock, int seq)
1098 {
1099         return !(seq & 1) && read_seqretry(lock, seq);
1100 }
1101
1102 /**
1103  * done_seqretry() - end seqlock_t "locking or lockless" reader section
1104  * @lock: Pointer to seqlock_t
1105  * @seq: count, from read_seqbegin_or_lock()
1106  *
1107  * done_seqretry finishes the seqlock_t read side critical section started
1108  * with read_seqbegin_or_lock() and validated by need_seqretry().
1109  */
1110 static inline void done_seqretry(seqlock_t *lock, int seq)
1111 {
1112         if (seq & 1)
1113                 read_sequnlock_excl(lock);
1114 }
1115
1116 /**
1117  * read_seqbegin_or_lock_irqsave() - begin a seqlock_t lockless reader, or
1118  *                                   a non-interruptible locking reader
1119  * @lock: Pointer to seqlock_t
1120  * @seq:  Marker and return parameter. Check read_seqbegin_or_lock().
1121  *
1122  * This is the _irqsave variant of read_seqbegin_or_lock(). Use it only if
1123  * the seqlock_t write section, *or other read sections*, can be invoked
1124  * from hardirq context.
1125  *
1126  * Note: Interrupts will be disabled only for "locking reader" mode.
1127  *
1128  * Return:
1129  *
1130  *   1. The saved local interrupts state in case of a locking reader, to
1131  *      be passed to done_seqretry_irqrestore().
1132  *
1133  *   2. The encountered sequence counter value, returned through @seq
1134  *      overloaded as a return parameter. Check read_seqbegin_or_lock().
1135  */
1136 static inline unsigned long
1137 read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq)
1138 {
1139         unsigned long flags = 0;
1140
1141         if (!(*seq & 1))        /* Even */
1142                 *seq = read_seqbegin(lock);
1143         else                    /* Odd */
1144                 read_seqlock_excl_irqsave(lock, flags);
1145
1146         return flags;
1147 }
1148
1149 /**
1150  * done_seqretry_irqrestore() - end a seqlock_t lockless reader, or a
1151  *                              non-interruptible locking reader section
1152  * @lock:  Pointer to seqlock_t
1153  * @seq:   Count, from read_seqbegin_or_lock_irqsave()
1154  * @flags: Caller's saved local interrupt state in case of a locking
1155  *         reader, also from read_seqbegin_or_lock_irqsave()
1156  *
1157  * This is the _irqrestore variant of done_seqretry(). The read section
1158  * must've been opened with read_seqbegin_or_lock_irqsave(), and validated
1159  * by need_seqretry().
1160  */
1161 static inline void
1162 done_seqretry_irqrestore(seqlock_t *lock, int seq, unsigned long flags)
1163 {
1164         if (seq & 1)
1165                 read_sequnlock_excl_irqrestore(lock, flags);
1166 }
1167 #endif /* __LINUX_SEQLOCK_H */