seqlock: seqcount_LOCKNAME_t: Introduce PREEMPT_RT support

Preemption must be disabled before entering a sequence counter write
side critical section.  Otherwise the read side section can preempt the
write side section and spin for the entire scheduler tick.  If that
reader belongs to a real-time scheduling class, it can spin forever and
the kernel will livelock.

Disabling preemption cannot be done for PREEMPT_RT though: it can lead
to higher latencies, and the write side sections will not be able to
acquire locks which become sleeping locks (e.g. spinlock_t).

To remain preemptible, while avoiding a possible livelock caused by the
reader preempting the writer, use a different technique: let the reader
detect if a seqcount_LOCKNAME_t writer is in progress. If that's the
case, acquire then release the associated LOCKNAME writer serialization
lock. This will allow any possibly-preempted writer to make progress
until the end of its writer serialization lock critical section.

Implement this lock-unlock technique for all seqcount_LOCKNAME_t with
an associated (PREEMPT_RT) sleeping lock.

References: 55f3560df975 ("seqlock: Extend seqcount API with associated locks")
Signed-off-by: Ahmed S. Darwish <a.darwish@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20200519214547.352050-1-a.darwish@linutronix.de

diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h
index f3b7827..2bc9510 100644 (file)
--- a/include/linux/seqlock.h
+++ b/include/linux/seqlock.h
@@ -17,6 +17,7 @@
 #include <linux/kcsan-checks.h>
 #include <linux/lockdep.h>
 #include <linux/mutex.h>
+#include <linux/ww_mutex.h>
 #include <linux/preempt.h>
 #include <linux/spinlock.h>
 
@@ -131,7 +132,23 @@ static inline void seqcount_lockdep_reader_access(const seqcount_t *s)
  * See Documentation/locking/seqlock.rst
  */
 
-#ifdef CONFIG_LOCKDEP
+/*
+ * For PREEMPT_RT, seqcount_LOCKNAME_t write side critical sections cannot
+ * disable preemption. It can lead to higher latencies, and the write side
+ * sections will not be able to acquire locks which become sleeping locks
+ * (e.g. spinlock_t).
+ *
+ * To remain preemptible while avoiding a possible livelock caused by the
+ * reader preempting the writer, use a different technique: let the reader
+ * detect if a seqcount_LOCKNAME_t writer is in progress. If that is the
+ * case, acquire then release the associated LOCKNAME writer serialization
+ * lock. This will allow any possibly-preempted writer to make progress
+ * until the end of its writer serialization lock critical section.
+ *
+ * This lock-unlock technique must be implemented for all of PREEMPT_RT
+ * sleeping locks.  See Documentation/locking/locktypes.rst
+ */
+#if defined(CONFIG_LOCKDEP) || defined(CONFIG_PREEMPT_RT)
 #define __SEQ_LOCK(expr)       expr
 #else
 #define __SEQ_LOCK(expr)
@@ -162,10 +179,12 @@ static inline void seqcount_lockdep_reader_access(const seqcount_t *s)
  *
  * @lockname:          "LOCKNAME" part of seqcount_LOCKNAME_t
  * @locktype:          LOCKNAME canonical C data type
- * @preemptible:       preemptibility of above lockname
+ * @preemptible:       preemptibility of above locktype
  * @lockmember:                argument for lockdep_assert_held()
+ * @lockbase:          associated lock release function (prefix only)
+ * @lock_acquire:      associated lock acquisition function (full call)
  */
-#define SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockmember) \
+#define SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockmember, lockbase, lock_acquire) \
 typedef struct seqcount_##lockname {                                   \
        seqcount_t              seqcount;                               \
        __SEQ_LOCK(locktype     *lock);                                 \
@@ -187,13 +206,33 @@ __seqprop_##lockname##_ptr(seqcount_##lockname##_t *s)                    \
 static __always_inline unsigned                                                \
 __seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s)      \
 {                                                                      \
-       return READ_ONCE(s->seqcount.sequence);                         \
+       unsigned seq = READ_ONCE(s->seqcount.sequence);                 \
+                                                                       \
+       if (!IS_ENABLED(CONFIG_PREEMPT_RT))                             \
+               return seq;                                             \
+                                                                       \
+       if (preemptible && unlikely(seq & 1)) {                         \
+               __SEQ_LOCK(lock_acquire);                               \
+               __SEQ_LOCK(lockbase##_unlock(s->lock));                 \
+                                                                       \
+               /*                                                      \
+                * Re-read the sequence counter since the (possibly     \
+                * preempted) writer made progress.                     \
+                */                                                     \
+               seq = READ_ONCE(s->seqcount.sequence);                  \
+       }                                                               \
+                                                                       \
+       return seq;                                                     \
 }                                                                      \
                                                                        \
 static __always_inline bool                                            \
 __seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s)   \
 {                                                                      \
-       return preemptible;                                             \
+       if (!IS_ENABLED(CONFIG_PREEMPT_RT))                             \
+               return preemptible;                                     \
+                                                                       \
+       /* PREEMPT_RT relies on the above LOCK+UNLOCK */                \
+       return false;                                                   \
 }                                                                      \
                                                                        \
 static __always_inline void                                            \
@@ -226,11 +265,13 @@ static inline void __seqprop_assert(const seqcount_t *s)
        lockdep_assert_preemption_disabled();
 }
 
-SEQCOUNT_LOCKNAME(raw_spinlock,        raw_spinlock_t,         false,  s->lock)
-SEQCOUNT_LOCKNAME(spinlock,    spinlock_t,             false,  s->lock)
-SEQCOUNT_LOCKNAME(rwlock,      rwlock_t,               false,  s->lock)
-SEQCOUNT_LOCKNAME(mutex,       struct mutex,           true,   s->lock)
-SEQCOUNT_LOCKNAME(ww_mutex,    struct ww_mutex,        true,   &s->lock->base)
+#define __SEQ_RT       IS_ENABLED(CONFIG_PREEMPT_RT)
+
+SEQCOUNT_LOCKNAME(raw_spinlock, raw_spinlock_t,  false,    s->lock,        raw_spin, raw_spin_lock(s->lock))
+SEQCOUNT_LOCKNAME(spinlock,     spinlock_t,      __SEQ_RT, s->lock,        spin,     spin_lock(s->lock))
+SEQCOUNT_LOCKNAME(rwlock,       rwlock_t,        __SEQ_RT, s->lock,        read,     read_lock(s->lock))
+SEQCOUNT_LOCKNAME(mutex,        struct mutex,    true,     s->lock,        mutex,    mutex_lock(s->lock))
+SEQCOUNT_LOCKNAME(ww_mutex,     struct ww_mutex, true,     &s->lock->base, ww_mutex, ww_mutex_lock(s->lock, NULL))
 
 /*
  * SEQCNT_LOCKNAME_ZERO - static initializer for seqcount_LOCKNAME_t