Merge tag 'locking-core-2021-10-31' of git://git.kernel.org/pub/scm/linux/kernel...

[linux-2.6-microblaze.git] / include / linux / sched.h

diff --git a/include/linux/sched.h b/include/linux/sched.h
index e0454e6..caf7071 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2037,7 +2037,7 @@ static inline int _cond_resched(void) { return 0; }
 #endif /* !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) */
 
 #define cond_resched() ({                      \
-       ___might_sleep(__FILE__, __LINE__, 0);  \
+       __might_resched(__FILE__, __LINE__, 0); \
        _cond_resched();                        \
 })
 
@@ -2045,19 +2045,38 @@ extern int __cond_resched_lock(spinlock_t *lock);
 extern int __cond_resched_rwlock_read(rwlock_t *lock);
 extern int __cond_resched_rwlock_write(rwlock_t *lock);
 
-#define cond_resched_lock(lock) ({                             \
-       ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
-       __cond_resched_lock(lock);                              \
+#define MIGHT_RESCHED_RCU_SHIFT                8
+#define MIGHT_RESCHED_PREEMPT_MASK     ((1U << MIGHT_RESCHED_RCU_SHIFT) - 1)
+
+#ifndef CONFIG_PREEMPT_RT
+/*
+ * Non RT kernels have an elevated preempt count due to the held lock,
+ * but are not allowed to be inside a RCU read side critical section
+ */
+# define PREEMPT_LOCK_RESCHED_OFFSETS  PREEMPT_LOCK_OFFSET
+#else
+/*
+ * spin/rw_lock() on RT implies rcu_read_lock(). The might_sleep() check in
+ * cond_resched*lock() has to take that into account because it checks for
+ * preempt_count() and rcu_preempt_depth().
+ */
+# define PREEMPT_LOCK_RESCHED_OFFSETS  \
+       (PREEMPT_LOCK_OFFSET + (1U << MIGHT_RESCHED_RCU_SHIFT))
+#endif
+
+#define cond_resched_lock(lock) ({                                             \
+       __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS);      \
+       __cond_resched_lock(lock);                                              \
 })
 
-#define cond_resched_rwlock_read(lock) ({                      \
-       __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \
-       __cond_resched_rwlock_read(lock);                       \
+#define cond_resched_rwlock_read(lock) ({                                      \
+       __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS);      \
+       __cond_resched_rwlock_read(lock);                                       \
 })
 
-#define cond_resched_rwlock_write(lock) ({                     \
-       __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \
-       __cond_resched_rwlock_write(lock);                      \
+#define cond_resched_rwlock_write(lock) ({                                     \
+       __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS);      \
+       __cond_resched_rwlock_write(lock);                                      \
 })
 
 static inline void cond_resched_rcu(void)