Merge tag 'docs-5.12' of git://git.lwn.net/linux

[linux-2.6-microblaze.git] / kernel / entry / common.c

diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index f9d491b..8442e5c 100644 (file)
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -184,6 +184,10 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
                 * enabled above.
                 */
                local_irq_disable_exit_to_user();
+
+               /* Check if any of the above work has queued a deferred wakeup */
+               rcu_nocb_flush_deferred_wakeup();
+
                ti_work = READ_ONCE(current_thread_info()->flags);
        }
 
@@ -197,6 +201,9 @@ static void exit_to_user_mode_prepare(struct pt_regs *regs)
 
        lockdep_assert_irqs_disabled();
 
+       /* Flush pending rcuog wakeup before the last need_resched() check */
+       rcu_nocb_flush_deferred_wakeup();
+
        if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
                ti_work = exit_to_user_mode_loop(regs, ti_work);
 
@@ -385,6 +392,9 @@ void irqentry_exit_cond_resched(void)
                        preempt_schedule_irq();
        }
 }
+#ifdef CONFIG_PREEMPT_DYNAMIC
+DEFINE_STATIC_CALL(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
+#endif
 
 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
 {
@@ -411,8 +421,13 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
                }
 
                instrumentation_begin();
-               if (IS_ENABLED(CONFIG_PREEMPTION))
+               if (IS_ENABLED(CONFIG_PREEMPTION)) {
+#ifdef CONFIG_PREEMT_DYNAMIC
+                       static_call(irqentry_exit_cond_resched)();
+#else
                        irqentry_exit_cond_resched();
+#endif
+               }
                /* Covers both tracing and lockdep */
                trace_hardirqs_on();
                instrumentation_end();