1 /* SPDX-License-Identifier: GPL-2.0+ */
3 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
4 * Internal non-public definitions that provide either classic
5 * or preemptible semantics.
7 * Copyright Red Hat, 2009
8 * Copyright IBM Corporation, 2009
10 * Author: Ingo Molnar <mingo@elte.hu>
11 * Paul E. McKenney <paulmck@linux.ibm.com>
14 #include "../locking/rtmutex_common.h"
16 #ifdef CONFIG_RCU_NOCB_CPU
17 static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
18 static bool __read_mostly rcu_nocb_poll; /* Offload kthread are to poll. */
19 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp)
21 return lockdep_is_held(&rdp->nocb_lock);
24 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)
26 /* Race on early boot between thread creation and assignment */
27 if (!rdp->nocb_cb_kthread || !rdp->nocb_gp_kthread)
30 if (current == rdp->nocb_cb_kthread || current == rdp->nocb_gp_kthread)
36 static inline bool rcu_running_nocb_timer(struct rcu_data *rdp)
38 return (timer_curr_running(&rdp->nocb_timer) && !in_irq());
41 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp)
46 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)
51 static inline bool rcu_running_nocb_timer(struct rcu_data *rdp)
56 #endif /* #ifdef CONFIG_RCU_NOCB_CPU */
58 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp)
61 * In order to read the offloaded state of an rdp is a safe
62 * and stable way and prevent from its value to be changed
63 * under us, we must either hold the barrier mutex, the cpu
64 * hotplug lock (read or write) or the nocb lock. Local
65 * non-preemptible reads are also safe. NOCB kthreads and
66 * timers have their own means of synchronization against the
67 * offloaded state updaters.
70 !(lockdep_is_held(&rcu_state.barrier_mutex) ||
71 (IS_ENABLED(CONFIG_HOTPLUG_CPU) && lockdep_is_cpus_held()) ||
72 rcu_lockdep_is_held_nocb(rdp) ||
73 (rdp == this_cpu_ptr(&rcu_data) &&
74 !(IS_ENABLED(CONFIG_PREEMPT_COUNT) && preemptible())) ||
75 rcu_current_is_nocb_kthread(rdp) ||
76 rcu_running_nocb_timer(rdp)),
77 "Unsafe read of RCU_NOCB offloaded state"
80 return rcu_segcblist_is_offloaded(&rdp->cblist);
84 * Check the RCU kernel configuration parameters and print informative
85 * messages about anything out of the ordinary.
87 static void __init rcu_bootup_announce_oddness(void)
89 if (IS_ENABLED(CONFIG_RCU_TRACE))
90 pr_info("\tRCU event tracing is enabled.\n");
91 if ((IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 64) ||
92 (!IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 32))
93 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d.\n",
96 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
97 if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
98 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
99 if (IS_ENABLED(CONFIG_PROVE_RCU))
100 pr_info("\tRCU lockdep checking is enabled.\n");
101 if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
102 pr_info("\tRCU strict (and thus non-scalable) grace periods enabled.\n");
103 if (RCU_NUM_LVLS >= 4)
104 pr_info("\tFour(or more)-level hierarchy is enabled.\n");
105 if (RCU_FANOUT_LEAF != 16)
106 pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
108 if (rcu_fanout_leaf != RCU_FANOUT_LEAF)
109 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n",
111 if (nr_cpu_ids != NR_CPUS)
112 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%u.\n", NR_CPUS, nr_cpu_ids);
113 #ifdef CONFIG_RCU_BOOST
114 pr_info("\tRCU priority boosting: priority %d delay %d ms.\n",
115 kthread_prio, CONFIG_RCU_BOOST_DELAY);
117 if (blimit != DEFAULT_RCU_BLIMIT)
118 pr_info("\tBoot-time adjustment of callback invocation limit to %ld.\n", blimit);
119 if (qhimark != DEFAULT_RCU_QHIMARK)
120 pr_info("\tBoot-time adjustment of callback high-water mark to %ld.\n", qhimark);
121 if (qlowmark != DEFAULT_RCU_QLOMARK)
122 pr_info("\tBoot-time adjustment of callback low-water mark to %ld.\n", qlowmark);
123 if (qovld != DEFAULT_RCU_QOVLD)
124 pr_info("\tBoot-time adjustment of callback overload level to %ld.\n", qovld);
125 if (jiffies_till_first_fqs != ULONG_MAX)
126 pr_info("\tBoot-time adjustment of first FQS scan delay to %ld jiffies.\n", jiffies_till_first_fqs);
127 if (jiffies_till_next_fqs != ULONG_MAX)
128 pr_info("\tBoot-time adjustment of subsequent FQS scan delay to %ld jiffies.\n", jiffies_till_next_fqs);
129 if (jiffies_till_sched_qs != ULONG_MAX)
130 pr_info("\tBoot-time adjustment of scheduler-enlistment delay to %ld jiffies.\n", jiffies_till_sched_qs);
131 if (rcu_kick_kthreads)
132 pr_info("\tKick kthreads if too-long grace period.\n");
133 if (IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD))
134 pr_info("\tRCU callback double-/use-after-free debug enabled.\n");
135 if (gp_preinit_delay)
136 pr_info("\tRCU debug GP pre-init slowdown %d jiffies.\n", gp_preinit_delay);
138 pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_init_delay);
139 if (gp_cleanup_delay)
140 pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_cleanup_delay);
142 pr_info("\tRCU_SOFTIRQ processing moved to rcuc kthreads.\n");
143 if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG))
144 pr_info("\tRCU debug extended QS entry/exit.\n");
145 rcupdate_announce_bootup_oddness();
148 #ifdef CONFIG_PREEMPT_RCU
150 static void rcu_report_exp_rnp(struct rcu_node *rnp, bool wake);
151 static void rcu_read_unlock_special(struct task_struct *t);
154 * Tell them what RCU they are running.
156 static void __init rcu_bootup_announce(void)
158 pr_info("Preemptible hierarchical RCU implementation.\n");
159 rcu_bootup_announce_oddness();
162 /* Flags for rcu_preempt_ctxt_queue() decision table. */
163 #define RCU_GP_TASKS 0x8
164 #define RCU_EXP_TASKS 0x4
165 #define RCU_GP_BLKD 0x2
166 #define RCU_EXP_BLKD 0x1
169 * Queues a task preempted within an RCU-preempt read-side critical
170 * section into the appropriate location within the ->blkd_tasks list,
171 * depending on the states of any ongoing normal and expedited grace
172 * periods. The ->gp_tasks pointer indicates which element the normal
173 * grace period is waiting on (NULL if none), and the ->exp_tasks pointer
174 * indicates which element the expedited grace period is waiting on (again,
175 * NULL if none). If a grace period is waiting on a given element in the
176 * ->blkd_tasks list, it also waits on all subsequent elements. Thus,
177 * adding a task to the tail of the list blocks any grace period that is
178 * already waiting on one of the elements. In contrast, adding a task
179 * to the head of the list won't block any grace period that is already
180 * waiting on one of the elements.
182 * This queuing is imprecise, and can sometimes make an ongoing grace
183 * period wait for a task that is not strictly speaking blocking it.
184 * Given the choice, we needlessly block a normal grace period rather than
185 * blocking an expedited grace period.
187 * Note that an endless sequence of expedited grace periods still cannot
188 * indefinitely postpone a normal grace period. Eventually, all of the
189 * fixed number of preempted tasks blocking the normal grace period that are
190 * not also blocking the expedited grace period will resume and complete
191 * their RCU read-side critical sections. At that point, the ->gp_tasks
192 * pointer will equal the ->exp_tasks pointer, at which point the end of
193 * the corresponding expedited grace period will also be the end of the
194 * normal grace period.
196 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
197 __releases(rnp->lock) /* But leaves rrupts disabled. */
199 int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
200 (rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
201 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +
202 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);
203 struct task_struct *t = current;
205 raw_lockdep_assert_held_rcu_node(rnp);
206 WARN_ON_ONCE(rdp->mynode != rnp);
207 WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
208 /* RCU better not be waiting on newly onlined CPUs! */
209 WARN_ON_ONCE(rnp->qsmaskinitnext & ~rnp->qsmaskinit & rnp->qsmask &
213 * Decide where to queue the newly blocked task. In theory,
214 * this could be an if-statement. In practice, when I tried
215 * that, it was quite messy.
217 switch (blkd_state) {
220 case RCU_EXP_TASKS + RCU_GP_BLKD:
222 case RCU_GP_TASKS + RCU_EXP_TASKS:
225 * Blocking neither GP, or first task blocking the normal
226 * GP but not blocking the already-waiting expedited GP.
227 * Queue at the head of the list to avoid unnecessarily
228 * blocking the already-waiting GPs.
230 list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
235 case RCU_GP_BLKD + RCU_EXP_BLKD:
236 case RCU_GP_TASKS + RCU_EXP_BLKD:
237 case RCU_GP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
238 case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
241 * First task arriving that blocks either GP, or first task
242 * arriving that blocks the expedited GP (with the normal
243 * GP already waiting), or a task arriving that blocks
244 * both GPs with both GPs already waiting. Queue at the
245 * tail of the list to avoid any GP waiting on any of the
246 * already queued tasks that are not blocking it.
248 list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks);
251 case RCU_EXP_TASKS + RCU_EXP_BLKD:
252 case RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
253 case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_EXP_BLKD:
256 * Second or subsequent task blocking the expedited GP.
257 * The task either does not block the normal GP, or is the
258 * first task blocking the normal GP. Queue just after
259 * the first task blocking the expedited GP.
261 list_add(&t->rcu_node_entry, rnp->exp_tasks);
264 case RCU_GP_TASKS + RCU_GP_BLKD:
265 case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD:
268 * Second or subsequent task blocking the normal GP.
269 * The task does not block the expedited GP. Queue just
270 * after the first task blocking the normal GP.
272 list_add(&t->rcu_node_entry, rnp->gp_tasks);
277 /* Yet another exercise in excessive paranoia. */
283 * We have now queued the task. If it was the first one to
284 * block either grace period, update the ->gp_tasks and/or
285 * ->exp_tasks pointers, respectively, to reference the newly
288 if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) {
289 WRITE_ONCE(rnp->gp_tasks, &t->rcu_node_entry);
290 WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq);
292 if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
293 WRITE_ONCE(rnp->exp_tasks, &t->rcu_node_entry);
294 WARN_ON_ONCE(!(blkd_state & RCU_GP_BLKD) !=
295 !(rnp->qsmask & rdp->grpmask));
296 WARN_ON_ONCE(!(blkd_state & RCU_EXP_BLKD) !=
297 !(rnp->expmask & rdp->grpmask));
298 raw_spin_unlock_rcu_node(rnp); /* interrupts remain disabled. */
301 * Report the quiescent state for the expedited GP. This expedited
302 * GP should not be able to end until we report, so there should be
303 * no need to check for a subsequent expedited GP. (Though we are
304 * still in a quiescent state in any case.)
306 if (blkd_state & RCU_EXP_BLKD && rdp->exp_deferred_qs)
307 rcu_report_exp_rdp(rdp);
309 WARN_ON_ONCE(rdp->exp_deferred_qs);
313 * Record a preemptible-RCU quiescent state for the specified CPU.
314 * Note that this does not necessarily mean that the task currently running
315 * on the CPU is in a quiescent state: Instead, it means that the current
316 * grace period need not wait on any RCU read-side critical section that
317 * starts later on this CPU. It also means that if the current task is
318 * in an RCU read-side critical section, it has already added itself to
319 * some leaf rcu_node structure's ->blkd_tasks list. In addition to the
320 * current task, there might be any number of other tasks blocked while
321 * in an RCU read-side critical section.
323 * Callers to this function must disable preemption.
325 static void rcu_qs(void)
327 RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n");
328 if (__this_cpu_read(rcu_data.cpu_no_qs.s)) {
329 trace_rcu_grace_period(TPS("rcu_preempt"),
330 __this_cpu_read(rcu_data.gp_seq),
332 __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false);
333 barrier(); /* Coordinate with rcu_flavor_sched_clock_irq(). */
334 WRITE_ONCE(current->rcu_read_unlock_special.b.need_qs, false);
339 * We have entered the scheduler, and the current task might soon be
340 * context-switched away from. If this task is in an RCU read-side
341 * critical section, we will no longer be able to rely on the CPU to
342 * record that fact, so we enqueue the task on the blkd_tasks list.
343 * The task will dequeue itself when it exits the outermost enclosing
344 * RCU read-side critical section. Therefore, the current grace period
345 * cannot be permitted to complete until the blkd_tasks list entries
346 * predating the current grace period drain, in other words, until
347 * rnp->gp_tasks becomes NULL.
349 * Caller must disable interrupts.
351 void rcu_note_context_switch(bool preempt)
353 struct task_struct *t = current;
354 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
355 struct rcu_node *rnp;
357 trace_rcu_utilization(TPS("Start context switch"));
358 lockdep_assert_irqs_disabled();
359 WARN_ON_ONCE(!preempt && rcu_preempt_depth() > 0);
360 if (rcu_preempt_depth() > 0 &&
361 !t->rcu_read_unlock_special.b.blocked) {
363 /* Possibly blocking in an RCU read-side critical section. */
365 raw_spin_lock_rcu_node(rnp);
366 t->rcu_read_unlock_special.b.blocked = true;
367 t->rcu_blocked_node = rnp;
370 * Verify the CPU's sanity, trace the preemption, and
371 * then queue the task as required based on the states
372 * of any ongoing and expedited grace periods.
374 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);
375 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
376 trace_rcu_preempt_task(rcu_state.name,
378 (rnp->qsmask & rdp->grpmask)
380 : rcu_seq_snap(&rnp->gp_seq));
381 rcu_preempt_ctxt_queue(rnp, rdp);
383 rcu_preempt_deferred_qs(t);
387 * Either we were not in an RCU read-side critical section to
388 * begin with, or we have now recorded that critical section
389 * globally. Either way, we can now note a quiescent state
390 * for this CPU. Again, if we were in an RCU read-side critical
391 * section, and if that critical section was blocking the current
392 * grace period, then the fact that the task has been enqueued
393 * means that we continue to block the current grace period.
396 if (rdp->exp_deferred_qs)
397 rcu_report_exp_rdp(rdp);
398 rcu_tasks_qs(current, preempt);
399 trace_rcu_utilization(TPS("End context switch"));
401 EXPORT_SYMBOL_GPL(rcu_note_context_switch);
404 * Check for preempted RCU readers blocking the current grace period
405 * for the specified rcu_node structure. If the caller needs a reliable
406 * answer, it must hold the rcu_node's ->lock.
408 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
410 return READ_ONCE(rnp->gp_tasks) != NULL;
413 /* limit value for ->rcu_read_lock_nesting. */
414 #define RCU_NEST_PMAX (INT_MAX / 2)
416 static void rcu_preempt_read_enter(void)
418 current->rcu_read_lock_nesting++;
421 static int rcu_preempt_read_exit(void)
423 return --current->rcu_read_lock_nesting;
426 static void rcu_preempt_depth_set(int val)
428 current->rcu_read_lock_nesting = val;
432 * Preemptible RCU implementation for rcu_read_lock().
433 * Just increment ->rcu_read_lock_nesting, shared state will be updated
436 void __rcu_read_lock(void)
438 rcu_preempt_read_enter();
439 if (IS_ENABLED(CONFIG_PROVE_LOCKING))
440 WARN_ON_ONCE(rcu_preempt_depth() > RCU_NEST_PMAX);
441 if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) && rcu_state.gp_kthread)
442 WRITE_ONCE(current->rcu_read_unlock_special.b.need_qs, true);
443 barrier(); /* critical section after entry code. */
445 EXPORT_SYMBOL_GPL(__rcu_read_lock);
448 * Preemptible RCU implementation for rcu_read_unlock().
449 * Decrement ->rcu_read_lock_nesting. If the result is zero (outermost
450 * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
451 * invoke rcu_read_unlock_special() to clean up after a context switch
452 * in an RCU read-side critical section and other special cases.
454 void __rcu_read_unlock(void)
456 struct task_struct *t = current;
458 barrier(); // critical section before exit code.
459 if (rcu_preempt_read_exit() == 0) {
460 barrier(); // critical-section exit before .s check.
461 if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s)))
462 rcu_read_unlock_special(t);
464 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
465 int rrln = rcu_preempt_depth();
467 WARN_ON_ONCE(rrln < 0 || rrln > RCU_NEST_PMAX);
470 EXPORT_SYMBOL_GPL(__rcu_read_unlock);
473 * Advance a ->blkd_tasks-list pointer to the next entry, instead
474 * returning NULL if at the end of the list.
476 static struct list_head *rcu_next_node_entry(struct task_struct *t,
477 struct rcu_node *rnp)
479 struct list_head *np;
481 np = t->rcu_node_entry.next;
482 if (np == &rnp->blkd_tasks)
488 * Return true if the specified rcu_node structure has tasks that were
489 * preempted within an RCU read-side critical section.
491 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
493 return !list_empty(&rnp->blkd_tasks);
497 * Report deferred quiescent states. The deferral time can
498 * be quite short, for example, in the case of the call from
499 * rcu_read_unlock_special().
502 rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags)
507 struct list_head *np;
508 bool drop_boost_mutex = false;
509 struct rcu_data *rdp;
510 struct rcu_node *rnp;
511 union rcu_special special;
514 * If RCU core is waiting for this CPU to exit its critical section,
515 * report the fact that it has exited. Because irqs are disabled,
516 * t->rcu_read_unlock_special cannot change.
518 special = t->rcu_read_unlock_special;
519 rdp = this_cpu_ptr(&rcu_data);
520 if (!special.s && !rdp->exp_deferred_qs) {
521 local_irq_restore(flags);
524 t->rcu_read_unlock_special.s = 0;
525 if (special.b.need_qs) {
526 if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) {
527 rcu_report_qs_rdp(rdp);
528 udelay(rcu_unlock_delay);
535 * Respond to a request by an expedited grace period for a
536 * quiescent state from this CPU. Note that requests from
537 * tasks are handled when removing the task from the
538 * blocked-tasks list below.
540 if (rdp->exp_deferred_qs)
541 rcu_report_exp_rdp(rdp);
543 /* Clean up if blocked during RCU read-side critical section. */
544 if (special.b.blocked) {
547 * Remove this task from the list it blocked on. The task
548 * now remains queued on the rcu_node corresponding to the
549 * CPU it first blocked on, so there is no longer any need
550 * to loop. Retain a WARN_ON_ONCE() out of sheer paranoia.
552 rnp = t->rcu_blocked_node;
553 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
554 WARN_ON_ONCE(rnp != t->rcu_blocked_node);
555 WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
556 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
557 WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq &&
558 (!empty_norm || rnp->qsmask));
559 empty_exp = sync_rcu_exp_done(rnp);
560 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
561 np = rcu_next_node_entry(t, rnp);
562 list_del_init(&t->rcu_node_entry);
563 t->rcu_blocked_node = NULL;
564 trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
565 rnp->gp_seq, t->pid);
566 if (&t->rcu_node_entry == rnp->gp_tasks)
567 WRITE_ONCE(rnp->gp_tasks, np);
568 if (&t->rcu_node_entry == rnp->exp_tasks)
569 WRITE_ONCE(rnp->exp_tasks, np);
570 if (IS_ENABLED(CONFIG_RCU_BOOST)) {
571 /* Snapshot ->boost_mtx ownership w/rnp->lock held. */
572 drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;
573 if (&t->rcu_node_entry == rnp->boost_tasks)
574 WRITE_ONCE(rnp->boost_tasks, np);
578 * If this was the last task on the current list, and if
579 * we aren't waiting on any CPUs, report the quiescent state.
580 * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
581 * so we must take a snapshot of the expedited state.
583 empty_exp_now = sync_rcu_exp_done(rnp);
584 if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {
585 trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
592 rcu_report_unblock_qs_rnp(rnp, flags);
594 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
597 /* Unboost if we were boosted. */
598 if (IS_ENABLED(CONFIG_RCU_BOOST) && drop_boost_mutex)
599 rt_mutex_futex_unlock(&rnp->boost_mtx);
602 * If this was the last task on the expedited lists,
603 * then we need to report up the rcu_node hierarchy.
605 if (!empty_exp && empty_exp_now)
606 rcu_report_exp_rnp(rnp, true);
608 local_irq_restore(flags);
613 * Is a deferred quiescent-state pending, and are we also not in
614 * an RCU read-side critical section? It is the caller's responsibility
615 * to ensure it is otherwise safe to report any deferred quiescent
616 * states. The reason for this is that it is safe to report a
617 * quiescent state during context switch even though preemption
618 * is disabled. This function cannot be expected to understand these
619 * nuances, so the caller must handle them.
621 static bool rcu_preempt_need_deferred_qs(struct task_struct *t)
623 return (__this_cpu_read(rcu_data.exp_deferred_qs) ||
624 READ_ONCE(t->rcu_read_unlock_special.s)) &&
625 rcu_preempt_depth() == 0;
629 * Report a deferred quiescent state if needed and safe to do so.
630 * As with rcu_preempt_need_deferred_qs(), "safe" involves only
631 * not being in an RCU read-side critical section. The caller must
632 * evaluate safety in terms of interrupt, softirq, and preemption
635 static void rcu_preempt_deferred_qs(struct task_struct *t)
639 if (!rcu_preempt_need_deferred_qs(t))
641 local_irq_save(flags);
642 rcu_preempt_deferred_qs_irqrestore(t, flags);
646 * Minimal handler to give the scheduler a chance to re-evaluate.
648 static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp)
650 struct rcu_data *rdp;
652 rdp = container_of(iwp, struct rcu_data, defer_qs_iw);
653 rdp->defer_qs_iw_pending = false;
657 * Handle special cases during rcu_read_unlock(), such as needing to
658 * notify RCU core processing or task having blocked during the RCU
659 * read-side critical section.
661 static void rcu_read_unlock_special(struct task_struct *t)
664 bool irqs_were_disabled;
665 bool preempt_bh_were_disabled =
666 !!(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK));
668 /* NMI handlers cannot block and cannot safely manipulate state. */
672 local_irq_save(flags);
673 irqs_were_disabled = irqs_disabled_flags(flags);
674 if (preempt_bh_were_disabled || irqs_were_disabled) {
675 bool expboost; // Expedited GP in flight or possible boosting.
676 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
677 struct rcu_node *rnp = rdp->mynode;
679 expboost = (t->rcu_blocked_node && READ_ONCE(t->rcu_blocked_node->exp_tasks)) ||
680 (rdp->grpmask & READ_ONCE(rnp->expmask)) ||
681 IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ||
682 (IS_ENABLED(CONFIG_RCU_BOOST) && irqs_were_disabled &&
683 t->rcu_blocked_node);
684 // Need to defer quiescent state until everything is enabled.
685 if (use_softirq && (in_irq() || (expboost && !irqs_were_disabled))) {
686 // Using softirq, safe to awaken, and either the
687 // wakeup is free or there is either an expedited
688 // GP in flight or a potential need to deboost.
689 raise_softirq_irqoff(RCU_SOFTIRQ);
691 // Enabling BH or preempt does reschedule, so...
692 // Also if no expediting and no possible deboosting,
693 // slow is OK. Plus nohz_full CPUs eventually get
695 set_tsk_need_resched(current);
696 set_preempt_need_resched();
697 if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled &&
698 expboost && !rdp->defer_qs_iw_pending && cpu_online(rdp->cpu)) {
699 // Get scheduler to re-evaluate and call hooks.
700 // If !IRQ_WORK, FQS scan will eventually IPI.
701 init_irq_work(&rdp->defer_qs_iw, rcu_preempt_deferred_qs_handler);
702 rdp->defer_qs_iw_pending = true;
703 irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu);
706 local_irq_restore(flags);
709 rcu_preempt_deferred_qs_irqrestore(t, flags);
713 * Check that the list of blocked tasks for the newly completed grace
714 * period is in fact empty. It is a serious bug to complete a grace
715 * period that still has RCU readers blocked! This function must be
716 * invoked -before- updating this rnp's ->gp_seq.
718 * Also, if there are blocked tasks on the list, they automatically
719 * block the newly created grace period, so set up ->gp_tasks accordingly.
721 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
723 struct task_struct *t;
725 RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_check_blocked_tasks() invoked with preemption enabled!!!\n");
726 raw_lockdep_assert_held_rcu_node(rnp);
727 if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
728 dump_blkd_tasks(rnp, 10);
729 if (rcu_preempt_has_tasks(rnp) &&
730 (rnp->qsmaskinit || rnp->wait_blkd_tasks)) {
731 WRITE_ONCE(rnp->gp_tasks, rnp->blkd_tasks.next);
732 t = container_of(rnp->gp_tasks, struct task_struct,
734 trace_rcu_unlock_preempted_task(TPS("rcu_preempt-GPS"),
735 rnp->gp_seq, t->pid);
737 WARN_ON_ONCE(rnp->qsmask);
741 * Check for a quiescent state from the current CPU, including voluntary
742 * context switches for Tasks RCU. When a task blocks, the task is
743 * recorded in the corresponding CPU's rcu_node structure, which is checked
744 * elsewhere, hence this function need only check for quiescent states
745 * related to the current CPU, not to those related to tasks.
747 static void rcu_flavor_sched_clock_irq(int user)
749 struct task_struct *t = current;
751 lockdep_assert_irqs_disabled();
752 if (user || rcu_is_cpu_rrupt_from_idle()) {
753 rcu_note_voluntary_context_switch(current);
755 if (rcu_preempt_depth() > 0 ||
756 (preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK))) {
757 /* No QS, force context switch if deferred. */
758 if (rcu_preempt_need_deferred_qs(t)) {
759 set_tsk_need_resched(t);
760 set_preempt_need_resched();
762 } else if (rcu_preempt_need_deferred_qs(t)) {
763 rcu_preempt_deferred_qs(t); /* Report deferred QS. */
765 } else if (!WARN_ON_ONCE(rcu_preempt_depth())) {
766 rcu_qs(); /* Report immediate QS. */
770 /* If GP is oldish, ask for help from rcu_read_unlock_special(). */
771 if (rcu_preempt_depth() > 0 &&
772 __this_cpu_read(rcu_data.core_needs_qs) &&
773 __this_cpu_read(rcu_data.cpu_no_qs.b.norm) &&
774 !t->rcu_read_unlock_special.b.need_qs &&
775 time_after(jiffies, rcu_state.gp_start + HZ))
776 t->rcu_read_unlock_special.b.need_qs = true;
780 * Check for a task exiting while in a preemptible-RCU read-side
781 * critical section, clean up if so. No need to issue warnings, as
782 * debug_check_no_locks_held() already does this if lockdep is enabled.
783 * Besides, if this function does anything other than just immediately
784 * return, there was a bug of some sort. Spewing warnings from this
785 * function is like as not to simply obscure important prior warnings.
789 struct task_struct *t = current;
791 if (unlikely(!list_empty(¤t->rcu_node_entry))) {
792 rcu_preempt_depth_set(1);
794 WRITE_ONCE(t->rcu_read_unlock_special.b.blocked, true);
795 } else if (unlikely(rcu_preempt_depth())) {
796 rcu_preempt_depth_set(1);
801 rcu_preempt_deferred_qs(current);
805 * Dump the blocked-tasks state, but limit the list dump to the
806 * specified number of elements.
809 dump_blkd_tasks(struct rcu_node *rnp, int ncheck)
813 struct list_head *lhp;
815 struct rcu_data *rdp;
816 struct rcu_node *rnp1;
818 raw_lockdep_assert_held_rcu_node(rnp);
819 pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
820 __func__, rnp->grplo, rnp->grphi, rnp->level,
821 (long)READ_ONCE(rnp->gp_seq), (long)rnp->completedqs);
822 for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)
823 pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx\n",
824 __func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext);
825 pr_info("%s: ->gp_tasks %p ->boost_tasks %p ->exp_tasks %p\n",
826 __func__, READ_ONCE(rnp->gp_tasks), data_race(rnp->boost_tasks),
827 READ_ONCE(rnp->exp_tasks));
828 pr_info("%s: ->blkd_tasks", __func__);
830 list_for_each(lhp, &rnp->blkd_tasks) {
836 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++) {
837 rdp = per_cpu_ptr(&rcu_data, cpu);
838 onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp));
839 pr_info("\t%d: %c online: %ld(%d) offline: %ld(%d)\n",
841 (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags,
842 (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags);
846 #else /* #ifdef CONFIG_PREEMPT_RCU */
849 * If strict grace periods are enabled, and if the calling
850 * __rcu_read_unlock() marks the beginning of a quiescent state, immediately
851 * report that quiescent state and, if requested, spin for a bit.
853 void rcu_read_unlock_strict(void)
855 struct rcu_data *rdp;
857 if (!IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ||
858 irqs_disabled() || preempt_count() || !rcu_state.gp_kthread)
860 rdp = this_cpu_ptr(&rcu_data);
861 rcu_report_qs_rdp(rdp);
862 udelay(rcu_unlock_delay);
864 EXPORT_SYMBOL_GPL(rcu_read_unlock_strict);
867 * Tell them what RCU they are running.
869 static void __init rcu_bootup_announce(void)
871 pr_info("Hierarchical RCU implementation.\n");
872 rcu_bootup_announce_oddness();
876 * Note a quiescent state for PREEMPTION=n. Because we do not need to know
877 * how many quiescent states passed, just if there was at least one since
878 * the start of the grace period, this just sets a flag. The caller must
879 * have disabled preemption.
881 static void rcu_qs(void)
883 RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!");
884 if (!__this_cpu_read(rcu_data.cpu_no_qs.s))
886 trace_rcu_grace_period(TPS("rcu_sched"),
887 __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs"));
888 __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false);
889 if (!__this_cpu_read(rcu_data.cpu_no_qs.b.exp))
891 __this_cpu_write(rcu_data.cpu_no_qs.b.exp, false);
892 rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
896 * Register an urgently needed quiescent state. If there is an
897 * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
898 * dyntick-idle quiescent state visible to other CPUs, which will in
899 * some cases serve for expedited as well as normal grace periods.
900 * Either way, register a lightweight quiescent state.
902 void rcu_all_qs(void)
906 if (!raw_cpu_read(rcu_data.rcu_urgent_qs))
909 /* Load rcu_urgent_qs before other flags. */
910 if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
914 this_cpu_write(rcu_data.rcu_urgent_qs, false);
915 if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) {
916 local_irq_save(flags);
917 rcu_momentary_dyntick_idle();
918 local_irq_restore(flags);
923 EXPORT_SYMBOL_GPL(rcu_all_qs);
926 * Note a PREEMPTION=n context switch. The caller must have disabled interrupts.
928 void rcu_note_context_switch(bool preempt)
930 trace_rcu_utilization(TPS("Start context switch"));
932 /* Load rcu_urgent_qs before other flags. */
933 if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs)))
935 this_cpu_write(rcu_data.rcu_urgent_qs, false);
936 if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs)))
937 rcu_momentary_dyntick_idle();
938 rcu_tasks_qs(current, preempt);
940 trace_rcu_utilization(TPS("End context switch"));
942 EXPORT_SYMBOL_GPL(rcu_note_context_switch);
945 * Because preemptible RCU does not exist, there are never any preempted
948 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
954 * Because there is no preemptible RCU, there can be no readers blocked.
956 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
962 * Because there is no preemptible RCU, there can be no deferred quiescent
965 static bool rcu_preempt_need_deferred_qs(struct task_struct *t)
969 static void rcu_preempt_deferred_qs(struct task_struct *t) { }
972 * Because there is no preemptible RCU, there can be no readers blocked,
973 * so there is no need to check for blocked tasks. So check only for
974 * bogus qsmask values.
976 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
978 WARN_ON_ONCE(rnp->qsmask);
982 * Check to see if this CPU is in a non-context-switch quiescent state,
983 * namely user mode and idle loop.
985 static void rcu_flavor_sched_clock_irq(int user)
987 if (user || rcu_is_cpu_rrupt_from_idle()) {
990 * Get here if this CPU took its interrupt from user
991 * mode or from the idle loop, and if this is not a
992 * nested interrupt. In this case, the CPU is in
993 * a quiescent state, so note it.
995 * No memory barrier is required here because rcu_qs()
996 * references only CPU-local variables that other CPUs
997 * neither access nor modify, at least not while the
998 * corresponding CPU is online.
1006 * Because preemptible RCU does not exist, tasks cannot possibly exit
1007 * while in preemptible RCU read-side critical sections.
1014 * Dump the guaranteed-empty blocked-tasks state. Trust but verify.
1017 dump_blkd_tasks(struct rcu_node *rnp, int ncheck)
1019 WARN_ON_ONCE(!list_empty(&rnp->blkd_tasks));
1022 #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
1025 * If boosting, set rcuc kthreads to realtime priority.
1027 static void rcu_cpu_kthread_setup(unsigned int cpu)
1029 #ifdef CONFIG_RCU_BOOST
1030 struct sched_param sp;
1032 sp.sched_priority = kthread_prio;
1033 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1034 #endif /* #ifdef CONFIG_RCU_BOOST */
1037 #ifdef CONFIG_RCU_BOOST
1040 * Carry out RCU priority boosting on the task indicated by ->exp_tasks
1041 * or ->boost_tasks, advancing the pointer to the next task in the
1042 * ->blkd_tasks list.
1044 * Note that irqs must be enabled: boosting the task can block.
1045 * Returns 1 if there are more tasks needing to be boosted.
1047 static int rcu_boost(struct rcu_node *rnp)
1049 unsigned long flags;
1050 struct task_struct *t;
1051 struct list_head *tb;
1053 if (READ_ONCE(rnp->exp_tasks) == NULL &&
1054 READ_ONCE(rnp->boost_tasks) == NULL)
1055 return 0; /* Nothing left to boost. */
1057 raw_spin_lock_irqsave_rcu_node(rnp, flags);
1060 * Recheck under the lock: all tasks in need of boosting
1061 * might exit their RCU read-side critical sections on their own.
1063 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
1064 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1069 * Preferentially boost tasks blocking expedited grace periods.
1070 * This cannot starve the normal grace periods because a second
1071 * expedited grace period must boost all blocked tasks, including
1072 * those blocking the pre-existing normal grace period.
1074 if (rnp->exp_tasks != NULL)
1075 tb = rnp->exp_tasks;
1077 tb = rnp->boost_tasks;
1080 * We boost task t by manufacturing an rt_mutex that appears to
1081 * be held by task t. We leave a pointer to that rt_mutex where
1082 * task t can find it, and task t will release the mutex when it
1083 * exits its outermost RCU read-side critical section. Then
1084 * simply acquiring this artificial rt_mutex will boost task
1085 * t's priority. (Thanks to tglx for suggesting this approach!)
1087 * Note that task t must acquire rnp->lock to remove itself from
1088 * the ->blkd_tasks list, which it will do from exit() if from
1089 * nowhere else. We therefore are guaranteed that task t will
1090 * stay around at least until we drop rnp->lock. Note that
1091 * rnp->lock also resolves races between our priority boosting
1092 * and task t's exiting its outermost RCU read-side critical
1095 t = container_of(tb, struct task_struct, rcu_node_entry);
1096 rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
1097 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1098 /* Lock only for side effect: boosts task t's priority. */
1099 rt_mutex_lock(&rnp->boost_mtx);
1100 rt_mutex_unlock(&rnp->boost_mtx); /* Then keep lockdep happy. */
1102 return READ_ONCE(rnp->exp_tasks) != NULL ||
1103 READ_ONCE(rnp->boost_tasks) != NULL;
1107 * Priority-boosting kthread, one per leaf rcu_node.
1109 static int rcu_boost_kthread(void *arg)
1111 struct rcu_node *rnp = (struct rcu_node *)arg;
1115 trace_rcu_utilization(TPS("Start boost kthread@init"));
1117 WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_WAITING);
1118 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
1119 rcu_wait(READ_ONCE(rnp->boost_tasks) ||
1120 READ_ONCE(rnp->exp_tasks));
1121 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
1122 WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_RUNNING);
1123 more2boost = rcu_boost(rnp);
1129 WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_YIELDING);
1130 trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
1131 schedule_timeout_idle(2);
1132 trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
1137 trace_rcu_utilization(TPS("End boost kthread@notreached"));
1142 * Check to see if it is time to start boosting RCU readers that are
1143 * blocking the current grace period, and, if so, tell the per-rcu_node
1144 * kthread to start boosting them. If there is an expedited grace
1145 * period in progress, it is always time to boost.
1147 * The caller must hold rnp->lock, which this function releases.
1148 * The ->boost_kthread_task is immortal, so we don't need to worry
1149 * about it going away.
1151 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1152 __releases(rnp->lock)
1154 raw_lockdep_assert_held_rcu_node(rnp);
1155 if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
1156 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1159 if (rnp->exp_tasks != NULL ||
1160 (rnp->gp_tasks != NULL &&
1161 rnp->boost_tasks == NULL &&
1163 (!time_after(rnp->boost_time, jiffies) || rcu_state.cbovld))) {
1164 if (rnp->exp_tasks == NULL)
1165 WRITE_ONCE(rnp->boost_tasks, rnp->gp_tasks);
1166 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1167 rcu_wake_cond(rnp->boost_kthread_task,
1168 READ_ONCE(rnp->boost_kthread_status));
1170 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1175 * Is the current CPU running the RCU-callbacks kthread?
1176 * Caller must have preemption disabled.
1178 static bool rcu_is_callbacks_kthread(void)
1180 return __this_cpu_read(rcu_data.rcu_cpu_kthread_task) == current;
1183 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1186 * Do priority-boost accounting for the start of a new grace period.
1188 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1190 rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1194 * Create an RCU-boost kthread for the specified node if one does not
1195 * already exist. We only create this kthread for preemptible RCU.
1196 * Returns zero if all is well, a negated errno otherwise.
1198 static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)
1200 int rnp_index = rnp - rcu_get_root();
1201 unsigned long flags;
1202 struct sched_param sp;
1203 struct task_struct *t;
1205 if (!IS_ENABLED(CONFIG_PREEMPT_RCU))
1208 if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)
1211 rcu_state.boost = 1;
1213 if (rnp->boost_kthread_task != NULL)
1216 t = kthread_create(rcu_boost_kthread, (void *)rnp,
1217 "rcub/%d", rnp_index);
1218 if (WARN_ON_ONCE(IS_ERR(t)))
1221 raw_spin_lock_irqsave_rcu_node(rnp, flags);
1222 rnp->boost_kthread_task = t;
1223 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1224 sp.sched_priority = kthread_prio;
1225 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1226 wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
1230 * Set the per-rcu_node kthread's affinity to cover all CPUs that are
1231 * served by the rcu_node in question. The CPU hotplug lock is still
1232 * held, so the value of rnp->qsmaskinit will be stable.
1234 * We don't include outgoingcpu in the affinity set, use -1 if there is
1235 * no outgoing CPU. If there are no CPUs left in the affinity set,
1236 * this function allows the kthread to execute on any CPU.
1238 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1240 struct task_struct *t = rnp->boost_kthread_task;
1241 unsigned long mask = rcu_rnp_online_cpus(rnp);
1247 if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
1249 for_each_leaf_node_possible_cpu(rnp, cpu)
1250 if ((mask & leaf_node_cpu_bit(rnp, cpu)) &&
1252 cpumask_set_cpu(cpu, cm);
1253 if (cpumask_weight(cm) == 0)
1255 set_cpus_allowed_ptr(t, cm);
1256 free_cpumask_var(cm);
1260 * Spawn boost kthreads -- called as soon as the scheduler is running.
1262 static void __init rcu_spawn_boost_kthreads(void)
1264 struct rcu_node *rnp;
1266 rcu_for_each_leaf_node(rnp)
1267 rcu_spawn_one_boost_kthread(rnp);
1270 static void rcu_prepare_kthreads(int cpu)
1272 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1273 struct rcu_node *rnp = rdp->mynode;
1275 /* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
1276 if (rcu_scheduler_fully_active)
1277 rcu_spawn_one_boost_kthread(rnp);
1280 #else /* #ifdef CONFIG_RCU_BOOST */
1282 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1283 __releases(rnp->lock)
1285 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1288 static bool rcu_is_callbacks_kthread(void)
1293 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1297 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1301 static void __init rcu_spawn_boost_kthreads(void)
1305 static void rcu_prepare_kthreads(int cpu)
1309 #endif /* #else #ifdef CONFIG_RCU_BOOST */
1311 #if !defined(CONFIG_RCU_FAST_NO_HZ)
1314 * Check to see if any future non-offloaded RCU-related work will need
1315 * to be done by the current CPU, even if none need be done immediately,
1316 * returning 1 if so. This function is part of the RCU implementation;
1317 * it is -not- an exported member of the RCU API.
1319 * Because we not have RCU_FAST_NO_HZ, just check whether or not this
1320 * CPU has RCU callbacks queued.
1322 int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1324 *nextevt = KTIME_MAX;
1325 return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) &&
1326 !rcu_rdp_is_offloaded(this_cpu_ptr(&rcu_data));
1330 * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
1333 static void rcu_cleanup_after_idle(void)
1338 * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
1341 static void rcu_prepare_for_idle(void)
1345 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1348 * This code is invoked when a CPU goes idle, at which point we want
1349 * to have the CPU do everything required for RCU so that it can enter
1350 * the energy-efficient dyntick-idle mode.
1352 * The following preprocessor symbol controls this:
1354 * RCU_IDLE_GP_DELAY gives the number of jiffies that a CPU is permitted
1355 * to sleep in dyntick-idle mode with RCU callbacks pending. This
1356 * is sized to be roughly one RCU grace period. Those energy-efficiency
1357 * benchmarkers who might otherwise be tempted to set this to a large
1358 * number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
1359 * system. And if you are -that- concerned about energy efficiency,
1360 * just power the system down and be done with it!
1362 * The value below works well in practice. If future workloads require
1363 * adjustment, they can be converted into kernel config parameters, though
1364 * making the state machine smarter might be a better option.
1366 #define RCU_IDLE_GP_DELAY 4 /* Roughly one grace period. */
1368 static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
1369 module_param(rcu_idle_gp_delay, int, 0644);
1372 * Try to advance callbacks on the current CPU, but only if it has been
1373 * awhile since the last time we did so. Afterwards, if there are any
1374 * callbacks ready for immediate invocation, return true.
1376 static bool __maybe_unused rcu_try_advance_all_cbs(void)
1378 bool cbs_ready = false;
1379 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1380 struct rcu_node *rnp;
1382 /* Exit early if we advanced recently. */
1383 if (jiffies == rdp->last_advance_all)
1385 rdp->last_advance_all = jiffies;
1390 * Don't bother checking unless a grace period has
1391 * completed since we last checked and there are
1392 * callbacks not yet ready to invoke.
1394 if ((rcu_seq_completed_gp(rdp->gp_seq,
1395 rcu_seq_current(&rnp->gp_seq)) ||
1396 unlikely(READ_ONCE(rdp->gpwrap))) &&
1397 rcu_segcblist_pend_cbs(&rdp->cblist))
1398 note_gp_changes(rdp);
1400 if (rcu_segcblist_ready_cbs(&rdp->cblist))
1406 * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
1407 * to invoke. If the CPU has callbacks, try to advance them. Tell the
1408 * caller about what to set the timeout.
1410 * The caller must have disabled interrupts.
1412 int rcu_needs_cpu(u64 basemono, u64 *nextevt)
1414 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1417 lockdep_assert_irqs_disabled();
1419 /* If no non-offloaded callbacks, RCU doesn't need the CPU. */
1420 if (rcu_segcblist_empty(&rdp->cblist) ||
1421 rcu_rdp_is_offloaded(rdp)) {
1422 *nextevt = KTIME_MAX;
1426 /* Attempt to advance callbacks. */
1427 if (rcu_try_advance_all_cbs()) {
1428 /* Some ready to invoke, so initiate later invocation. */
1432 rdp->last_accelerate = jiffies;
1434 /* Request timer and round. */
1435 dj = round_up(rcu_idle_gp_delay + jiffies, rcu_idle_gp_delay) - jiffies;
1437 *nextevt = basemono + dj * TICK_NSEC;
1442 * Prepare a CPU for idle from an RCU perspective. The first major task is to
1443 * sense whether nohz mode has been enabled or disabled via sysfs. The second
1444 * major task is to accelerate (that is, assign grace-period numbers to) any
1445 * recently arrived callbacks.
1447 * The caller must have disabled interrupts.
1449 static void rcu_prepare_for_idle(void)
1452 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1453 struct rcu_node *rnp;
1456 lockdep_assert_irqs_disabled();
1457 if (rcu_rdp_is_offloaded(rdp))
1460 /* Handle nohz enablement switches conservatively. */
1461 tne = READ_ONCE(tick_nohz_active);
1462 if (tne != rdp->tick_nohz_enabled_snap) {
1463 if (!rcu_segcblist_empty(&rdp->cblist))
1464 invoke_rcu_core(); /* force nohz to see update. */
1465 rdp->tick_nohz_enabled_snap = tne;
1472 * If we have not yet accelerated this jiffy, accelerate all
1473 * callbacks on this CPU.
1475 if (rdp->last_accelerate == jiffies)
1477 rdp->last_accelerate = jiffies;
1478 if (rcu_segcblist_pend_cbs(&rdp->cblist)) {
1480 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
1481 needwake = rcu_accelerate_cbs(rnp, rdp);
1482 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
1484 rcu_gp_kthread_wake();
1489 * Clean up for exit from idle. Attempt to advance callbacks based on
1490 * any grace periods that elapsed while the CPU was idle, and if any
1491 * callbacks are now ready to invoke, initiate invocation.
1493 static void rcu_cleanup_after_idle(void)
1495 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1497 lockdep_assert_irqs_disabled();
1498 if (rcu_rdp_is_offloaded(rdp))
1500 if (rcu_try_advance_all_cbs())
1504 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1506 #ifdef CONFIG_RCU_NOCB_CPU
1509 * Offload callback processing from the boot-time-specified set of CPUs
1510 * specified by rcu_nocb_mask. For the CPUs in the set, there are kthreads
1511 * created that pull the callbacks from the corresponding CPU, wait for
1512 * a grace period to elapse, and invoke the callbacks. These kthreads
1513 * are organized into GP kthreads, which manage incoming callbacks, wait for
1514 * grace periods, and awaken CB kthreads, and the CB kthreads, which only
1515 * invoke callbacks. Each GP kthread invokes its own CBs. The no-CBs CPUs
1516 * do a wake_up() on their GP kthread when they insert a callback into any
1517 * empty list, unless the rcu_nocb_poll boot parameter has been specified,
1518 * in which case each kthread actively polls its CPU. (Which isn't so great
1519 * for energy efficiency, but which does reduce RCU's overhead on that CPU.)
1521 * This is intended to be used in conjunction with Frederic Weisbecker's
1522 * adaptive-idle work, which would seriously reduce OS jitter on CPUs
1523 * running CPU-bound user-mode computations.
1525 * Offloading of callbacks can also be used as an energy-efficiency
1526 * measure because CPUs with no RCU callbacks queued are more aggressive
1527 * about entering dyntick-idle mode.
1532 * Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters.
1533 * If the list is invalid, a warning is emitted and all CPUs are offloaded.
1535 static int __init rcu_nocb_setup(char *str)
1537 alloc_bootmem_cpumask_var(&rcu_nocb_mask);
1538 if (!strcasecmp(str, "all")) /* legacy: use "0-N" instead */
1539 cpumask_setall(rcu_nocb_mask);
1541 if (cpulist_parse(str, rcu_nocb_mask)) {
1542 pr_warn("rcu_nocbs= bad CPU range, all CPUs set\n");
1543 cpumask_setall(rcu_nocb_mask);
1547 __setup("rcu_nocbs=", rcu_nocb_setup);
1549 static int __init parse_rcu_nocb_poll(char *arg)
1551 rcu_nocb_poll = true;
1554 early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
1557 * Don't bother bypassing ->cblist if the call_rcu() rate is low.
1558 * After all, the main point of bypassing is to avoid lock contention
1559 * on ->nocb_lock, which only can happen at high call_rcu() rates.
1561 static int nocb_nobypass_lim_per_jiffy = 16 * 1000 / HZ;
1562 module_param(nocb_nobypass_lim_per_jiffy, int, 0);
1565 * Acquire the specified rcu_data structure's ->nocb_bypass_lock. If the
1566 * lock isn't immediately available, increment ->nocb_lock_contended to
1567 * flag the contention.
1569 static void rcu_nocb_bypass_lock(struct rcu_data *rdp)
1570 __acquires(&rdp->nocb_bypass_lock)
1572 lockdep_assert_irqs_disabled();
1573 if (raw_spin_trylock(&rdp->nocb_bypass_lock))
1575 atomic_inc(&rdp->nocb_lock_contended);
1576 WARN_ON_ONCE(smp_processor_id() != rdp->cpu);
1577 smp_mb__after_atomic(); /* atomic_inc() before lock. */
1578 raw_spin_lock(&rdp->nocb_bypass_lock);
1579 smp_mb__before_atomic(); /* atomic_dec() after lock. */
1580 atomic_dec(&rdp->nocb_lock_contended);
1584 * Spinwait until the specified rcu_data structure's ->nocb_lock is
1585 * not contended. Please note that this is extremely special-purpose,
1586 * relying on the fact that at most two kthreads and one CPU contend for
1587 * this lock, and also that the two kthreads are guaranteed to have frequent
1588 * grace-period-duration time intervals between successive acquisitions
1589 * of the lock. This allows us to use an extremely simple throttling
1590 * mechanism, and further to apply it only to the CPU doing floods of
1591 * call_rcu() invocations. Don't try this at home!
1593 static void rcu_nocb_wait_contended(struct rcu_data *rdp)
1595 WARN_ON_ONCE(smp_processor_id() != rdp->cpu);
1596 while (WARN_ON_ONCE(atomic_read(&rdp->nocb_lock_contended)))
1601 * Conditionally acquire the specified rcu_data structure's
1602 * ->nocb_bypass_lock.
1604 static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp)
1606 lockdep_assert_irqs_disabled();
1607 return raw_spin_trylock(&rdp->nocb_bypass_lock);
1611 * Release the specified rcu_data structure's ->nocb_bypass_lock.
1613 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)
1614 __releases(&rdp->nocb_bypass_lock)
1616 lockdep_assert_irqs_disabled();
1617 raw_spin_unlock(&rdp->nocb_bypass_lock);
1621 * Acquire the specified rcu_data structure's ->nocb_lock, but only
1622 * if it corresponds to a no-CBs CPU.
1624 static void rcu_nocb_lock(struct rcu_data *rdp)
1626 lockdep_assert_irqs_disabled();
1627 if (!rcu_rdp_is_offloaded(rdp))
1629 raw_spin_lock(&rdp->nocb_lock);
1633 * Release the specified rcu_data structure's ->nocb_lock, but only
1634 * if it corresponds to a no-CBs CPU.
1636 static void rcu_nocb_unlock(struct rcu_data *rdp)
1638 if (rcu_rdp_is_offloaded(rdp)) {
1639 lockdep_assert_irqs_disabled();
1640 raw_spin_unlock(&rdp->nocb_lock);
1645 * Release the specified rcu_data structure's ->nocb_lock and restore
1646 * interrupts, but only if it corresponds to a no-CBs CPU.
1648 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
1649 unsigned long flags)
1651 if (rcu_rdp_is_offloaded(rdp)) {
1652 lockdep_assert_irqs_disabled();
1653 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
1655 local_irq_restore(flags);
1659 /* Lockdep check that ->cblist may be safely accessed. */
1660 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
1662 lockdep_assert_irqs_disabled();
1663 if (rcu_rdp_is_offloaded(rdp))
1664 lockdep_assert_held(&rdp->nocb_lock);
1668 * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
1671 static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
1676 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
1678 return &rnp->nocb_gp_wq[rcu_seq_ctr(rnp->gp_seq) & 0x1];
1681 static void rcu_init_one_nocb(struct rcu_node *rnp)
1683 init_swait_queue_head(&rnp->nocb_gp_wq[0]);
1684 init_swait_queue_head(&rnp->nocb_gp_wq[1]);
1687 /* Is the specified CPU a no-CBs CPU? */
1688 bool rcu_is_nocb_cpu(int cpu)
1690 if (cpumask_available(rcu_nocb_mask))
1691 return cpumask_test_cpu(cpu, rcu_nocb_mask);
1696 * Kick the GP kthread for this NOCB group. Caller holds ->nocb_lock
1697 * and this function releases it.
1699 static bool wake_nocb_gp(struct rcu_data *rdp, bool force,
1700 unsigned long flags)
1701 __releases(rdp->nocb_lock)
1703 bool needwake = false;
1704 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1706 lockdep_assert_held(&rdp->nocb_lock);
1707 if (!READ_ONCE(rdp_gp->nocb_gp_kthread)) {
1708 rcu_nocb_unlock_irqrestore(rdp, flags);
1709 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
1710 TPS("AlreadyAwake"));
1714 if (READ_ONCE(rdp->nocb_defer_wakeup) > RCU_NOCB_WAKE_NOT) {
1715 WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);
1716 del_timer(&rdp->nocb_timer);
1718 rcu_nocb_unlock_irqrestore(rdp, flags);
1719 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
1720 if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) {
1721 WRITE_ONCE(rdp_gp->nocb_gp_sleep, false);
1723 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake"));
1725 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
1727 wake_up_process(rdp_gp->nocb_gp_kthread);
1733 * Arrange to wake the GP kthread for this NOCB group at some future
1734 * time when it is safe to do so.
1736 static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,
1739 if (rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_OFF)
1741 if (rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT)
1742 mod_timer(&rdp->nocb_timer, jiffies + 1);
1743 if (rdp->nocb_defer_wakeup < waketype)
1744 WRITE_ONCE(rdp->nocb_defer_wakeup, waketype);
1745 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason);
1749 * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
1750 * However, if there is a callback to be enqueued and if ->nocb_bypass
1751 * proves to be initially empty, just return false because the no-CB GP
1752 * kthread may need to be awakened in this case.
1754 * Note that this function always returns true if rhp is NULL.
1756 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
1759 struct rcu_cblist rcl;
1761 WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp));
1762 rcu_lockdep_assert_cblist_protected(rdp);
1763 lockdep_assert_held(&rdp->nocb_bypass_lock);
1764 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) {
1765 raw_spin_unlock(&rdp->nocb_bypass_lock);
1768 /* Note: ->cblist.len already accounts for ->nocb_bypass contents. */
1770 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */
1771 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp);
1772 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl);
1773 WRITE_ONCE(rdp->nocb_bypass_first, j);
1774 rcu_nocb_bypass_unlock(rdp);
1779 * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
1780 * However, if there is a callback to be enqueued and if ->nocb_bypass
1781 * proves to be initially empty, just return false because the no-CB GP
1782 * kthread may need to be awakened in this case.
1784 * Note that this function always returns true if rhp is NULL.
1786 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
1789 if (!rcu_rdp_is_offloaded(rdp))
1791 rcu_lockdep_assert_cblist_protected(rdp);
1792 rcu_nocb_bypass_lock(rdp);
1793 return rcu_nocb_do_flush_bypass(rdp, rhp, j);
1797 * If the ->nocb_bypass_lock is immediately available, flush the
1798 * ->nocb_bypass queue into ->cblist.
1800 static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)
1802 rcu_lockdep_assert_cblist_protected(rdp);
1803 if (!rcu_rdp_is_offloaded(rdp) ||
1804 !rcu_nocb_bypass_trylock(rdp))
1806 WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j));
1810 * See whether it is appropriate to use the ->nocb_bypass list in order
1811 * to control contention on ->nocb_lock. A limited number of direct
1812 * enqueues are permitted into ->cblist per jiffy. If ->nocb_bypass
1813 * is non-empty, further callbacks must be placed into ->nocb_bypass,
1814 * otherwise rcu_barrier() breaks. Use rcu_nocb_flush_bypass() to switch
1815 * back to direct use of ->cblist. However, ->nocb_bypass should not be
1816 * used if ->cblist is empty, because otherwise callbacks can be stranded
1817 * on ->nocb_bypass because we cannot count on the current CPU ever again
1818 * invoking call_rcu(). The general rule is that if ->nocb_bypass is
1819 * non-empty, the corresponding no-CBs grace-period kthread must not be
1820 * in an indefinite sleep state.
1822 * Finally, it is not permitted to use the bypass during early boot,
1823 * as doing so would confuse the auto-initialization code. Besides
1824 * which, there is no point in worrying about lock contention while
1825 * there is only one CPU in operation.
1827 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
1828 bool *was_alldone, unsigned long flags)
1831 unsigned long cur_gp_seq;
1832 unsigned long j = jiffies;
1833 long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
1835 lockdep_assert_irqs_disabled();
1837 // Pure softirq/rcuc based processing: no bypassing, no
1839 if (!rcu_rdp_is_offloaded(rdp)) {
1840 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
1844 // In the process of (de-)offloading: no bypassing, but
1846 if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) {
1848 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
1849 return false; /* Not offloaded, no bypassing. */
1852 // Don't use ->nocb_bypass during early boot.
1853 if (rcu_scheduler_active != RCU_SCHEDULER_RUNNING) {
1855 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
1856 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
1860 // If we have advanced to a new jiffy, reset counts to allow
1861 // moving back from ->nocb_bypass to ->cblist.
1862 if (j == rdp->nocb_nobypass_last) {
1863 c = rdp->nocb_nobypass_count + 1;
1865 WRITE_ONCE(rdp->nocb_nobypass_last, j);
1866 c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy;
1867 if (ULONG_CMP_LT(rdp->nocb_nobypass_count,
1868 nocb_nobypass_lim_per_jiffy))
1870 else if (c > nocb_nobypass_lim_per_jiffy)
1871 c = nocb_nobypass_lim_per_jiffy;
1873 WRITE_ONCE(rdp->nocb_nobypass_count, c);
1875 // If there hasn't yet been all that many ->cblist enqueues
1876 // this jiffy, tell the caller to enqueue onto ->cblist. But flush
1877 // ->nocb_bypass first.
1878 if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) {
1880 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
1882 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
1884 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j));
1885 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
1886 return false; // Caller must enqueue the callback.
1889 // If ->nocb_bypass has been used too long or is too full,
1890 // flush ->nocb_bypass to ->cblist.
1891 if ((ncbs && j != READ_ONCE(rdp->nocb_bypass_first)) ||
1894 if (!rcu_nocb_flush_bypass(rdp, rhp, j)) {
1895 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
1897 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
1899 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
1900 return false; // Caller must enqueue the callback.
1902 if (j != rdp->nocb_gp_adv_time &&
1903 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
1904 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {
1905 rcu_advance_cbs_nowake(rdp->mynode, rdp);
1906 rdp->nocb_gp_adv_time = j;
1908 rcu_nocb_unlock_irqrestore(rdp, flags);
1909 return true; // Callback already enqueued.
1912 // We need to use the bypass.
1913 rcu_nocb_wait_contended(rdp);
1914 rcu_nocb_bypass_lock(rdp);
1915 ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
1916 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */
1917 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);
1919 WRITE_ONCE(rdp->nocb_bypass_first, j);
1920 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ"));
1922 rcu_nocb_bypass_unlock(rdp);
1923 smp_mb(); /* Order enqueue before wake. */
1925 local_irq_restore(flags);
1927 // No-CBs GP kthread might be indefinitely asleep, if so, wake.
1928 rcu_nocb_lock(rdp); // Rare during call_rcu() flood.
1929 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) {
1930 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
1931 TPS("FirstBQwake"));
1932 __call_rcu_nocb_wake(rdp, true, flags);
1934 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
1935 TPS("FirstBQnoWake"));
1936 rcu_nocb_unlock_irqrestore(rdp, flags);
1939 return true; // Callback already enqueued.
1943 * Awaken the no-CBs grace-period kthead if needed, either due to it
1944 * legitimately being asleep or due to overload conditions.
1946 * If warranted, also wake up the kthread servicing this CPUs queues.
1948 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,
1949 unsigned long flags)
1950 __releases(rdp->nocb_lock)
1952 unsigned long cur_gp_seq;
1955 struct task_struct *t;
1957 // If we are being polled or there is no kthread, just leave.
1958 t = READ_ONCE(rdp->nocb_gp_kthread);
1959 if (rcu_nocb_poll || !t) {
1960 rcu_nocb_unlock_irqrestore(rdp, flags);
1961 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
1962 TPS("WakeNotPoll"));
1965 // Need to actually to a wakeup.
1966 len = rcu_segcblist_n_cbs(&rdp->cblist);
1968 rdp->qlen_last_fqs_check = len;
1969 if (!irqs_disabled_flags(flags)) {
1970 /* ... if queue was empty ... */
1971 wake_nocb_gp(rdp, false, flags);
1972 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
1975 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE,
1976 TPS("WakeEmptyIsDeferred"));
1977 rcu_nocb_unlock_irqrestore(rdp, flags);
1979 } else if (len > rdp->qlen_last_fqs_check + qhimark) {
1980 /* ... or if many callbacks queued. */
1981 rdp->qlen_last_fqs_check = len;
1983 if (j != rdp->nocb_gp_adv_time &&
1984 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
1985 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {
1986 rcu_advance_cbs_nowake(rdp->mynode, rdp);
1987 rdp->nocb_gp_adv_time = j;
1989 smp_mb(); /* Enqueue before timer_pending(). */
1990 if ((rdp->nocb_cb_sleep ||
1991 !rcu_segcblist_ready_cbs(&rdp->cblist)) &&
1992 !timer_pending(&rdp->nocb_bypass_timer))
1993 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,
1994 TPS("WakeOvfIsDeferred"));
1995 rcu_nocb_unlock_irqrestore(rdp, flags);
1997 rcu_nocb_unlock_irqrestore(rdp, flags);
1998 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));
2003 /* Wake up the no-CBs GP kthread to flush ->nocb_bypass. */
2004 static void do_nocb_bypass_wakeup_timer(struct timer_list *t)
2006 unsigned long flags;
2007 struct rcu_data *rdp = from_timer(rdp, t, nocb_bypass_timer);
2009 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer"));
2010 rcu_nocb_lock_irqsave(rdp, flags);
2011 smp_mb__after_spinlock(); /* Timer expire before wakeup. */
2012 __call_rcu_nocb_wake(rdp, true, flags);
2016 * Check if we ignore this rdp.
2018 * We check that without holding the nocb lock but
2019 * we make sure not to miss a freshly offloaded rdp
2020 * with the current ordering:
2022 * rdp_offload_toggle() nocb_gp_enabled_cb()
2023 * ------------------------- ----------------------------
2024 * WRITE flags LOCK nocb_gp_lock
2025 * LOCK nocb_gp_lock READ/WRITE nocb_gp_sleep
2026 * READ/WRITE nocb_gp_sleep UNLOCK nocb_gp_lock
2027 * UNLOCK nocb_gp_lock READ flags
2029 static inline bool nocb_gp_enabled_cb(struct rcu_data *rdp)
2031 u8 flags = SEGCBLIST_OFFLOADED | SEGCBLIST_KTHREAD_GP;
2033 return rcu_segcblist_test_flags(&rdp->cblist, flags);
2036 static inline bool nocb_gp_update_state_deoffloading(struct rcu_data *rdp,
2037 bool *needwake_state)
2039 struct rcu_segcblist *cblist = &rdp->cblist;
2041 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) {
2042 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) {
2043 rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_GP);
2044 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB))
2045 *needwake_state = true;
2051 * De-offloading. Clear our flag and notify the de-offload worker.
2052 * We will ignore this rdp until it ever gets re-offloaded.
2054 WARN_ON_ONCE(!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP));
2055 rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_GP);
2056 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB))
2057 *needwake_state = true;
2063 * No-CBs GP kthreads come here to wait for additional callbacks to show up
2064 * or for grace periods to end.
2066 static void nocb_gp_wait(struct rcu_data *my_rdp)
2068 bool bypass = false;
2070 int __maybe_unused cpu = my_rdp->cpu;
2071 unsigned long cur_gp_seq;
2072 unsigned long flags;
2073 bool gotcbs = false;
2074 unsigned long j = jiffies;
2075 bool needwait_gp = false; // This prevents actual uninitialized use.
2078 struct rcu_data *rdp;
2079 struct rcu_node *rnp;
2080 unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning.
2081 bool wasempty = false;
2084 * Each pass through the following loop checks for CBs and for the
2085 * nearest grace period (if any) to wait for next. The CB kthreads
2086 * and the global grace-period kthread are awakened if needed.
2088 WARN_ON_ONCE(my_rdp->nocb_gp_rdp != my_rdp);
2089 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_cb_rdp) {
2090 bool needwake_state = false;
2092 if (!nocb_gp_enabled_cb(rdp))
2094 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check"));
2095 rcu_nocb_lock_irqsave(rdp, flags);
2096 if (nocb_gp_update_state_deoffloading(rdp, &needwake_state)) {
2097 rcu_nocb_unlock_irqrestore(rdp, flags);
2099 swake_up_one(&rdp->nocb_state_wq);
2102 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
2104 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) ||
2105 bypass_ncbs > 2 * qhimark)) {
2106 // Bypass full or old, so flush it.
2107 (void)rcu_nocb_try_flush_bypass(rdp, j);
2108 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
2109 } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) {
2110 rcu_nocb_unlock_irqrestore(rdp, flags);
2112 swake_up_one(&rdp->nocb_state_wq);
2113 continue; /* No callbacks here, try next. */
2116 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
2121 if (bypass) { // Avoid race with first bypass CB.
2122 WRITE_ONCE(my_rdp->nocb_defer_wakeup,
2124 del_timer(&my_rdp->nocb_timer);
2126 // Advance callbacks if helpful and low contention.
2127 needwake_gp = false;
2128 if (!rcu_segcblist_restempty(&rdp->cblist,
2129 RCU_NEXT_READY_TAIL) ||
2130 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
2131 rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) {
2132 raw_spin_lock_rcu_node(rnp); /* irqs disabled. */
2133 needwake_gp = rcu_advance_cbs(rnp, rdp);
2134 wasempty = rcu_segcblist_restempty(&rdp->cblist,
2135 RCU_NEXT_READY_TAIL);
2136 raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */
2138 // Need to wait on some grace period?
2139 WARN_ON_ONCE(wasempty &&
2140 !rcu_segcblist_restempty(&rdp->cblist,
2141 RCU_NEXT_READY_TAIL));
2142 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {
2144 ULONG_CMP_LT(cur_gp_seq, wait_gp_seq))
2145 wait_gp_seq = cur_gp_seq;
2147 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
2150 if (rcu_segcblist_ready_cbs(&rdp->cblist)) {
2151 needwake = rdp->nocb_cb_sleep;
2152 WRITE_ONCE(rdp->nocb_cb_sleep, false);
2153 smp_mb(); /* CB invocation -after- GP end. */
2157 rcu_nocb_unlock_irqrestore(rdp, flags);
2159 swake_up_one(&rdp->nocb_cb_wq);
2163 rcu_gp_kthread_wake();
2165 swake_up_one(&rdp->nocb_state_wq);
2168 my_rdp->nocb_gp_bypass = bypass;
2169 my_rdp->nocb_gp_gp = needwait_gp;
2170 my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq : 0;
2171 if (bypass && !rcu_nocb_poll) {
2172 // At least one child with non-empty ->nocb_bypass, so set
2173 // timer in order to avoid stranding its callbacks.
2174 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
2175 mod_timer(&my_rdp->nocb_bypass_timer, j + 2);
2176 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
2178 if (rcu_nocb_poll) {
2179 /* Polling, so trace if first poll in the series. */
2181 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll"));
2182 schedule_timeout_idle(1);
2183 } else if (!needwait_gp) {
2184 /* Wait for callbacks to appear. */
2185 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep"));
2186 swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq,
2187 !READ_ONCE(my_rdp->nocb_gp_sleep));
2188 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep"));
2190 rnp = my_rdp->mynode;
2191 trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait"));
2192 swait_event_interruptible_exclusive(
2193 rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1],
2194 rcu_seq_done(&rnp->gp_seq, wait_gp_seq) ||
2195 !READ_ONCE(my_rdp->nocb_gp_sleep));
2196 trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait"));
2198 if (!rcu_nocb_poll) {
2199 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
2201 del_timer(&my_rdp->nocb_bypass_timer);
2202 WRITE_ONCE(my_rdp->nocb_gp_sleep, true);
2203 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
2205 my_rdp->nocb_gp_seq = -1;
2206 WARN_ON(signal_pending(current));
2210 * No-CBs grace-period-wait kthread. There is one of these per group
2211 * of CPUs, but only once at least one CPU in that group has come online
2212 * at least once since boot. This kthread checks for newly posted
2213 * callbacks from any of the CPUs it is responsible for, waits for a
2214 * grace period, then awakens all of the rcu_nocb_cb_kthread() instances
2215 * that then have callback-invocation work to do.
2217 static int rcu_nocb_gp_kthread(void *arg)
2219 struct rcu_data *rdp = arg;
2222 WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1);
2224 cond_resched_tasks_rcu_qs();
2229 static inline bool nocb_cb_can_run(struct rcu_data *rdp)
2231 u8 flags = SEGCBLIST_OFFLOADED | SEGCBLIST_KTHREAD_CB;
2232 return rcu_segcblist_test_flags(&rdp->cblist, flags);
2235 static inline bool nocb_cb_wait_cond(struct rcu_data *rdp)
2237 return nocb_cb_can_run(rdp) && !READ_ONCE(rdp->nocb_cb_sleep);
2241 * Invoke any ready callbacks from the corresponding no-CBs CPU,
2242 * then, if there are no more, wait for more to appear.
2244 static void nocb_cb_wait(struct rcu_data *rdp)
2246 struct rcu_segcblist *cblist = &rdp->cblist;
2247 unsigned long cur_gp_seq;
2248 unsigned long flags;
2249 bool needwake_state = false;
2250 bool needwake_gp = false;
2251 bool can_sleep = true;
2252 struct rcu_node *rnp = rdp->mynode;
2254 local_irq_save(flags);
2255 rcu_momentary_dyntick_idle();
2256 local_irq_restore(flags);
2258 * Disable BH to provide the expected environment. Also, when
2259 * transitioning to/from NOCB mode, a self-requeuing callback might
2260 * be invoked from softirq. A short grace period could cause both
2261 * instances of this callback would execute concurrently.
2266 lockdep_assert_irqs_enabled();
2267 rcu_nocb_lock_irqsave(rdp, flags);
2268 if (rcu_segcblist_nextgp(cblist, &cur_gp_seq) &&
2269 rcu_seq_done(&rnp->gp_seq, cur_gp_seq) &&
2270 raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */
2271 needwake_gp = rcu_advance_cbs(rdp->mynode, rdp);
2272 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
2275 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) {
2276 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) {
2277 rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_CB);
2278 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP))
2279 needwake_state = true;
2281 if (rcu_segcblist_ready_cbs(cblist))
2285 * De-offloading. Clear our flag and notify the de-offload worker.
2286 * We won't touch the callbacks and keep sleeping until we ever
2289 WARN_ON_ONCE(!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB));
2290 rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_CB);
2291 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP))
2292 needwake_state = true;
2295 WRITE_ONCE(rdp->nocb_cb_sleep, can_sleep);
2297 if (rdp->nocb_cb_sleep)
2298 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep"));
2300 rcu_nocb_unlock_irqrestore(rdp, flags);
2302 rcu_gp_kthread_wake();
2305 swake_up_one(&rdp->nocb_state_wq);
2308 swait_event_interruptible_exclusive(rdp->nocb_cb_wq,
2309 nocb_cb_wait_cond(rdp));
2311 // VVV Ensure CB invocation follows _sleep test.
2312 if (smp_load_acquire(&rdp->nocb_cb_sleep)) { // ^^^
2313 WARN_ON(signal_pending(current));
2314 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));
2316 } while (!nocb_cb_can_run(rdp));
2320 * Per-rcu_data kthread, but only for no-CBs CPUs. Repeatedly invoke
2321 * nocb_cb_wait() to do the dirty work.
2323 static int rcu_nocb_cb_kthread(void *arg)
2325 struct rcu_data *rdp = arg;
2327 // Each pass through this loop does one callback batch, and,
2328 // if there are no more ready callbacks, waits for them.
2331 cond_resched_tasks_rcu_qs();
2336 /* Is a deferred wakeup of rcu_nocb_kthread() required? */
2337 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2339 return READ_ONCE(rdp->nocb_defer_wakeup) > RCU_NOCB_WAKE_NOT;
2342 /* Do a deferred wakeup of rcu_nocb_kthread(). */
2343 static bool do_nocb_deferred_wakeup_common(struct rcu_data *rdp)
2345 unsigned long flags;
2349 rcu_nocb_lock_irqsave(rdp, flags);
2350 if (!rcu_nocb_need_deferred_wakeup(rdp)) {
2351 rcu_nocb_unlock_irqrestore(rdp, flags);
2354 ndw = READ_ONCE(rdp->nocb_defer_wakeup);
2355 ret = wake_nocb_gp(rdp, ndw == RCU_NOCB_WAKE_FORCE, flags);
2356 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake"));
2361 /* Do a deferred wakeup of rcu_nocb_kthread() from a timer handler. */
2362 static void do_nocb_deferred_wakeup_timer(struct timer_list *t)
2364 struct rcu_data *rdp = from_timer(rdp, t, nocb_timer);
2366 do_nocb_deferred_wakeup_common(rdp);
2370 * Do a deferred wakeup of rcu_nocb_kthread() from fastpath.
2371 * This means we do an inexact common-case check. Note that if
2372 * we miss, ->nocb_timer will eventually clean things up.
2374 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp)
2376 if (rcu_nocb_need_deferred_wakeup(rdp))
2377 return do_nocb_deferred_wakeup_common(rdp);
2381 void rcu_nocb_flush_deferred_wakeup(void)
2383 do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data));
2385 EXPORT_SYMBOL_GPL(rcu_nocb_flush_deferred_wakeup);
2387 static int rdp_offload_toggle(struct rcu_data *rdp,
2388 bool offload, unsigned long flags)
2389 __releases(rdp->nocb_lock)
2391 struct rcu_segcblist *cblist = &rdp->cblist;
2392 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
2393 bool wake_gp = false;
2395 rcu_segcblist_offload(cblist, offload);
2397 if (rdp->nocb_cb_sleep)
2398 rdp->nocb_cb_sleep = false;
2399 rcu_nocb_unlock_irqrestore(rdp, flags);
2402 * Ignore former value of nocb_cb_sleep and force wake up as it could
2403 * have been spuriously set to false already.
2405 swake_up_one(&rdp->nocb_cb_wq);
2407 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
2408 if (rdp_gp->nocb_gp_sleep) {
2409 rdp_gp->nocb_gp_sleep = false;
2412 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
2415 wake_up_process(rdp_gp->nocb_gp_kthread);
2420 static long rcu_nocb_rdp_deoffload(void *arg)
2422 struct rcu_data *rdp = arg;
2423 struct rcu_segcblist *cblist = &rdp->cblist;
2424 unsigned long flags;
2427 WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id());
2429 pr_info("De-offloading %d\n", rdp->cpu);
2431 rcu_nocb_lock_irqsave(rdp, flags);
2433 * Flush once and for all now. This suffices because we are
2434 * running on the target CPU holding ->nocb_lock (thus having
2435 * interrupts disabled), and because rdp_offload_toggle()
2436 * invokes rcu_segcblist_offload(), which clears SEGCBLIST_OFFLOADED.
2437 * Thus future calls to rcu_segcblist_completely_offloaded() will
2438 * return false, which means that future calls to rcu_nocb_try_bypass()
2439 * will refuse to put anything into the bypass.
2441 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
2442 ret = rdp_offload_toggle(rdp, false, flags);
2443 swait_event_exclusive(rdp->nocb_state_wq,
2444 !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB |
2445 SEGCBLIST_KTHREAD_GP));
2446 rcu_nocb_lock_irqsave(rdp, flags);
2447 /* Make sure nocb timer won't stay around */
2448 WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_OFF);
2449 rcu_nocb_unlock_irqrestore(rdp, flags);
2450 del_timer_sync(&rdp->nocb_timer);
2453 * Theoretically we could set SEGCBLIST_SOFTIRQ_ONLY with CB unlocked
2454 * and IRQs disabled but let's be paranoid.
2456 rcu_nocb_lock_irqsave(rdp, flags);
2457 rcu_segcblist_set_flags(cblist, SEGCBLIST_SOFTIRQ_ONLY);
2459 * With SEGCBLIST_SOFTIRQ_ONLY, we can't use
2460 * rcu_nocb_unlock_irqrestore() anymore.
2462 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
2465 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
2471 int rcu_nocb_cpu_deoffload(int cpu)
2473 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
2476 if (rdp == rdp->nocb_gp_rdp) {
2477 pr_info("Can't deoffload an rdp GP leader (yet)\n");
2480 mutex_lock(&rcu_state.barrier_mutex);
2482 if (rcu_rdp_is_offloaded(rdp)) {
2483 if (cpu_online(cpu)) {
2484 ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp);
2486 cpumask_clear_cpu(cpu, rcu_nocb_mask);
2488 pr_info("NOCB: Can't CB-deoffload an offline CPU\n");
2493 mutex_unlock(&rcu_state.barrier_mutex);
2497 EXPORT_SYMBOL_GPL(rcu_nocb_cpu_deoffload);
2499 static long rcu_nocb_rdp_offload(void *arg)
2501 struct rcu_data *rdp = arg;
2502 struct rcu_segcblist *cblist = &rdp->cblist;
2503 unsigned long flags;
2506 WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id());
2508 * For now we only support re-offload, ie: the rdp must have been
2509 * offloaded on boot first.
2511 if (!rdp->nocb_gp_rdp)
2514 pr_info("Offloading %d\n", rdp->cpu);
2516 * Can't use rcu_nocb_lock_irqsave() while we are in
2517 * SEGCBLIST_SOFTIRQ_ONLY mode.
2519 raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
2520 /* Re-enable nocb timer */
2521 WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);
2523 * We didn't take the nocb lock while working on the
2524 * rdp->cblist in SEGCBLIST_SOFTIRQ_ONLY mode.
2525 * Every modifications that have been done previously on
2526 * rdp->cblist must be visible remotely by the nocb kthreads
2527 * upon wake up after reading the cblist flags.
2529 * The layout against nocb_lock enforces that ordering:
2531 * __rcu_nocb_rdp_offload() nocb_cb_wait()/nocb_gp_wait()
2532 * ------------------------- ----------------------------
2533 * WRITE callbacks rcu_nocb_lock()
2534 * rcu_nocb_lock() READ flags
2535 * WRITE flags READ callbacks
2536 * rcu_nocb_unlock() rcu_nocb_unlock()
2538 ret = rdp_offload_toggle(rdp, true, flags);
2539 swait_event_exclusive(rdp->nocb_state_wq,
2540 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB) &&
2541 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP));
2546 int rcu_nocb_cpu_offload(int cpu)
2548 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
2551 mutex_lock(&rcu_state.barrier_mutex);
2553 if (!rcu_rdp_is_offloaded(rdp)) {
2554 if (cpu_online(cpu)) {
2555 ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp);
2557 cpumask_set_cpu(cpu, rcu_nocb_mask);
2559 pr_info("NOCB: Can't CB-offload an offline CPU\n");
2564 mutex_unlock(&rcu_state.barrier_mutex);
2568 EXPORT_SYMBOL_GPL(rcu_nocb_cpu_offload);
2570 void __init rcu_init_nohz(void)
2573 bool need_rcu_nocb_mask = false;
2574 struct rcu_data *rdp;
2576 #if defined(CONFIG_NO_HZ_FULL)
2577 if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
2578 need_rcu_nocb_mask = true;
2579 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2581 if (!cpumask_available(rcu_nocb_mask) && need_rcu_nocb_mask) {
2582 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
2583 pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
2587 if (!cpumask_available(rcu_nocb_mask))
2590 #if defined(CONFIG_NO_HZ_FULL)
2591 if (tick_nohz_full_running)
2592 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
2593 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2595 if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
2596 pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n");
2597 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
2600 if (cpumask_empty(rcu_nocb_mask))
2601 pr_info("\tOffload RCU callbacks from CPUs: (none).\n");
2603 pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
2604 cpumask_pr_args(rcu_nocb_mask));
2606 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
2608 for_each_cpu(cpu, rcu_nocb_mask) {
2609 rdp = per_cpu_ptr(&rcu_data, cpu);
2610 if (rcu_segcblist_empty(&rdp->cblist))
2611 rcu_segcblist_init(&rdp->cblist);
2612 rcu_segcblist_offload(&rdp->cblist, true);
2613 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB);
2614 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_GP);
2616 rcu_organize_nocb_kthreads();
2619 /* Initialize per-rcu_data variables for no-CBs CPUs. */
2620 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2622 init_swait_queue_head(&rdp->nocb_cb_wq);
2623 init_swait_queue_head(&rdp->nocb_gp_wq);
2624 init_swait_queue_head(&rdp->nocb_state_wq);
2625 raw_spin_lock_init(&rdp->nocb_lock);
2626 raw_spin_lock_init(&rdp->nocb_bypass_lock);
2627 raw_spin_lock_init(&rdp->nocb_gp_lock);
2628 timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0);
2629 timer_setup(&rdp->nocb_bypass_timer, do_nocb_bypass_wakeup_timer, 0);
2630 rcu_cblist_init(&rdp->nocb_bypass);
2634 * If the specified CPU is a no-CBs CPU that does not already have its
2635 * rcuo CB kthread, spawn it. Additionally, if the rcuo GP kthread
2636 * for this CPU's group has not yet been created, spawn it as well.
2638 static void rcu_spawn_one_nocb_kthread(int cpu)
2640 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
2641 struct rcu_data *rdp_gp;
2642 struct task_struct *t;
2645 * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
2646 * then nothing to do.
2648 if (!rcu_is_nocb_cpu(cpu) || rdp->nocb_cb_kthread)
2651 /* If we didn't spawn the GP kthread first, reorganize! */
2652 rdp_gp = rdp->nocb_gp_rdp;
2653 if (!rdp_gp->nocb_gp_kthread) {
2654 t = kthread_run(rcu_nocb_gp_kthread, rdp_gp,
2655 "rcuog/%d", rdp_gp->cpu);
2656 if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo GP kthread, OOM is now expected behavior\n", __func__))
2658 WRITE_ONCE(rdp_gp->nocb_gp_kthread, t);
2661 /* Spawn the kthread for this CPU. */
2662 t = kthread_run(rcu_nocb_cb_kthread, rdp,
2663 "rcuo%c/%d", rcu_state.abbr, cpu);
2664 if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo CB kthread, OOM is now expected behavior\n", __func__))
2666 WRITE_ONCE(rdp->nocb_cb_kthread, t);
2667 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);
2671 * If the specified CPU is a no-CBs CPU that does not already have its
2672 * rcuo kthread, spawn it.
2674 static void rcu_spawn_cpu_nocb_kthread(int cpu)
2676 if (rcu_scheduler_fully_active)
2677 rcu_spawn_one_nocb_kthread(cpu);
2681 * Once the scheduler is running, spawn rcuo kthreads for all online
2682 * no-CBs CPUs. This assumes that the early_initcall()s happen before
2683 * non-boot CPUs come online -- if this changes, we will need to add
2684 * some mutual exclusion.
2686 static void __init rcu_spawn_nocb_kthreads(void)
2690 for_each_online_cpu(cpu)
2691 rcu_spawn_cpu_nocb_kthread(cpu);
2694 /* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */
2695 static int rcu_nocb_gp_stride = -1;
2696 module_param(rcu_nocb_gp_stride, int, 0444);
2699 * Initialize GP-CB relationships for all no-CBs CPU.
2701 static void __init rcu_organize_nocb_kthreads(void)
2704 bool firsttime = true;
2705 bool gotnocbs = false;
2706 bool gotnocbscbs = true;
2707 int ls = rcu_nocb_gp_stride;
2708 int nl = 0; /* Next GP kthread. */
2709 struct rcu_data *rdp;
2710 struct rcu_data *rdp_gp = NULL; /* Suppress misguided gcc warn. */
2711 struct rcu_data *rdp_prev = NULL;
2713 if (!cpumask_available(rcu_nocb_mask))
2716 ls = nr_cpu_ids / int_sqrt(nr_cpu_ids);
2717 rcu_nocb_gp_stride = ls;
2721 * Each pass through this loop sets up one rcu_data structure.
2722 * Should the corresponding CPU come online in the future, then
2723 * we will spawn the needed set of rcu_nocb_kthread() kthreads.
2725 for_each_cpu(cpu, rcu_nocb_mask) {
2726 rdp = per_cpu_ptr(&rcu_data, cpu);
2727 if (rdp->cpu >= nl) {
2728 /* New GP kthread, set up for CBs & next GP. */
2730 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
2731 rdp->nocb_gp_rdp = rdp;
2735 pr_cont("%s\n", gotnocbscbs
2736 ? "" : " (self only)");
2737 gotnocbscbs = false;
2739 pr_alert("%s: No-CB GP kthread CPU %d:",
2743 /* Another CB kthread, link to previous GP kthread. */
2745 rdp->nocb_gp_rdp = rdp_gp;
2746 rdp_prev->nocb_next_cb_rdp = rdp;
2748 pr_cont(" %d", cpu);
2752 if (gotnocbs && dump_tree)
2753 pr_cont("%s\n", gotnocbscbs ? "" : " (self only)");
2757 * Bind the current task to the offloaded CPUs. If there are no offloaded
2758 * CPUs, leave the task unbound. Splat if the bind attempt fails.
2760 void rcu_bind_current_to_nocb(void)
2762 if (cpumask_available(rcu_nocb_mask) && cpumask_weight(rcu_nocb_mask))
2763 WARN_ON(sched_setaffinity(current->pid, rcu_nocb_mask));
2765 EXPORT_SYMBOL_GPL(rcu_bind_current_to_nocb);
2767 // The ->on_cpu field is available only in CONFIG_SMP=y, so...
2769 static char *show_rcu_should_be_on_cpu(struct task_struct *tsp)
2771 return tsp && task_is_running(tsp) && !tsp->on_cpu ? "!" : "";
2773 #else // #ifdef CONFIG_SMP
2774 static char *show_rcu_should_be_on_cpu(struct task_struct *tsp)
2778 #endif // #else #ifdef CONFIG_SMP
2781 * Dump out nocb grace-period kthread state for the specified rcu_data
2784 static void show_rcu_nocb_gp_state(struct rcu_data *rdp)
2786 struct rcu_node *rnp = rdp->mynode;
2788 pr_info("nocb GP %d %c%c%c%c%c%c %c[%c%c] %c%c:%ld rnp %d:%d %lu %c CPU %d%s\n",
2790 "kK"[!!rdp->nocb_gp_kthread],
2791 "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)],
2792 "dD"[!!rdp->nocb_defer_wakeup],
2793 "tT"[timer_pending(&rdp->nocb_timer)],
2794 "bB"[timer_pending(&rdp->nocb_bypass_timer)],
2795 "sS"[!!rdp->nocb_gp_sleep],
2796 ".W"[swait_active(&rdp->nocb_gp_wq)],
2797 ".W"[swait_active(&rnp->nocb_gp_wq[0])],
2798 ".W"[swait_active(&rnp->nocb_gp_wq[1])],
2799 ".B"[!!rdp->nocb_gp_bypass],
2800 ".G"[!!rdp->nocb_gp_gp],
2801 (long)rdp->nocb_gp_seq,
2802 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops),
2803 rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.',
2804 rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1,
2805 show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread));
2808 /* Dump out nocb kthread state for the specified rcu_data structure. */
2809 static void show_rcu_nocb_state(struct rcu_data *rdp)
2813 struct rcu_segcblist *rsclp = &rdp->cblist;
2818 if (rdp->nocb_gp_rdp == rdp)
2819 show_rcu_nocb_gp_state(rdp);
2821 sprintf(bufw, "%ld", rsclp->gp_seq[RCU_WAIT_TAIL]);
2822 sprintf(bufr, "%ld", rsclp->gp_seq[RCU_NEXT_READY_TAIL]);
2823 pr_info(" CB %d^%d->%d %c%c%c%c%c%c F%ld L%ld C%d %c%c%s%c%s%c%c q%ld %c CPU %d%s\n",
2824 rdp->cpu, rdp->nocb_gp_rdp->cpu,
2825 rdp->nocb_next_cb_rdp ? rdp->nocb_next_cb_rdp->cpu : -1,
2826 "kK"[!!rdp->nocb_cb_kthread],
2827 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)],
2828 "cC"[!!atomic_read(&rdp->nocb_lock_contended)],
2829 "lL"[raw_spin_is_locked(&rdp->nocb_lock)],
2830 "sS"[!!rdp->nocb_cb_sleep],
2831 ".W"[swait_active(&rdp->nocb_cb_wq)],
2832 jiffies - rdp->nocb_bypass_first,
2833 jiffies - rdp->nocb_nobypass_last,
2834 rdp->nocb_nobypass_count,
2835 ".D"[rcu_segcblist_ready_cbs(rsclp)],
2836 ".W"[!rcu_segcblist_segempty(rsclp, RCU_WAIT_TAIL)],
2837 rcu_segcblist_segempty(rsclp, RCU_WAIT_TAIL) ? "" : bufw,
2838 ".R"[!rcu_segcblist_segempty(rsclp, RCU_NEXT_READY_TAIL)],
2839 rcu_segcblist_segempty(rsclp, RCU_NEXT_READY_TAIL) ? "" : bufr,
2840 ".N"[!rcu_segcblist_segempty(rsclp, RCU_NEXT_TAIL)],
2841 ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)],
2842 rcu_segcblist_n_cbs(&rdp->cblist),
2843 rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.',
2844 rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1,
2845 show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread));
2847 /* It is OK for GP kthreads to have GP state. */
2848 if (rdp->nocb_gp_rdp == rdp)
2851 waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock);
2852 wastimer = timer_pending(&rdp->nocb_bypass_timer);
2853 wassleep = swait_active(&rdp->nocb_gp_wq);
2854 if (!rdp->nocb_gp_sleep && !waslocked && !wastimer && !wassleep)
2855 return; /* Nothing untowards. */
2857 pr_info(" nocb GP activity on CB-only CPU!!! %c%c%c%c %c\n",
2859 "dD"[!!rdp->nocb_defer_wakeup],
2861 "sS"[!!rdp->nocb_gp_sleep],
2865 #else /* #ifdef CONFIG_RCU_NOCB_CPU */
2867 /* No ->nocb_lock to acquire. */
2868 static void rcu_nocb_lock(struct rcu_data *rdp)
2872 /* No ->nocb_lock to release. */
2873 static void rcu_nocb_unlock(struct rcu_data *rdp)
2877 /* No ->nocb_lock to release. */
2878 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
2879 unsigned long flags)
2881 local_irq_restore(flags);
2884 /* Lockdep check that ->cblist may be safely accessed. */
2885 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
2887 lockdep_assert_irqs_disabled();
2890 static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
2894 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
2899 static void rcu_init_one_nocb(struct rcu_node *rnp)
2903 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
2909 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
2910 bool *was_alldone, unsigned long flags)
2915 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty,
2916 unsigned long flags)
2918 WARN_ON_ONCE(1); /* Should be dead code! */
2921 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
2925 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
2930 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp)
2935 static void rcu_spawn_cpu_nocb_kthread(int cpu)
2939 static void __init rcu_spawn_nocb_kthreads(void)
2943 static void show_rcu_nocb_state(struct rcu_data *rdp)
2947 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
2950 * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the
2951 * grace-period kthread will do force_quiescent_state() processing?
2952 * The idea is to avoid waking up RCU core processing on such a
2953 * CPU unless the grace period has extended for too long.
2955 * This code relies on the fact that all NO_HZ_FULL CPUs are also
2956 * CONFIG_RCU_NOCB_CPU CPUs.
2958 static bool rcu_nohz_full_cpu(void)
2960 #ifdef CONFIG_NO_HZ_FULL
2961 if (tick_nohz_full_cpu(smp_processor_id()) &&
2962 (!rcu_gp_in_progress() ||
2963 time_before(jiffies, READ_ONCE(rcu_state.gp_start) + HZ)))
2965 #endif /* #ifdef CONFIG_NO_HZ_FULL */
2970 * Bind the RCU grace-period kthreads to the housekeeping CPU.
2972 static void rcu_bind_gp_kthread(void)
2974 if (!tick_nohz_full_enabled())
2976 housekeeping_affine(current, HK_FLAG_RCU);
2979 /* Record the current task on dyntick-idle entry. */
2980 static void noinstr rcu_dynticks_task_enter(void)
2982 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
2983 WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id());
2984 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
2987 /* Record no current task on dyntick-idle exit. */
2988 static void noinstr rcu_dynticks_task_exit(void)
2990 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
2991 WRITE_ONCE(current->rcu_tasks_idle_cpu, -1);
2992 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
2995 /* Turn on heavyweight RCU tasks trace readers on idle/user entry. */
2996 static void rcu_dynticks_task_trace_enter(void)
2998 #ifdef CONFIG_TASKS_RCU_TRACE
2999 if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB))
3000 current->trc_reader_special.b.need_mb = true;
3001 #endif /* #ifdef CONFIG_TASKS_RCU_TRACE */
3004 /* Turn off heavyweight RCU tasks trace readers on idle/user exit. */
3005 static void rcu_dynticks_task_trace_exit(void)
3007 #ifdef CONFIG_TASKS_RCU_TRACE
3008 if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB))
3009 current->trc_reader_special.b.need_mb = false;
3010 #endif /* #ifdef CONFIG_TASKS_RCU_TRACE */