rcu-tasks: Track blocked RCU Tasks Trace readers

This commit places any task that has ever blocked within its current
RCU Tasks Trace read-side critical section on a per-CPU list within the
rcu_tasks_percpu structure.  Tasks are removed from this list when they
exit by the exit_tasks_rcu_finish_trace() function.  The purpose of this
commit is to provide the information needed to eliminate the current
scan of the full task list.

This commit offsets the INT_MIN value for ->trc_reader_nesting with the
new nesting level in order to avoid queueing tasks that are exiting
their read-side critical sections.

[ paulmck: Apply kernel test robot feedback. ]
[ paulmck: Apply feedback from syzbot+9bb26e7c5e8e4fa7e641@syzkaller.appspotmail.com ]

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: syzbot <syzbot+9bb26e7c5e8e4fa7e641@syzkaller.appspotmail.com>
Tested-by: "Zhang, Qiang1" <qiang1.zhang@intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Martin KaFai Lau <kafai@fb.com>
Cc: KP Singh <kpsingh@kernel.org>

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 1e728d5..ebdfeea 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -174,12 +174,19 @@ void synchronize_rcu_tasks(void);
 #define TRC_NEED_QS_CHECKED    0x2  // Task has been checked for needing quiescent state.
 
 u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new);
+void rcu_tasks_trace_qs_blkd(struct task_struct *t);
 
 # define rcu_tasks_trace_qs(t)                                                 \
        do {                                                                    \
+               int ___rttq_nesting = READ_ONCE((t)->trc_reader_nesting);       \
+                                                                               \
                if (likely(!READ_ONCE((t)->trc_reader_special.b.need_qs)) &&    \
-                   likely(!READ_ONCE((t)->trc_reader_nesting)))                \
+                   likely(!___rttq_nesting)) {                                 \
                        rcu_trc_cmpxchg_need_qs((t), 0, TRC_NEED_QS_CHECKED);   \
+               } else if (___rttq_nesting && ___rttq_nesting != INT_MIN &&     \
+                          !READ_ONCE((t)->trc_reader_special.b.blocked)) {     \
+                       rcu_tasks_trace_qs_blkd(t);                             \
+               }                                                               \
        } while (0)
 # else
 # define rcu_tasks_trace_qs(t) do { } while (0)
@@ -188,7 +195,7 @@ u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new);
 #define rcu_tasks_qs(t, preempt)                                       \
 do {                                                                   \
        rcu_tasks_classic_qs((t), (preempt));                           \
-       rcu_tasks_trace_qs((t));                                        \
+       rcu_tasks_trace_qs(t);                                          \
 } while (0)
 
 # ifdef CONFIG_TASKS_RUDE_RCU

diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h
index 6f9c358..9bc8cbb 100644 (file)
--- a/include/linux/rcupdate_trace.h
+++ b/include/linux/rcupdate_trace.h
@@ -75,7 +75,7 @@ static inline void rcu_read_unlock_trace(void)
        nesting = READ_ONCE(t->trc_reader_nesting) - 1;
        barrier(); // Critical section before disabling.
        // Disable IPI-based setting of .need_qs.
-       WRITE_ONCE(t->trc_reader_nesting, INT_MIN);
+       WRITE_ONCE(t->trc_reader_nesting, INT_MIN + nesting);
        if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) {
                WRITE_ONCE(t->trc_reader_nesting, nesting);
                return;  // We assume shallow reader nesting.

diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index fd4508a..bab75ec 100644 (file)
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -1261,6 +1261,24 @@ void rcu_read_unlock_trace_special(struct task_struct *t)
 }
 EXPORT_SYMBOL_GPL(rcu_read_unlock_trace_special);
 
+/* Add a newly blocked reader task to its CPU's list. */
+void rcu_tasks_trace_qs_blkd(struct task_struct *t)
+{
+       unsigned long flags;
+       struct rcu_tasks_percpu *rtpcp;
+
+       local_irq_save(flags);
+       rtpcp = this_cpu_ptr(rcu_tasks_trace.rtpcpu);
+       raw_spin_lock_rcu_node(rtpcp); // irqs already disabled
+       t->trc_blkd_cpu = smp_processor_id();
+       if (!rtpcp->rtp_blkd_tasks.next)
+               INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);
+       list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks);
+       t->trc_reader_special.b.blocked = true;
+       raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
+}
+EXPORT_SYMBOL_GPL(rcu_tasks_trace_qs_blkd);
+
 /* Add a task to the holdout list, if it is not already on the list. */
 static void trc_add_holdout(struct task_struct *t, struct list_head *bhp)
 {
@@ -1586,9 +1604,11 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp)
 /* Report any needed quiescent state for this exiting task. */
 static void exit_tasks_rcu_finish_trace(struct task_struct *t)
 {
+       union rcu_special trs = READ_ONCE(t->trc_reader_special);
+
        rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED);
        WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting));
-       if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS))
+       if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS) || trs.b.blocked)
                rcu_read_unlock_trace_special(t);
        else
                WRITE_ONCE(t->trc_reader_nesting, 0);