Merge tag 'rcu.2022.05.19a' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck...

Pull RCU update from Paul McKenney:

 - Documentation updates

 - Miscellaneous fixes

 - Callback-offloading updates, mainly simplifications

 - RCU-tasks updates, including some -rt fixups, handling of systems
   with sparse CPU numbering, and a fix for a boot-time race-condition
   failure

 - Put SRCU on a memory diet in order to reduce the size of the
   srcu_struct structure

 - Torture-test updates fixing some bugs in tests and closing some
   testing holes

 - Torture-test updates for the RCU tasks flavors, most notably ensuring
   that building rcutorture and friends does not change the
   RCU-tasks-related Kconfig options

 - Torture-test scripting updates

 - Expedited grace-period updates, most notably providing
   milliseconds-scale (not all that) soft real-time response from
   synchronize_rcu_expedited().

   This is also the first time in almost 30 years of RCU that someone
   other than me has pushed for a reduction in the RCU CPU stall-warning
   timeout, in this case by more than three orders of magnitude from 21
   seconds to 20 milliseconds. This tighter timeout applies only to
   expedited grace periods

* tag 'rcu.2022.05.19a' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu: (80 commits)
  rcu: Move expedited grace period (GP) work to RT kthread_worker
  rcu: Introduce CONFIG_RCU_EXP_CPU_STALL_TIMEOUT
  srcu: Drop needless initialization of sdp in srcu_gp_start()
  srcu: Prevent expedited GPs and blocking readers from consuming CPU
  srcu: Add contention check to call_srcu() srcu_data ->lock acquisition
  srcu: Automatically determine size-transition strategy at boot
  rcutorture: Make torture.sh allow for --kasan
  rcutorture: Make torture.sh refscale and rcuscale specify Tasks Trace RCU
  rcutorture: Make kvm.sh allow more memory for --kasan runs
  torture: Save "make allmodconfig" .config file
  scftorture: Remove extraneous "scf" from per_version_boot_params
  rcutorture: Adjust scenarios' Kconfig options for CONFIG_PREEMPT_DYNAMIC
  torture: Enable CSD-lock stall reports for scftorture
  torture: Skip vmlinux check for kvm-again.sh runs
  scftorture: Adjust for TASKS_RCU Kconfig option being selected
  rcuscale: Allow rcuscale without RCU Tasks Rude/Trace
  rcuscale: Allow rcuscale without RCU Tasks
  refscale: Allow refscale without RCU Tasks Rude/Trace
  refscale: Allow refscale without RCU Tasks
  rcutorture: Allow specifying per-scenario stat_interval
  ...

diff --combined arch/Kconfig
index 31c4fdc,1bf29ce..513a29c
--- 1/arch/Kconfig
--- 2/arch/Kconfig
+++ b/arch/Kconfig
@@@ -35,6 -35,7 +35,7 @@@ config KPROBE
        depends on MODULES
        depends on HAVE_KPROBES
        select KALLSYMS
+       select TASKS_RCU if PREEMPTION
        help
          Kprobes allows you to trap at almost any kernel address and
          execute a callback function.  register_kprobe() establishes
@@@ -854,8 -855,10 +855,8 @@@ config HAVE_ARCH_HUGE_VMA
  
  #
  #  Archs that select this would be capable of PMD-sized vmaps (i.e.,
 -#  arch_vmap_pmd_supported() returns true), and they must make no assumptions
 -#  that vmalloc memory is mapped with PAGE_SIZE ptes. The VM_NO_HUGE_VMAP flag
 -#  can be used to prohibit arch-specific allocations from using hugepages to
 -#  help with this (e.g., modules may require it).
 +#  arch_vmap_pmd_supported() returns true). The VM_ALLOW_HUGE_VMAP flag
 +#  must be used to enable allocations to use hugepages.
  #
  config HAVE_ARCH_HUGE_VMALLOC
        depends on HAVE_ARCH_HUGE_VMAP

diff --combined include/linux/sched.h
index a8911b1,67f06f7..b3278f8
--- 1/include/linux/sched.h
--- 2/include/linux/sched.h
+++ b/include/linux/sched.h
@@@ -1443,7 -1443,6 +1443,7 @@@ struct task_struct 
        int                             pagefault_disabled;
  #ifdef CONFIG_MMU
        struct task_struct              *oom_reaper_list;
 +      struct timer_list               oom_reaper_timer;
  #endif
  #ifdef CONFIG_VMAP_STACK
        struct vm_struct                *stack_vm_area;
@@@ -2118,6 -2117,47 +2118,47 @@@ static inline void cond_resched_rcu(voi
  #endif
  }
  
+ #ifdef CONFIG_PREEMPT_DYNAMIC
+ 
+ extern bool preempt_model_none(void);
+ extern bool preempt_model_voluntary(void);
+ extern bool preempt_model_full(void);
+ 
+ #else
+ 
+ static inline bool preempt_model_none(void)
+ {
+       return IS_ENABLED(CONFIG_PREEMPT_NONE);
+ }
+ static inline bool preempt_model_voluntary(void)
+ {
+       return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY);
+ }
+ static inline bool preempt_model_full(void)
+ {
+       return IS_ENABLED(CONFIG_PREEMPT);
+ }
+ 
+ #endif
+ 
+ static inline bool preempt_model_rt(void)
+ {
+       return IS_ENABLED(CONFIG_PREEMPT_RT);
+ }
+ 
+ /*
+  * Does the preemption model allow non-cooperative preemption?
+  *
+  * For !CONFIG_PREEMPT_DYNAMIC kernels this is an exact match with
+  * CONFIG_PREEMPTION; for CONFIG_PREEMPT_DYNAMIC this doesn't work as the
+  * kernel is *built* with CONFIG_PREEMPTION=y but may run with e.g. the
+  * PREEMPT_NONE model.
+  */
+ static inline bool preempt_model_preemptible(void)
+ {
+       return preempt_model_full() || preempt_model_rt();
+ }
+ 
  /*
   * Does a critical section need to be broken due to another
   * task waiting?: (technically does not depend on CONFIG_PREEMPTION,

diff --combined kernel/sched/core.c
index d58c038,068c088..2a05096
--- 1/kernel/sched/core.c
--- 2/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@@ -5752,8 -5752,6 +5752,8 @@@ static inline struct task_struct *pick_
  
  extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi);
  
 +static void queue_core_balance(struct rq *rq);
 +
  static struct task_struct *
  pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
  {
@@@ -5803,7 -5801,7 +5803,7 @@@
                }
  
                rq->core_pick = NULL;
 -              return next;
 +              goto out;
        }
  
        put_prev_task_balance(rq, prev, rf);
@@@ -5853,7 -5851,7 +5853,7 @@@
                         */
                        WARN_ON_ONCE(fi_before);
                        task_vruntime_update(rq, next, false);
 -                      goto done;
 +                      goto out_set_next;
                }
        }
  
@@@ -5972,12 -5970,8 +5972,12 @@@
                resched_curr(rq_i);
        }
  
 -done:
 +out_set_next:
        set_next_task(rq, next);
 +out:
 +      if (rq->core->core_forceidle_count && next == rq->idle)
 +              queue_core_balance(rq);
 +
        return next;
  }
  
@@@ -6006,7 -6000,7 +6006,7 @@@ static bool try_steal_cookie(int this, 
                if (p == src->core_pick || p == src->curr)
                        goto next;
  
 -              if (!cpumask_test_cpu(this, &p->cpus_mask))
 +              if (!is_cpu_allowed(p, this))
                        goto next;
  
                if (p->core_occupation > dst->idle->core_occupation)
@@@ -6072,7 -6066,7 +6072,7 @@@ static void sched_core_balance(struct r
  
  static DEFINE_PER_CPU(struct callback_head, core_balance_head);
  
 -void queue_core_balance(struct rq *rq)
 +static void queue_core_balance(struct rq *rq)
  {
        if (!sched_core_enabled(rq))
                return;
@@@ -6382,7 -6376,7 +6382,7 @@@ static void __sched notrace __schedule(
                migrate_disable_switch(rq, prev);
                psi_sched_switch(prev, next, !task_on_rq_queued(prev));
  
 -              trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev_state, prev, next);
 +              trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state);
  
                /* Also unlocks the rq: */
                rq = context_switch(rq, prev, next, &rf);
@@@ -8415,6 -8409,18 +8415,18 @@@ static void __init preempt_dynamic_init
        }
  }
  
+ #define PREEMPT_MODEL_ACCESSOR(mode) \
+       bool preempt_model_##mode(void)                                          \
+       {                                                                        \
+               WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \
+               return preempt_dynamic_mode == preempt_dynamic_##mode;           \
+       }                                                                        \
+       EXPORT_SYMBOL_GPL(preempt_model_##mode)
+ 
+ PREEMPT_MODEL_ACCESSOR(none);
+ PREEMPT_MODEL_ACCESSOR(voluntary);
+ PREEMPT_MODEL_ACCESSOR(full);
+ 
  #else /* !CONFIG_PREEMPT_DYNAMIC */
  
  static inline void preempt_dynamic_init(void) { }

diff --combined kernel/smp.c
index 65a630f,6a1f1da..df9393a
--- 1/kernel/smp.c
--- 2/kernel/smp.c
+++ b/kernel/smp.c
@@@ -183,7 -183,9 +183,9 @@@ static DEFINE_PER_CPU(smp_call_func_t, 
  static DEFINE_PER_CPU(void *, cur_csd_info);
  static DEFINE_PER_CPU(struct cfd_seq_local, cfd_seq_local);
  
- #define CSD_LOCK_TIMEOUT (5ULL * NSEC_PER_SEC)
+ static ulong csd_lock_timeout = 5000;  /* CSD lock timeout in milliseconds. */
+ module_param(csd_lock_timeout, ulong, 0444);
+ 
  static atomic_t csd_bug_count = ATOMIC_INIT(0);
  static u64 cfd_seq;
  
@@@ -329,6 -331,7 +331,7 @@@ static bool csd_lock_wait_toolong(struc
        u64 ts2, ts_delta;
        call_single_data_t *cpu_cur_csd;
        unsigned int flags = READ_ONCE(csd->node.u_flags);
+       unsigned long long csd_lock_timeout_ns = csd_lock_timeout * NSEC_PER_MSEC;
  
        if (!(flags & CSD_FLAG_LOCK)) {
                if (!unlikely(*bug_id))
@@@ -341,7 -344,7 +344,7 @@@
  
        ts2 = sched_clock();
        ts_delta = ts2 - *ts1;
-       if (likely(ts_delta <= CSD_LOCK_TIMEOUT))
+       if (likely(ts_delta <= csd_lock_timeout_ns || csd_lock_timeout_ns == 0))
                return false;
  
        firsttime = !*bug_id;
@@@ -579,7 -582,7 +582,7 @@@ static void flush_smp_call_function_que
  
        /* There shouldn't be any pending callbacks on an offline CPU. */
        if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
 -                   !warned && !llist_empty(head))) {
 +                   !warned && entry != NULL)) {
                warned = true;
                WARN(1, "IPI on offline CPU %d\n", smp_processor_id());