cpuidle: Allow cpuidle drivers to take over RCU-idle

Some drivers have to do significant work, some of which relies on RCU
still being active. Instead of using RCU_NONIDLE in the drivers and
flipping RCU back on, allow drivers to take over RCU-idle duty.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Ulf Hansson <ulf.hansson@linaro.org>
Tested-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>

diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index 04becd7..6c7e562 100644 (file)
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -138,6 +138,7 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv,
                                struct cpuidle_device *dev, int index)
 {
        ktime_t time_start, time_end;
+       struct cpuidle_state *target_state = &drv->states[index];
 
        time_start = ns_to_ktime(local_clock());
 
@@ -153,8 +154,9 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv,
         * suspended is generally unsafe.
         */
        stop_critical_timings();
-       rcu_idle_enter();
-       drv->states[index].enter_s2idle(dev, drv, index);
+       if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
+               rcu_idle_enter();
+       target_state->enter_s2idle(dev, drv, index);
        if (WARN_ON_ONCE(!irqs_disabled()))
                local_irq_disable();
        /*
@@ -162,7 +164,8 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv,
         * first CPU executing it calls functions containing RCU read-side
         * critical sections, so tell RCU about that.
         */
-       rcu_idle_exit();
+       if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
+               rcu_idle_exit();
        tick_unfreeze();
        start_critical_timings();
 
@@ -239,9 +242,11 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
        time_start = ns_to_ktime(local_clock());
 
        stop_critical_timings();
-       rcu_idle_enter();
+       if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
+               rcu_idle_enter();
        entered_state = target_state->enter(dev, drv, index);
-       rcu_idle_exit();
+       if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
+               rcu_idle_exit();
        start_critical_timings();
 
        sched_clock_idle_wakeup_event();

diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h
index 75895e6..6175c77 100644 (file)
--- a/include/linux/cpuidle.h
+++ b/include/linux/cpuidle.h
@@ -82,6 +82,7 @@ struct cpuidle_state {
 #define CPUIDLE_FLAG_UNUSABLE          BIT(3) /* avoid using this state */
 #define CPUIDLE_FLAG_OFF               BIT(4) /* disable this state by default */
 #define CPUIDLE_FLAG_TLB_FLUSHED       BIT(5) /* idle-state flushes TLBs */
+#define CPUIDLE_FLAG_RCU_IDLE          BIT(6) /* idle-state takes care of RCU */
 
 struct cpuidle_device_kobj;
 struct cpuidle_state_kobj;