Commit 8747f202 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Rafael J. Wysocki

cpuidle: Allow cpuidle drivers to take over RCU-idle

Some drivers have to do significant work, some of which relies on RCU
still being active. Instead of using RCU_NONIDLE in the drivers and
flipping RCU back on, allow drivers to take over RCU-idle duty.
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarUlf Hansson <ulf.hansson@linaro.org>
Tested-by: default avatarBorislav Petkov <bp@suse.de>
Signed-off-by: default avatarRafael J. Wysocki <rafael.j.wysocki@intel.com>
parent a889a23a
......@@ -138,6 +138,7 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv,
struct cpuidle_device *dev, int index)
{
ktime_t time_start, time_end;
struct cpuidle_state *target_state = &drv->states[index];
time_start = ns_to_ktime(local_clock());
......@@ -153,8 +154,9 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv,
* suspended is generally unsafe.
*/
stop_critical_timings();
rcu_idle_enter();
drv->states[index].enter_s2idle(dev, drv, index);
if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
rcu_idle_enter();
target_state->enter_s2idle(dev, drv, index);
if (WARN_ON_ONCE(!irqs_disabled()))
local_irq_disable();
/*
......@@ -162,7 +164,8 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv,
* first CPU executing it calls functions containing RCU read-side
* critical sections, so tell RCU about that.
*/
rcu_idle_exit();
if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
rcu_idle_exit();
tick_unfreeze();
start_critical_timings();
......@@ -239,9 +242,11 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
time_start = ns_to_ktime(local_clock());
stop_critical_timings();
rcu_idle_enter();
if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
rcu_idle_enter();
entered_state = target_state->enter(dev, drv, index);
rcu_idle_exit();
if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
rcu_idle_exit();
start_critical_timings();
sched_clock_idle_wakeup_event();
......
......@@ -82,6 +82,7 @@ struct cpuidle_state {
#define CPUIDLE_FLAG_UNUSABLE BIT(3) /* avoid using this state */
#define CPUIDLE_FLAG_OFF BIT(4) /* disable this state by default */
#define CPUIDLE_FLAG_TLB_FLUSHED BIT(5) /* idle-state flushes TLBs */
#define CPUIDLE_FLAG_RCU_IDLE BIT(6) /* idle-state takes care of RCU */
struct cpuidle_device_kobj;
struct cpuidle_state_kobj;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment