Commit e8dc676e authored by Rafael J. Wysocki's avatar Rafael J. Wysocki

Merge branch 'pm-cpuidle'

* pm-cpuidle:
  ACPI: processor: Take over RCU-idle for C3-BM idle
  cpuidle: Allow cpuidle drivers to take over RCU-idle
  ACPI: processor: Use CPUIDLE_FLAG_TLB_FLUSHED
  ACPI: processor: Use CPUIDLE_FLAG_TIMER_STOP
parents e1c2d96c 1fecfdbb
...@@ -74,8 +74,6 @@ static inline void arch_acpi_set_pdc_bits(u32 *buf) ...@@ -74,8 +74,6 @@ static inline void arch_acpi_set_pdc_bits(u32 *buf)
buf[2] |= ACPI_PDC_EST_CAPABILITY_SMP; buf[2] |= ACPI_PDC_EST_CAPABILITY_SMP;
} }
#define acpi_unlazy_tlb(x)
#ifdef CONFIG_ACPI_NUMA #ifdef CONFIG_ACPI_NUMA
extern cpumask_t early_cpu_possible_map; extern cpumask_t early_cpu_possible_map;
#define for_each_possible_early_cpu(cpu) \ #define for_each_possible_early_cpu(cpu) \
......
...@@ -159,8 +159,6 @@ static inline u64 x86_default_get_root_pointer(void) ...@@ -159,8 +159,6 @@ static inline u64 x86_default_get_root_pointer(void)
extern int x86_acpi_numa_init(void); extern int x86_acpi_numa_init(void);
#endif /* CONFIG_ACPI_NUMA */ #endif /* CONFIG_ACPI_NUMA */
#define acpi_unlazy_tlb(x) leave_mm(x)
#ifdef CONFIG_ACPI_APEI #ifdef CONFIG_ACPI_APEI
static inline pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr) static inline pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr)
{ {
......
...@@ -161,18 +161,10 @@ static void lapic_timer_propagate_broadcast(struct acpi_processor *pr) ...@@ -161,18 +161,10 @@ static void lapic_timer_propagate_broadcast(struct acpi_processor *pr)
} }
/* Power(C) State timer broadcast control */ /* Power(C) State timer broadcast control */
static void lapic_timer_state_broadcast(struct acpi_processor *pr, static bool lapic_timer_needs_broadcast(struct acpi_processor *pr,
struct acpi_processor_cx *cx, struct acpi_processor_cx *cx)
int broadcast)
{ {
int state = cx - pr->power.states; return cx - pr->power.states >= pr->power.timer_broadcast_on_state;
if (state >= pr->power.timer_broadcast_on_state) {
if (broadcast)
tick_broadcast_enter();
else
tick_broadcast_exit();
}
} }
#else #else
...@@ -180,9 +172,9 @@ static void lapic_timer_state_broadcast(struct acpi_processor *pr, ...@@ -180,9 +172,9 @@ static void lapic_timer_state_broadcast(struct acpi_processor *pr,
static void lapic_timer_check_state(int state, struct acpi_processor *pr, static void lapic_timer_check_state(int state, struct acpi_processor *pr,
struct acpi_processor_cx *cstate) { } struct acpi_processor_cx *cstate) { }
static void lapic_timer_propagate_broadcast(struct acpi_processor *pr) { } static void lapic_timer_propagate_broadcast(struct acpi_processor *pr) { }
static void lapic_timer_state_broadcast(struct acpi_processor *pr,
struct acpi_processor_cx *cx, static bool lapic_timer_needs_broadcast(struct acpi_processor *pr,
int broadcast) struct acpi_processor_cx *cx)
{ {
} }
...@@ -566,32 +558,43 @@ static DEFINE_RAW_SPINLOCK(c3_lock); ...@@ -566,32 +558,43 @@ static DEFINE_RAW_SPINLOCK(c3_lock);
/** /**
* acpi_idle_enter_bm - enters C3 with proper BM handling * acpi_idle_enter_bm - enters C3 with proper BM handling
* @drv: cpuidle driver
* @pr: Target processor * @pr: Target processor
* @cx: Target state context * @cx: Target state context
* @timer_bc: Whether or not to change timer mode to broadcast * @index: index of target state
*/ */
static void acpi_idle_enter_bm(struct acpi_processor *pr, static int acpi_idle_enter_bm(struct cpuidle_driver *drv,
struct acpi_processor_cx *cx, bool timer_bc) struct acpi_processor *pr,
struct acpi_processor_cx *cx,
int index)
{ {
acpi_unlazy_tlb(smp_processor_id()); static struct acpi_processor_cx safe_cx = {
.entry_method = ACPI_CSTATE_HALT,
/* };
* Must be done before busmaster disable as we might need to
* access HPET !
*/
if (timer_bc)
lapic_timer_state_broadcast(pr, cx, 1);
/* /*
* disable bus master * disable bus master
* bm_check implies we need ARB_DIS * bm_check implies we need ARB_DIS
* bm_control implies whether we can do ARB_DIS * bm_control implies whether we can do ARB_DIS
* *
* That leaves a case where bm_check is set and bm_control is * That leaves a case where bm_check is set and bm_control is not set.
* not set. In that case we cannot do much, we enter C3 * In that case we cannot do much, we enter C3 without doing anything.
* without doing anything.
*/ */
if (pr->flags.bm_control) { bool dis_bm = pr->flags.bm_control;
/* If we can skip BM, demote to a safe state. */
if (!cx->bm_sts_skip && acpi_idle_bm_check()) {
dis_bm = false;
index = drv->safe_state_index;
if (index >= 0) {
cx = this_cpu_read(acpi_cstate[index]);
} else {
cx = &safe_cx;
index = -EBUSY;
}
}
if (dis_bm) {
raw_spin_lock(&c3_lock); raw_spin_lock(&c3_lock);
c3_cpu_count++; c3_cpu_count++;
/* Disable bus master arbitration when all CPUs are in C3 */ /* Disable bus master arbitration when all CPUs are in C3 */
...@@ -600,18 +603,21 @@ static void acpi_idle_enter_bm(struct acpi_processor *pr, ...@@ -600,18 +603,21 @@ static void acpi_idle_enter_bm(struct acpi_processor *pr,
raw_spin_unlock(&c3_lock); raw_spin_unlock(&c3_lock);
} }
rcu_idle_enter();
acpi_idle_do_entry(cx); acpi_idle_do_entry(cx);
rcu_idle_exit();
/* Re-enable bus master arbitration */ /* Re-enable bus master arbitration */
if (pr->flags.bm_control) { if (dis_bm) {
raw_spin_lock(&c3_lock); raw_spin_lock(&c3_lock);
acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, 0); acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, 0);
c3_cpu_count--; c3_cpu_count--;
raw_spin_unlock(&c3_lock); raw_spin_unlock(&c3_lock);
} }
if (timer_bc) return index;
lapic_timer_state_broadcast(pr, cx, 0);
} }
static int acpi_idle_enter(struct cpuidle_device *dev, static int acpi_idle_enter(struct cpuidle_device *dev,
...@@ -625,32 +631,21 @@ static int acpi_idle_enter(struct cpuidle_device *dev, ...@@ -625,32 +631,21 @@ static int acpi_idle_enter(struct cpuidle_device *dev,
return -EINVAL; return -EINVAL;
if (cx->type != ACPI_STATE_C1) { if (cx->type != ACPI_STATE_C1) {
if (cx->type == ACPI_STATE_C3 && pr->flags.bm_check)
return acpi_idle_enter_bm(drv, pr, cx, index);
/* C2 to C1 demotion. */
if (acpi_idle_fallback_to_c1(pr) && num_online_cpus() > 1) { if (acpi_idle_fallback_to_c1(pr) && num_online_cpus() > 1) {
index = ACPI_IDLE_STATE_START; index = ACPI_IDLE_STATE_START;
cx = per_cpu(acpi_cstate[index], dev->cpu); cx = per_cpu(acpi_cstate[index], dev->cpu);
} else if (cx->type == ACPI_STATE_C3 && pr->flags.bm_check) {
if (cx->bm_sts_skip || !acpi_idle_bm_check()) {
acpi_idle_enter_bm(pr, cx, true);
return index;
} else if (drv->safe_state_index >= 0) {
index = drv->safe_state_index;
cx = per_cpu(acpi_cstate[index], dev->cpu);
} else {
acpi_safe_halt();
return -EBUSY;
} }
} }
}
lapic_timer_state_broadcast(pr, cx, 1);
if (cx->type == ACPI_STATE_C3) if (cx->type == ACPI_STATE_C3)
ACPI_FLUSH_CPU_CACHE(); ACPI_FLUSH_CPU_CACHE();
acpi_idle_do_entry(cx); acpi_idle_do_entry(cx);
lapic_timer_state_broadcast(pr, cx, 0);
return index; return index;
} }
...@@ -666,7 +661,13 @@ static int acpi_idle_enter_s2idle(struct cpuidle_device *dev, ...@@ -666,7 +661,13 @@ static int acpi_idle_enter_s2idle(struct cpuidle_device *dev,
return 0; return 0;
if (pr->flags.bm_check) { if (pr->flags.bm_check) {
acpi_idle_enter_bm(pr, cx, false); u8 bm_sts_skip = cx->bm_sts_skip;
/* Don't check BM_STS, do an unconditional ARB_DIS for S2IDLE */
cx->bm_sts_skip = 1;
acpi_idle_enter_bm(drv, pr, cx, index);
cx->bm_sts_skip = bm_sts_skip;
return 0; return 0;
} else { } else {
ACPI_FLUSH_CPU_CACHE(); ACPI_FLUSH_CPU_CACHE();
...@@ -682,11 +683,13 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, ...@@ -682,11 +683,13 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr,
{ {
int i, count = ACPI_IDLE_STATE_START; int i, count = ACPI_IDLE_STATE_START;
struct acpi_processor_cx *cx; struct acpi_processor_cx *cx;
struct cpuidle_state *state;
if (max_cstate == 0) if (max_cstate == 0)
max_cstate = 1; max_cstate = 1;
for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) { for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
state = &acpi_idle_driver.states[count];
cx = &pr->power.states[i]; cx = &pr->power.states[i];
if (!cx->valid) if (!cx->valid)
...@@ -694,6 +697,15 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, ...@@ -694,6 +697,15 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr,
per_cpu(acpi_cstate[count], dev->cpu) = cx; per_cpu(acpi_cstate[count], dev->cpu) = cx;
if (lapic_timer_needs_broadcast(pr, cx))
state->flags |= CPUIDLE_FLAG_TIMER_STOP;
if (cx->type == ACPI_STATE_C3) {
state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
if (pr->flags.bm_check)
state->flags |= CPUIDLE_FLAG_RCU_IDLE;
}
count++; count++;
if (count == CPUIDLE_STATE_MAX) if (count == CPUIDLE_STATE_MAX)
break; break;
......
...@@ -138,6 +138,7 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv, ...@@ -138,6 +138,7 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv,
struct cpuidle_device *dev, int index) struct cpuidle_device *dev, int index)
{ {
ktime_t time_start, time_end; ktime_t time_start, time_end;
struct cpuidle_state *target_state = &drv->states[index];
time_start = ns_to_ktime(local_clock()); time_start = ns_to_ktime(local_clock());
...@@ -153,8 +154,9 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv, ...@@ -153,8 +154,9 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv,
* suspended is generally unsafe. * suspended is generally unsafe.
*/ */
stop_critical_timings(); stop_critical_timings();
if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
rcu_idle_enter(); rcu_idle_enter();
drv->states[index].enter_s2idle(dev, drv, index); target_state->enter_s2idle(dev, drv, index);
if (WARN_ON_ONCE(!irqs_disabled())) if (WARN_ON_ONCE(!irqs_disabled()))
local_irq_disable(); local_irq_disable();
/* /*
...@@ -162,6 +164,7 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv, ...@@ -162,6 +164,7 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv,
* first CPU executing it calls functions containing RCU read-side * first CPU executing it calls functions containing RCU read-side
* critical sections, so tell RCU about that. * critical sections, so tell RCU about that.
*/ */
if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
rcu_idle_exit(); rcu_idle_exit();
tick_unfreeze(); tick_unfreeze();
start_critical_timings(); start_critical_timings();
...@@ -239,8 +242,10 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, ...@@ -239,8 +242,10 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
time_start = ns_to_ktime(local_clock()); time_start = ns_to_ktime(local_clock());
stop_critical_timings(); stop_critical_timings();
if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
rcu_idle_enter(); rcu_idle_enter();
entered_state = target_state->enter(dev, drv, index); entered_state = target_state->enter(dev, drv, index);
if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
rcu_idle_exit(); rcu_idle_exit();
start_critical_timings(); start_critical_timings();
......
...@@ -82,6 +82,7 @@ struct cpuidle_state { ...@@ -82,6 +82,7 @@ struct cpuidle_state {
#define CPUIDLE_FLAG_UNUSABLE BIT(3) /* avoid using this state */ #define CPUIDLE_FLAG_UNUSABLE BIT(3) /* avoid using this state */
#define CPUIDLE_FLAG_OFF BIT(4) /* disable this state by default */ #define CPUIDLE_FLAG_OFF BIT(4) /* disable this state by default */
#define CPUIDLE_FLAG_TLB_FLUSHED BIT(5) /* idle-state flushes TLBs */ #define CPUIDLE_FLAG_TLB_FLUSHED BIT(5) /* idle-state flushes TLBs */
#define CPUIDLE_FLAG_RCU_IDLE BIT(6) /* idle-state takes care of RCU */
struct cpuidle_device_kobj; struct cpuidle_device_kobj;
struct cpuidle_state_kobj; struct cpuidle_state_kobj;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment