Commit 21ca6d2c authored by Rafael J. Wysocki's avatar Rafael J. Wysocki

cpufreq: schedutil: Add iowait boosting

Modify the schedutil cpufreq governor to boost the CPU
frequency if the SCHED_CPUFREQ_IOWAIT flag is passed to
it via cpufreq_update_util().

If that happens, the frequency is set to the maximum during
the first update after receiving the SCHED_CPUFREQ_IOWAIT flag
and then the boost is reduced by half during each following update.
Signed-off-by: default avatarRafael J. Wysocki <rafael.j.wysocki@intel.com>
Looks-good-to: Steve Muckle <smuckle@linaro.org>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
parent 8c34ab19
...@@ -47,11 +47,13 @@ struct sugov_cpu { ...@@ -47,11 +47,13 @@ struct sugov_cpu {
struct sugov_policy *sg_policy; struct sugov_policy *sg_policy;
unsigned int cached_raw_freq; unsigned int cached_raw_freq;
unsigned long iowait_boost;
unsigned long iowait_boost_max;
u64 last_update;
/* The fields below are only needed when sharing a policy. */ /* The fields below are only needed when sharing a policy. */
unsigned long util; unsigned long util;
unsigned long max; unsigned long max;
u64 last_update;
unsigned int flags; unsigned int flags;
}; };
...@@ -155,6 +157,36 @@ static void sugov_get_util(unsigned long *util, unsigned long *max) ...@@ -155,6 +157,36 @@ static void sugov_get_util(unsigned long *util, unsigned long *max)
*max = cfs_max; *max = cfs_max;
} }
static void sugov_set_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,
unsigned int flags)
{
if (flags & SCHED_CPUFREQ_IOWAIT) {
sg_cpu->iowait_boost = sg_cpu->iowait_boost_max;
} else if (sg_cpu->iowait_boost) {
s64 delta_ns = time - sg_cpu->last_update;
/* Clear iowait_boost if the CPU apprears to have been idle. */
if (delta_ns > TICK_NSEC)
sg_cpu->iowait_boost = 0;
}
}
static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, unsigned long *util,
unsigned long *max)
{
unsigned long boost_util = sg_cpu->iowait_boost;
unsigned long boost_max = sg_cpu->iowait_boost_max;
if (!boost_util)
return;
if (*util * boost_max < *max * boost_util) {
*util = boost_util;
*max = boost_max;
}
sg_cpu->iowait_boost >>= 1;
}
static void sugov_update_single(struct update_util_data *hook, u64 time, static void sugov_update_single(struct update_util_data *hook, u64 time,
unsigned int flags) unsigned int flags)
{ {
...@@ -164,6 +196,9 @@ static void sugov_update_single(struct update_util_data *hook, u64 time, ...@@ -164,6 +196,9 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
unsigned long util, max; unsigned long util, max;
unsigned int next_f; unsigned int next_f;
sugov_set_iowait_boost(sg_cpu, time, flags);
sg_cpu->last_update = time;
if (!sugov_should_update_freq(sg_policy, time)) if (!sugov_should_update_freq(sg_policy, time))
return; return;
...@@ -171,6 +206,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time, ...@@ -171,6 +206,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
next_f = policy->cpuinfo.max_freq; next_f = policy->cpuinfo.max_freq;
} else { } else {
sugov_get_util(&util, &max); sugov_get_util(&util, &max);
sugov_iowait_boost(sg_cpu, &util, &max);
next_f = get_next_freq(sg_cpu, util, max); next_f = get_next_freq(sg_cpu, util, max);
} }
sugov_update_commit(sg_policy, time, next_f); sugov_update_commit(sg_policy, time, next_f);
...@@ -189,6 +225,8 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, ...@@ -189,6 +225,8 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
if (flags & SCHED_CPUFREQ_RT_DL) if (flags & SCHED_CPUFREQ_RT_DL)
return max_f; return max_f;
sugov_iowait_boost(sg_cpu, &util, &max);
for_each_cpu(j, policy->cpus) { for_each_cpu(j, policy->cpus) {
struct sugov_cpu *j_sg_cpu; struct sugov_cpu *j_sg_cpu;
unsigned long j_util, j_max; unsigned long j_util, j_max;
...@@ -203,12 +241,13 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, ...@@ -203,12 +241,13 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
* frequency update and the time elapsed between the last update * frequency update and the time elapsed between the last update
* of the CPU utilization and the last frequency update is long * of the CPU utilization and the last frequency update is long
* enough, don't take the CPU into account as it probably is * enough, don't take the CPU into account as it probably is
* idle now. * idle now (and clear iowait_boost for it).
*/ */
delta_ns = last_freq_update_time - j_sg_cpu->last_update; delta_ns = last_freq_update_time - j_sg_cpu->last_update;
if (delta_ns > TICK_NSEC) if (delta_ns > TICK_NSEC) {
j_sg_cpu->iowait_boost = 0;
continue; continue;
}
if (j_sg_cpu->flags & SCHED_CPUFREQ_RT_DL) if (j_sg_cpu->flags & SCHED_CPUFREQ_RT_DL)
return max_f; return max_f;
...@@ -218,6 +257,8 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, ...@@ -218,6 +257,8 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
util = j_util; util = j_util;
max = j_max; max = j_max;
} }
sugov_iowait_boost(j_sg_cpu, &util, &max);
} }
return get_next_freq(sg_cpu, util, max); return get_next_freq(sg_cpu, util, max);
...@@ -238,6 +279,8 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time, ...@@ -238,6 +279,8 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
sg_cpu->util = util; sg_cpu->util = util;
sg_cpu->max = max; sg_cpu->max = max;
sg_cpu->flags = flags; sg_cpu->flags = flags;
sugov_set_iowait_boost(sg_cpu, time, flags);
sg_cpu->last_update = time; sg_cpu->last_update = time;
if (sugov_should_update_freq(sg_policy, time)) { if (sugov_should_update_freq(sg_policy, time)) {
...@@ -470,6 +513,8 @@ static int sugov_start(struct cpufreq_policy *policy) ...@@ -470,6 +513,8 @@ static int sugov_start(struct cpufreq_policy *policy)
sg_cpu->flags = SCHED_CPUFREQ_RT; sg_cpu->flags = SCHED_CPUFREQ_RT;
sg_cpu->last_update = 0; sg_cpu->last_update = 0;
sg_cpu->cached_raw_freq = 0; sg_cpu->cached_raw_freq = 0;
sg_cpu->iowait_boost = 0;
sg_cpu->iowait_boost_max = policy->cpuinfo.max_freq;
cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,
sugov_update_shared); sugov_update_shared);
} else { } else {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment