Commit e496187d authored by Patrick Bellasi's avatar Patrick Bellasi Committed by Ingo Molnar

sched/uclamp: Enforce last task's UCLAMP_MAX

When a task sleeps it removes its max utilization clamp from its CPU.
However, the blocked utilization on that CPU can be higher than the max
clamp value enforced while the task was running. This allows undesired
CPU frequency increases while a CPU is idle, for example, when another
CPU on the same frequency domain triggers a frequency update, since
schedutil can now see the full not clamped blocked utilization of the
idle CPU.

Fix this by using:

  uclamp_rq_dec_id(p, rq, UCLAMP_MAX)
    uclamp_rq_max_value(rq, UCLAMP_MAX, clamp_value)

to detect when a CPU has no more RUNNABLE clamped tasks and to flag this
condition.

Don't track any minimum utilization clamps since an idle CPU never
requires a minimum frequency. The decay of the blocked utilization is
good enough to reduce the CPU frequency.
Signed-off-by: default avatarPatrick Bellasi <patrick.bellasi@arm.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Alessio Balsini <balsini@android.com>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Joel Fernandes <joelaf@google.com>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Morten Rasmussen <morten.rasmussen@arm.com>
Cc: Paul Turner <pjt@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Quentin Perret <quentin.perret@arm.com>
Cc: Rafael J . Wysocki <rafael.j.wysocki@intel.com>
Cc: Steve Muckle <smuckle@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Todd Kjos <tkjos@google.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Cc: Viresh Kumar <viresh.kumar@linaro.org>
Link: https://lkml.kernel.org/r/20190621084217.8167-4-patrick.bellasi@arm.comSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 60daf9c1
...@@ -803,8 +803,36 @@ static inline void uclamp_se_set(struct uclamp_se *uc_se, unsigned int value) ...@@ -803,8 +803,36 @@ static inline void uclamp_se_set(struct uclamp_se *uc_se, unsigned int value)
uc_se->bucket_id = uclamp_bucket_id(value); uc_se->bucket_id = uclamp_bucket_id(value);
} }
static inline unsigned int
uclamp_idle_value(struct rq *rq, unsigned int clamp_id,
unsigned int clamp_value)
{
/*
* Avoid blocked utilization pushing up the frequency when we go
* idle (which drops the max-clamp) by retaining the last known
* max-clamp.
*/
if (clamp_id == UCLAMP_MAX) {
rq->uclamp_flags |= UCLAMP_FLAG_IDLE;
return clamp_value;
}
return uclamp_none(UCLAMP_MIN);
}
static inline void uclamp_idle_reset(struct rq *rq, unsigned int clamp_id,
unsigned int clamp_value)
{
/* Reset max-clamp retention only on idle exit */
if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE))
return;
WRITE_ONCE(rq->uclamp[clamp_id].value, clamp_value);
}
static inline static inline
unsigned int uclamp_rq_max_value(struct rq *rq, unsigned int clamp_id) unsigned int uclamp_rq_max_value(struct rq *rq, unsigned int clamp_id,
unsigned int clamp_value)
{ {
struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket;
int bucket_id = UCLAMP_BUCKETS - 1; int bucket_id = UCLAMP_BUCKETS - 1;
...@@ -820,7 +848,7 @@ unsigned int uclamp_rq_max_value(struct rq *rq, unsigned int clamp_id) ...@@ -820,7 +848,7 @@ unsigned int uclamp_rq_max_value(struct rq *rq, unsigned int clamp_id)
} }
/* No tasks -- default clamp values */ /* No tasks -- default clamp values */
return uclamp_none(clamp_id); return uclamp_idle_value(rq, clamp_id, clamp_value);
} }
/* /*
...@@ -845,6 +873,8 @@ static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, ...@@ -845,6 +873,8 @@ static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p,
bucket = &uc_rq->bucket[uc_se->bucket_id]; bucket = &uc_rq->bucket[uc_se->bucket_id];
bucket->tasks++; bucket->tasks++;
uclamp_idle_reset(rq, clamp_id, uc_se->value);
/* /*
* Local max aggregation: rq buckets always track the max * Local max aggregation: rq buckets always track the max
* "requested" clamp value of its RUNNABLE tasks. * "requested" clamp value of its RUNNABLE tasks.
...@@ -871,6 +901,7 @@ static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, ...@@ -871,6 +901,7 @@ static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p,
struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
struct uclamp_se *uc_se = &p->uclamp[clamp_id]; struct uclamp_se *uc_se = &p->uclamp[clamp_id];
struct uclamp_bucket *bucket; struct uclamp_bucket *bucket;
unsigned int bkt_clamp;
unsigned int rq_clamp; unsigned int rq_clamp;
lockdep_assert_held(&rq->lock); lockdep_assert_held(&rq->lock);
...@@ -895,8 +926,10 @@ static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, ...@@ -895,8 +926,10 @@ static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p,
* e.g. due to future modification, warn and fixup the expected value. * e.g. due to future modification, warn and fixup the expected value.
*/ */
SCHED_WARN_ON(bucket->value > rq_clamp); SCHED_WARN_ON(bucket->value > rq_clamp);
if (bucket->value >= rq_clamp) if (bucket->value >= rq_clamp) {
WRITE_ONCE(uc_rq->value, uclamp_rq_max_value(rq, clamp_id)); bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value);
WRITE_ONCE(uc_rq->value, bkt_clamp);
}
} }
static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p)
...@@ -908,6 +941,10 @@ static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) ...@@ -908,6 +941,10 @@ static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p)
for_each_clamp_id(clamp_id) for_each_clamp_id(clamp_id)
uclamp_rq_inc_id(rq, p, clamp_id); uclamp_rq_inc_id(rq, p, clamp_id);
/* Reset clamp idle holding when there is one RUNNABLE task */
if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)
rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE;
} }
static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p)
...@@ -926,8 +963,10 @@ static void __init init_uclamp(void) ...@@ -926,8 +963,10 @@ static void __init init_uclamp(void)
unsigned int clamp_id; unsigned int clamp_id;
int cpu; int cpu;
for_each_possible_cpu(cpu) for_each_possible_cpu(cpu) {
memset(&cpu_rq(cpu)->uclamp, 0, sizeof(struct uclamp_rq)); memset(&cpu_rq(cpu)->uclamp, 0, sizeof(struct uclamp_rq));
cpu_rq(cpu)->uclamp_flags = 0;
}
for_each_clamp_id(clamp_id) { for_each_clamp_id(clamp_id) {
uclamp_se_set(&init_task.uclamp[clamp_id], uclamp_se_set(&init_task.uclamp[clamp_id],
......
...@@ -870,6 +870,8 @@ struct rq { ...@@ -870,6 +870,8 @@ struct rq {
#ifdef CONFIG_UCLAMP_TASK #ifdef CONFIG_UCLAMP_TASK
/* Utilization clamp values based on CPU's RUNNABLE tasks */ /* Utilization clamp values based on CPU's RUNNABLE tasks */
struct uclamp_rq uclamp[UCLAMP_CNT] ____cacheline_aligned; struct uclamp_rq uclamp[UCLAMP_CNT] ____cacheline_aligned;
unsigned int uclamp_flags;
#define UCLAMP_FLAG_IDLE 0x01
#endif #endif
struct cfs_rq cfs; struct cfs_rq cfs;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment