Commit 1c1b8a7b authored by Dietmar Eggemann's avatar Dietmar Eggemann Committed by Ingo Molnar

sched/fair: Replace source_load() & target_load() with weighted_cpuload()

With LB_BIAS disabled, source_load() & target_load() return
weighted_cpuload(). Replace both with calls to weighted_cpuload().

The function to obtain the load index (sd->*_idx) for an sd,
get_sd_load_idx(), can be removed as well.

Finally, get rid of the sched feature LB_BIAS.
Signed-off-by: default avatarDietmar Eggemann <dietmar.eggemann@arm.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarRik van Riel <riel@surriel.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Morten Rasmussen <morten.rasmussen@arm.com>
Cc: Patrick Bellasi <patrick.bellasi@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Quentin Perret <quentin.perret@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Valentin Schneider <valentin.schneider@arm.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://lkml.kernel.org/r/20190527062116.11512-3-dietmar.eggemann@arm.comSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 5e83eafb
...@@ -1467,8 +1467,6 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page, ...@@ -1467,8 +1467,6 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
} }
static unsigned long weighted_cpuload(struct rq *rq); static unsigned long weighted_cpuload(struct rq *rq);
static unsigned long source_load(int cpu, int type);
static unsigned long target_load(int cpu, int type);
/* Cached statistics for all CPUs within a node */ /* Cached statistics for all CPUs within a node */
struct numa_stats { struct numa_stats {
...@@ -5333,45 +5331,11 @@ static struct { ...@@ -5333,45 +5331,11 @@ static struct {
#endif /* CONFIG_NO_HZ_COMMON */ #endif /* CONFIG_NO_HZ_COMMON */
/* Used instead of source_load when we know the type == 0 */
static unsigned long weighted_cpuload(struct rq *rq) static unsigned long weighted_cpuload(struct rq *rq)
{ {
return cfs_rq_runnable_load_avg(&rq->cfs); return cfs_rq_runnable_load_avg(&rq->cfs);
} }
/*
* Return a low guess at the load of a migration-source CPU weighted
* according to the scheduling class and "nice" value.
*
* We want to under-estimate the load of migration sources, to
* balance conservatively.
*/
static unsigned long source_load(int cpu, int type)
{
struct rq *rq = cpu_rq(cpu);
unsigned long total = weighted_cpuload(rq);
if (type == 0 || !sched_feat(LB_BIAS))
return total;
return min(rq->cpu_load[type-1], total);
}
/*
* Return a high guess at the load of a migration-target CPU weighted
* according to the scheduling class and "nice" value.
*/
static unsigned long target_load(int cpu, int type)
{
struct rq *rq = cpu_rq(cpu);
unsigned long total = weighted_cpuload(rq);
if (type == 0 || !sched_feat(LB_BIAS))
return total;
return max(rq->cpu_load[type-1], total);
}
static unsigned long capacity_of(int cpu) static unsigned long capacity_of(int cpu)
{ {
return cpu_rq(cpu)->cpu_capacity; return cpu_rq(cpu)->cpu_capacity;
...@@ -5479,7 +5443,7 @@ wake_affine_weight(struct sched_domain *sd, struct task_struct *p, ...@@ -5479,7 +5443,7 @@ wake_affine_weight(struct sched_domain *sd, struct task_struct *p,
s64 this_eff_load, prev_eff_load; s64 this_eff_load, prev_eff_load;
unsigned long task_load; unsigned long task_load;
this_eff_load = target_load(this_cpu, sd->wake_idx); this_eff_load = weighted_cpuload(cpu_rq(this_cpu));
if (sync) { if (sync) {
unsigned long current_load = task_h_load(current); unsigned long current_load = task_h_load(current);
...@@ -5497,7 +5461,7 @@ wake_affine_weight(struct sched_domain *sd, struct task_struct *p, ...@@ -5497,7 +5461,7 @@ wake_affine_weight(struct sched_domain *sd, struct task_struct *p,
this_eff_load *= 100; this_eff_load *= 100;
this_eff_load *= capacity_of(prev_cpu); this_eff_load *= capacity_of(prev_cpu);
prev_eff_load = source_load(prev_cpu, sd->wake_idx); prev_eff_load = weighted_cpuload(cpu_rq(prev_cpu));
prev_eff_load -= task_load; prev_eff_load -= task_load;
if (sched_feat(WA_BIAS)) if (sched_feat(WA_BIAS))
prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2;
...@@ -5558,14 +5522,10 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, ...@@ -5558,14 +5522,10 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p,
unsigned long this_runnable_load = ULONG_MAX; unsigned long this_runnable_load = ULONG_MAX;
unsigned long min_avg_load = ULONG_MAX, this_avg_load = ULONG_MAX; unsigned long min_avg_load = ULONG_MAX, this_avg_load = ULONG_MAX;
unsigned long most_spare = 0, this_spare = 0; unsigned long most_spare = 0, this_spare = 0;
int load_idx = sd->forkexec_idx;
int imbalance_scale = 100 + (sd->imbalance_pct-100)/2; int imbalance_scale = 100 + (sd->imbalance_pct-100)/2;
unsigned long imbalance = scale_load_down(NICE_0_LOAD) * unsigned long imbalance = scale_load_down(NICE_0_LOAD) *
(sd->imbalance_pct-100) / 100; (sd->imbalance_pct-100) / 100;
if (sd_flag & SD_BALANCE_WAKE)
load_idx = sd->wake_idx;
do { do {
unsigned long load, avg_load, runnable_load; unsigned long load, avg_load, runnable_load;
unsigned long spare_cap, max_spare_cap; unsigned long spare_cap, max_spare_cap;
...@@ -5589,12 +5549,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, ...@@ -5589,12 +5549,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p,
max_spare_cap = 0; max_spare_cap = 0;
for_each_cpu(i, sched_group_span(group)) { for_each_cpu(i, sched_group_span(group)) {
/* Bias balancing toward CPUs of our domain */ load = weighted_cpuload(cpu_rq(i));
if (local_group)
load = source_load(i, load_idx);
else
load = target_load(i, load_idx);
runnable_load += load; runnable_load += load;
avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs); avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs);
...@@ -7676,34 +7631,6 @@ static inline void init_sd_lb_stats(struct sd_lb_stats *sds) ...@@ -7676,34 +7631,6 @@ static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
}; };
} }
/**
* get_sd_load_idx - Obtain the load index for a given sched domain.
* @sd: The sched_domain whose load_idx is to be obtained.
* @idle: The idle status of the CPU for whose sd load_idx is obtained.
*
* Return: The load index.
*/
static inline int get_sd_load_idx(struct sched_domain *sd,
enum cpu_idle_type idle)
{
int load_idx;
switch (idle) {
case CPU_NOT_IDLE:
load_idx = sd->busy_idx;
break;
case CPU_NEWLY_IDLE:
load_idx = sd->newidle_idx;
break;
default:
load_idx = sd->idle_idx;
break;
}
return load_idx;
}
static unsigned long scale_rt_capacity(struct sched_domain *sd, int cpu) static unsigned long scale_rt_capacity(struct sched_domain *sd, int cpu)
{ {
struct rq *rq = cpu_rq(cpu); struct rq *rq = cpu_rq(cpu);
...@@ -7992,9 +7919,6 @@ static inline void update_sg_lb_stats(struct lb_env *env, ...@@ -7992,9 +7919,6 @@ static inline void update_sg_lb_stats(struct lb_env *env,
struct sg_lb_stats *sgs, struct sg_lb_stats *sgs,
int *sg_status) int *sg_status)
{ {
int local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(group));
int load_idx = get_sd_load_idx(env->sd, env->idle);
unsigned long load;
int i, nr_running; int i, nr_running;
memset(sgs, 0, sizeof(*sgs)); memset(sgs, 0, sizeof(*sgs));
...@@ -8005,13 +7929,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, ...@@ -8005,13 +7929,7 @@ static inline void update_sg_lb_stats(struct lb_env *env,
if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false)) if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false))
env->flags |= LBF_NOHZ_AGAIN; env->flags |= LBF_NOHZ_AGAIN;
/* Bias balancing toward CPUs of our domain: */ sgs->group_load += weighted_cpuload(rq);
if (local_group)
load = target_load(i, load_idx);
else
load = source_load(i, load_idx);
sgs->group_load += load;
sgs->group_util += cpu_util(i); sgs->group_util += cpu_util(i);
sgs->sum_nr_running += rq->cfs.h_nr_running; sgs->sum_nr_running += rq->cfs.h_nr_running;
......
...@@ -39,7 +39,6 @@ SCHED_FEAT(WAKEUP_PREEMPTION, true) ...@@ -39,7 +39,6 @@ SCHED_FEAT(WAKEUP_PREEMPTION, true)
SCHED_FEAT(HRTICK, false) SCHED_FEAT(HRTICK, false)
SCHED_FEAT(DOUBLE_TICK, false) SCHED_FEAT(DOUBLE_TICK, false)
SCHED_FEAT(LB_BIAS, false)
/* /*
* Decrement CPU capacity based on time not spent running tasks * Decrement CPU capacity based on time not spent running tasks
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment