Commit f9a25f77 authored by Mathieu Poirier's avatar Mathieu Poirier Committed by Ingo Molnar

cpusets: Rebuild root domain deadline accounting information

When the topology of root domains is modified by CPUset or CPUhotplug
operations information about the current deadline bandwidth held in the
root domain is lost.

This patch addresses the issue by recalculating the lost deadline
bandwidth information by circling through the deadline tasks held in
CPUsets and adding their current load to the root domain they are
associated with.
Tested-by: default avatarDietmar Eggemann <dietmar.eggemann@arm.com>
Signed-off-by: default avatarMathieu Poirier <mathieu.poirier@linaro.org>
Signed-off-by: default avatarJuri Lelli <juri.lelli@redhat.com>
[ Various additional modifications. ]
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: bristot@redhat.com
Cc: claudio@evidence.eu.com
Cc: lizefan@huawei.com
Cc: longman@redhat.com
Cc: luca.abeni@santannapisa.it
Cc: rostedt@goodmis.org
Cc: tj@kernel.org
Cc: tommaso.cucinotta@santannapisa.it
Link: https://lkml.kernel.org/r/20190719140000.31694-4-juri.lelli@redhat.comSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 4b211f2b
...@@ -150,6 +150,7 @@ struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset, ...@@ -150,6 +150,7 @@ struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset,
struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset, struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset,
struct cgroup_subsys_state **dst_cssp); struct cgroup_subsys_state **dst_cssp);
void cgroup_enable_task_cg_lists(void);
void css_task_iter_start(struct cgroup_subsys_state *css, unsigned int flags, void css_task_iter_start(struct cgroup_subsys_state *css, unsigned int flags,
struct css_task_iter *it); struct css_task_iter *it);
struct task_struct *css_task_iter_next(struct css_task_iter *it); struct task_struct *css_task_iter_next(struct css_task_iter *it);
......
...@@ -295,6 +295,11 @@ enum uclamp_id { ...@@ -295,6 +295,11 @@ enum uclamp_id {
UCLAMP_CNT UCLAMP_CNT
}; };
#ifdef CONFIG_SMP
extern struct root_domain def_root_domain;
extern struct mutex sched_domains_mutex;
#endif
struct sched_info { struct sched_info {
#ifdef CONFIG_SCHED_INFO #ifdef CONFIG_SCHED_INFO
/* Cumulative counters: */ /* Cumulative counters: */
......
...@@ -24,3 +24,11 @@ static inline bool dl_time_before(u64 a, u64 b) ...@@ -24,3 +24,11 @@ static inline bool dl_time_before(u64 a, u64 b)
{ {
return (s64)(a - b) < 0; return (s64)(a - b) < 0;
} }
#ifdef CONFIG_SMP
struct root_domain;
extern void dl_add_task_root_domain(struct task_struct *p);
extern void dl_clear_root_domain(struct root_domain *rd);
#endif /* CONFIG_SMP */
...@@ -1891,7 +1891,7 @@ static int cgroup_reconfigure(struct fs_context *fc) ...@@ -1891,7 +1891,7 @@ static int cgroup_reconfigure(struct fs_context *fc)
*/ */
static bool use_task_css_set_links __read_mostly; static bool use_task_css_set_links __read_mostly;
static void cgroup_enable_task_cg_lists(void) void cgroup_enable_task_cg_lists(void)
{ {
struct task_struct *p, *g; struct task_struct *p, *g;
......
...@@ -45,6 +45,7 @@ ...@@ -45,6 +45,7 @@
#include <linux/proc_fs.h> #include <linux/proc_fs.h>
#include <linux/rcupdate.h> #include <linux/rcupdate.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/sched/deadline.h>
#include <linux/sched/mm.h> #include <linux/sched/mm.h>
#include <linux/sched/task.h> #include <linux/sched/task.h>
#include <linux/seq_file.h> #include <linux/seq_file.h>
...@@ -894,6 +895,67 @@ static int generate_sched_domains(cpumask_var_t **domains, ...@@ -894,6 +895,67 @@ static int generate_sched_domains(cpumask_var_t **domains,
return ndoms; return ndoms;
} }
static void update_tasks_root_domain(struct cpuset *cs)
{
struct css_task_iter it;
struct task_struct *task;
css_task_iter_start(&cs->css, 0, &it);
while ((task = css_task_iter_next(&it)))
dl_add_task_root_domain(task);
css_task_iter_end(&it);
}
static void rebuild_root_domains(void)
{
struct cpuset *cs = NULL;
struct cgroup_subsys_state *pos_css;
lockdep_assert_held(&cpuset_mutex);
lockdep_assert_cpus_held();
lockdep_assert_held(&sched_domains_mutex);
cgroup_enable_task_cg_lists();
rcu_read_lock();
/*
* Clear default root domain DL accounting, it will be computed again
* if a task belongs to it.
*/
dl_clear_root_domain(&def_root_domain);
cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
if (cpumask_empty(cs->effective_cpus)) {
pos_css = css_rightmost_descendant(pos_css);
continue;
}
css_get(&cs->css);
rcu_read_unlock();
update_tasks_root_domain(cs);
rcu_read_lock();
css_put(&cs->css);
}
rcu_read_unlock();
}
static void
partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
struct sched_domain_attr *dattr_new)
{
mutex_lock(&sched_domains_mutex);
partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);
rebuild_root_domains();
mutex_unlock(&sched_domains_mutex);
}
/* /*
* Rebuild scheduler domains. * Rebuild scheduler domains.
* *
...@@ -931,7 +993,7 @@ static void rebuild_sched_domains_locked(void) ...@@ -931,7 +993,7 @@ static void rebuild_sched_domains_locked(void)
ndoms = generate_sched_domains(&doms, &attr); ndoms = generate_sched_domains(&doms, &attr);
/* Have scheduler rebuild the domains */ /* Have scheduler rebuild the domains */
partition_sched_domains(ndoms, doms, attr); partition_and_rebuild_sched_domains(ndoms, doms, attr);
out: out:
put_online_cpus(); put_online_cpus();
} }
......
...@@ -2283,6 +2283,36 @@ void __init init_sched_dl_class(void) ...@@ -2283,6 +2283,36 @@ void __init init_sched_dl_class(void)
GFP_KERNEL, cpu_to_node(i)); GFP_KERNEL, cpu_to_node(i));
} }
void dl_add_task_root_domain(struct task_struct *p)
{
struct rq_flags rf;
struct rq *rq;
struct dl_bw *dl_b;
rq = task_rq_lock(p, &rf);
if (!dl_task(p))
goto unlock;
dl_b = &rq->rd->dl_bw;
raw_spin_lock(&dl_b->lock);
__dl_add(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span));
raw_spin_unlock(&dl_b->lock);
unlock:
task_rq_unlock(rq, p, &rf);
}
void dl_clear_root_domain(struct root_domain *rd)
{
unsigned long flags;
raw_spin_lock_irqsave(&rd->dl_bw.lock, flags);
rd->dl_bw.total_bw = 0;
raw_spin_unlock_irqrestore(&rd->dl_bw.lock, flags);
}
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
static void switched_from_dl(struct rq *rq, struct task_struct *p) static void switched_from_dl(struct rq *rq, struct task_struct *p)
......
...@@ -778,9 +778,6 @@ struct root_domain { ...@@ -778,9 +778,6 @@ struct root_domain {
struct perf_domain __rcu *pd; struct perf_domain __rcu *pd;
}; };
extern struct root_domain def_root_domain;
extern struct mutex sched_domains_mutex;
extern void init_defrootdomain(void); extern void init_defrootdomain(void);
extern int sched_init_domains(const struct cpumask *cpu_map); extern int sched_init_domains(const struct cpumask *cpu_map);
extern void rq_attach_root(struct rq *rq, struct root_domain *rd); extern void rq_attach_root(struct rq *rq, struct root_domain *rd);
......
...@@ -2203,8 +2203,19 @@ void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new[], ...@@ -2203,8 +2203,19 @@ void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new[],
for (i = 0; i < ndoms_cur; i++) { for (i = 0; i < ndoms_cur; i++) {
for (j = 0; j < n && !new_topology; j++) { for (j = 0; j < n && !new_topology; j++) {
if (cpumask_equal(doms_cur[i], doms_new[j]) && if (cpumask_equal(doms_cur[i], doms_new[j]) &&
dattrs_equal(dattr_cur, i, dattr_new, j)) dattrs_equal(dattr_cur, i, dattr_new, j)) {
struct root_domain *rd;
/*
* This domain won't be destroyed and as such
* its dl_bw->total_bw needs to be cleared. It
* will be recomputed in function
* update_tasks_root_domain().
*/
rd = cpu_rq(cpumask_any(doms_cur[i]))->rd;
dl_clear_root_domain(rd);
goto match1; goto match1;
}
} }
/* No match - a current sched domain not in new doms_new[] */ /* No match - a current sched domain not in new doms_new[] */
detach_destroy_domains(doms_cur[i]); detach_destroy_domains(doms_cur[i]);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment