Commit 72465447 authored by Kirill Tkhai's avatar Kirill Tkhai Committed by Ingo Molnar

sched, nohz: Change rq->nr_running to always use wrappers

Sometimes ->nr_running may cross 2 but interrupt is not being
sent to rq's cpu. In this case we don't reenable the timer.
Looks like this may be the reason for rare unexpected effects,
if nohz is enabled.

Patch replaces all places of direct changing of nr_running
and makes add_nr_running() caring about crossing border.
Signed-off-by: default avatarKirill Tkhai <tkhai@yandex.ru>
Acked-by: default avatarFrederic Weisbecker <fweisbec@gmail.com>
Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/20140508225830.2469.97461.stgit@localhostSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 52a08ef1
...@@ -741,7 +741,7 @@ void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) ...@@ -741,7 +741,7 @@ void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
WARN_ON(!dl_prio(prio)); WARN_ON(!dl_prio(prio));
dl_rq->dl_nr_running++; dl_rq->dl_nr_running++;
inc_nr_running(rq_of_dl_rq(dl_rq)); add_nr_running(rq_of_dl_rq(dl_rq), 1);
inc_dl_deadline(dl_rq, deadline); inc_dl_deadline(dl_rq, deadline);
inc_dl_migration(dl_se, dl_rq); inc_dl_migration(dl_se, dl_rq);
...@@ -755,7 +755,7 @@ void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) ...@@ -755,7 +755,7 @@ void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
WARN_ON(!dl_prio(prio)); WARN_ON(!dl_prio(prio));
WARN_ON(!dl_rq->dl_nr_running); WARN_ON(!dl_rq->dl_nr_running);
dl_rq->dl_nr_running--; dl_rq->dl_nr_running--;
dec_nr_running(rq_of_dl_rq(dl_rq)); sub_nr_running(rq_of_dl_rq(dl_rq), 1);
dec_dl_deadline(dl_rq, dl_se->deadline); dec_dl_deadline(dl_rq, dl_se->deadline);
dec_dl_migration(dl_se, dl_rq); dec_dl_migration(dl_se, dl_rq);
......
...@@ -3325,7 +3325,7 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) ...@@ -3325,7 +3325,7 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
} }
if (!se) if (!se)
rq->nr_running -= task_delta; sub_nr_running(rq, task_delta);
cfs_rq->throttled = 1; cfs_rq->throttled = 1;
cfs_rq->throttled_clock = rq_clock(rq); cfs_rq->throttled_clock = rq_clock(rq);
...@@ -3376,7 +3376,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) ...@@ -3376,7 +3376,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
} }
if (!se) if (!se)
rq->nr_running += task_delta; add_nr_running(rq, task_delta);
/* determine whether we need to wake up potentially idle cpu */ /* determine whether we need to wake up potentially idle cpu */
if (rq->curr == rq->idle && rq->cfs.nr_running) if (rq->curr == rq->idle && rq->cfs.nr_running)
...@@ -3908,7 +3908,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) ...@@ -3908,7 +3908,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
if (!se) { if (!se) {
update_rq_runnable_avg(rq, rq->nr_running); update_rq_runnable_avg(rq, rq->nr_running);
inc_nr_running(rq); add_nr_running(rq, 1);
} }
hrtick_update(rq); hrtick_update(rq);
} }
...@@ -3968,7 +3968,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) ...@@ -3968,7 +3968,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
} }
if (!se) { if (!se) {
dec_nr_running(rq); sub_nr_running(rq, 1);
update_rq_runnable_avg(rq, 1); update_rq_runnable_avg(rq, 1);
} }
hrtick_update(rq); hrtick_update(rq);
......
...@@ -973,7 +973,7 @@ dequeue_top_rt_rq(struct rt_rq *rt_rq) ...@@ -973,7 +973,7 @@ dequeue_top_rt_rq(struct rt_rq *rt_rq)
BUG_ON(!rq->nr_running); BUG_ON(!rq->nr_running);
rq->nr_running -= rt_rq->rt_nr_running; sub_nr_running(rq, rt_rq->rt_nr_running);
rt_rq->rt_queued = 0; rt_rq->rt_queued = 0;
} }
...@@ -989,7 +989,7 @@ enqueue_top_rt_rq(struct rt_rq *rt_rq) ...@@ -989,7 +989,7 @@ enqueue_top_rt_rq(struct rt_rq *rt_rq)
if (rt_rq_throttled(rt_rq) || !rt_rq->rt_nr_running) if (rt_rq_throttled(rt_rq) || !rt_rq->rt_nr_running)
return; return;
rq->nr_running += rt_rq->rt_nr_running; add_nr_running(rq, rt_rq->rt_nr_running);
rt_rq->rt_queued = 1; rt_rq->rt_queued = 1;
} }
......
...@@ -1206,12 +1206,14 @@ extern void update_idle_cpu_load(struct rq *this_rq); ...@@ -1206,12 +1206,14 @@ extern void update_idle_cpu_load(struct rq *this_rq);
extern void init_task_runnable_average(struct task_struct *p); extern void init_task_runnable_average(struct task_struct *p);
static inline void inc_nr_running(struct rq *rq) static inline void add_nr_running(struct rq *rq, unsigned count)
{ {
rq->nr_running++; unsigned prev_nr = rq->nr_running;
rq->nr_running = prev_nr + count;
#ifdef CONFIG_NO_HZ_FULL #ifdef CONFIG_NO_HZ_FULL
if (rq->nr_running == 2) { if (prev_nr < 2 && rq->nr_running >= 2) {
if (tick_nohz_full_cpu(rq->cpu)) { if (tick_nohz_full_cpu(rq->cpu)) {
/* Order rq->nr_running write against the IPI */ /* Order rq->nr_running write against the IPI */
smp_wmb(); smp_wmb();
...@@ -1221,9 +1223,9 @@ static inline void inc_nr_running(struct rq *rq) ...@@ -1221,9 +1223,9 @@ static inline void inc_nr_running(struct rq *rq)
#endif #endif
} }
static inline void dec_nr_running(struct rq *rq) static inline void sub_nr_running(struct rq *rq, unsigned count)
{ {
rq->nr_running--; rq->nr_running -= count;
} }
static inline void rq_last_tick_reset(struct rq *rq) static inline void rq_last_tick_reset(struct rq *rq)
......
...@@ -41,13 +41,13 @@ pick_next_task_stop(struct rq *rq, struct task_struct *prev) ...@@ -41,13 +41,13 @@ pick_next_task_stop(struct rq *rq, struct task_struct *prev)
static void static void
enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags)
{ {
inc_nr_running(rq); add_nr_running(rq, 1);
} }
static void static void
dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags) dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags)
{ {
dec_nr_running(rq); sub_nr_running(rq, 1);
} }
static void yield_task_stop(struct rq *rq) static void yield_task_stop(struct rq *rq)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment