Commit 727c3991 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'sched-urgent-2022-06-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fix from Thomas Gleixner:
 "A single scheduler fix plugging a race between sched_setscheduler()
  and balance_push().

  sched_setscheduler() spliced the balance callbacks accross a lock
  break which makes it possible for an interleaving schedule() to
  observe an empty list"

* tag 'sched-urgent-2022-06-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  sched: Fix balance_push() vs __sched_setscheduler()
parents 4afb6515 04193d59
......@@ -4798,25 +4798,55 @@ static void do_balance_callbacks(struct rq *rq, struct callback_head *head)
static void balance_push(struct rq *rq);
/*
* balance_push_callback is a right abuse of the callback interface and plays
* by significantly different rules.
*
* Where the normal balance_callback's purpose is to be ran in the same context
* that queued it (only later, when it's safe to drop rq->lock again),
* balance_push_callback is specifically targeted at __schedule().
*
* This abuse is tolerated because it places all the unlikely/odd cases behind
* a single test, namely: rq->balance_callback == NULL.
*/
struct callback_head balance_push_callback = {
.next = NULL,
.func = (void (*)(struct callback_head *))balance_push,
};
static inline struct callback_head *splice_balance_callbacks(struct rq *rq)
static inline struct callback_head *
__splice_balance_callbacks(struct rq *rq, bool split)
{
struct callback_head *head = rq->balance_callback;
if (likely(!head))
return NULL;
lockdep_assert_rq_held(rq);
if (head)
/*
* Must not take balance_push_callback off the list when
* splice_balance_callbacks() and balance_callbacks() are not
* in the same rq->lock section.
*
* In that case it would be possible for __schedule() to interleave
* and observe the list empty.
*/
if (split && head == &balance_push_callback)
head = NULL;
else
rq->balance_callback = NULL;
return head;
}
static inline struct callback_head *splice_balance_callbacks(struct rq *rq)
{
return __splice_balance_callbacks(rq, true);
}
static void __balance_callbacks(struct rq *rq)
{
do_balance_callbacks(rq, splice_balance_callbacks(rq));
do_balance_callbacks(rq, __splice_balance_callbacks(rq, false));
}
static inline void balance_callbacks(struct rq *rq, struct callback_head *head)
......
......@@ -1693,6 +1693,11 @@ queue_balance_callback(struct rq *rq,
{
lockdep_assert_rq_held(rq);
/*
* Don't (re)queue an already queued item; nor queue anything when
* balance_push() is active, see the comment with
* balance_push_callback.
*/
if (unlikely(head->next || rq->balance_callback == &balance_push_callback))
return;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment