Commit d4f7dae8 authored by Frederic Weisbecker's avatar Frederic Weisbecker Committed by Thomas Gleixner

timers: Spare timer softirq until next expiry

Now that the core timer infrastructure doesn't depend anymore on
periodic base->clk increments, even when the CPU is not in NO_HZ mode,
timer softirqs can be skipped until there are timers to expire.

Some spurious softirqs can still remain since base->next_expiry doesn't
keep track of canceled timers but this still reduces the number of softirqs
significantly: ~15 times less for HZ=1000 and ~5 times less for HZ=100.
Signed-off-by: default avatarFrederic Weisbecker <frederic@kernel.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Tested-by: default avatarJuri Lelli <juri.lelli@redhat.com>
Link: https://lkml.kernel.org/r/20200717140551.29076-11-frederic@kernel.org
parent 1f8a4212
...@@ -1458,10 +1458,10 @@ static void expire_timers(struct timer_base *base, struct hlist_head *head) ...@@ -1458,10 +1458,10 @@ static void expire_timers(struct timer_base *base, struct hlist_head *head)
} }
} }
static int __collect_expired_timers(struct timer_base *base, static int collect_expired_timers(struct timer_base *base,
struct hlist_head *heads) struct hlist_head *heads)
{ {
unsigned long clk = base->clk; unsigned long clk = base->clk = base->next_expiry;
struct hlist_head *vec; struct hlist_head *vec;
int i, levels = 0; int i, levels = 0;
unsigned int idx; unsigned int idx;
...@@ -1684,40 +1684,6 @@ void timer_clear_idle(void) ...@@ -1684,40 +1684,6 @@ void timer_clear_idle(void)
*/ */
base->is_idle = false; base->is_idle = false;
} }
static int collect_expired_timers(struct timer_base *base,
struct hlist_head *heads)
{
unsigned long now = READ_ONCE(jiffies);
/*
* NOHZ optimization. After a long idle sleep we need to forward the
* base to current jiffies. Avoid a loop by searching the bitfield for
* the next expiring timer.
*/
if ((long)(now - base->clk) > 2) {
/*
* If the next timer is ahead of time forward to current
* jiffies, otherwise forward to the next expiry time:
*/
if (time_after(base->next_expiry, now)) {
/*
* The call site will increment base->clk and then
* terminate the expiry loop immediately.
*/
base->clk = now;
return 0;
}
base->clk = base->next_expiry;
}
return __collect_expired_timers(base, heads);
}
#else
static inline int collect_expired_timers(struct timer_base *base,
struct hlist_head *heads)
{
return __collect_expired_timers(base, heads);
}
#endif #endif
/* /*
...@@ -1750,7 +1716,7 @@ static inline void __run_timers(struct timer_base *base) ...@@ -1750,7 +1716,7 @@ static inline void __run_timers(struct timer_base *base)
struct hlist_head heads[LVL_DEPTH]; struct hlist_head heads[LVL_DEPTH];
int levels; int levels;
if (!time_after_eq(jiffies, base->clk)) if (time_before(jiffies, base->next_expiry))
return; return;
timer_base_lock_expiry(base); timer_base_lock_expiry(base);
...@@ -1763,7 +1729,8 @@ static inline void __run_timers(struct timer_base *base) ...@@ -1763,7 +1729,8 @@ static inline void __run_timers(struct timer_base *base)
*/ */
base->must_forward_clk = false; base->must_forward_clk = false;
while (time_after_eq(jiffies, base->clk)) { while (time_after_eq(jiffies, base->clk) &&
time_after_eq(jiffies, base->next_expiry)) {
levels = collect_expired_timers(base, heads); levels = collect_expired_timers(base, heads);
base->clk++; base->clk++;
...@@ -1798,12 +1765,12 @@ void run_local_timers(void) ...@@ -1798,12 +1765,12 @@ void run_local_timers(void)
hrtimer_run_queues(); hrtimer_run_queues();
/* Raise the softirq only if required. */ /* Raise the softirq only if required. */
if (time_before(jiffies, base->clk)) { if (time_before(jiffies, base->next_expiry)) {
if (!IS_ENABLED(CONFIG_NO_HZ_COMMON)) if (!IS_ENABLED(CONFIG_NO_HZ_COMMON))
return; return;
/* CPU is awake, so check the deferrable base. */ /* CPU is awake, so check the deferrable base. */
base++; base++;
if (time_before(jiffies, base->clk)) if (time_before(jiffies, base->next_expiry))
return; return;
} }
raise_softirq(TIMER_SOFTIRQ); raise_softirq(TIMER_SOFTIRQ);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment