Commit c300ba25 authored by Steven Rostedt's avatar Steven Rostedt Committed by Ingo Molnar

sched_clock: and multiplier for TSC to gtod drift

The sched_clock code currently tries to keep all CPU clocks of all CPUS
somewhat in sync. At every clock tick it records the gtod clock and
uses that and jiffies and the TSC to calculate a CPU clock that tries to
stay in sync with all the other CPUs.

ftrace depends heavily on this timer and it detects when this timer
"jumps".  One problem is that the TSC and the gtod also drift.
When the TSC is 0.1% faster or slower than the gtod it is very noticeable
in ftrace. To help compensate for this, I've added a multiplier that
tries to keep the CPU clock updating at the same rate as the gtod.

I've tried various ways to get it to be in sync and this ended up being
the most reliable. At every scheduler tick we calculate the new multiplier:

  multi = delta_gtod / delta_TSC

This means we perform a 64 bit divide at the tick (once a HZ). A shift
is used to handle the accuracy.

Other methods that failed due to dynamic HZ are:

(not used)  multi += (gtod - tsc) / delta_gtod
(not used)  multi += (gtod - (last_tsc + delta_tsc)) / delta_gtod

as well as other variants.

This code still allows for a slight drift between TSC and gtod, but
it keeps the damage down to a minimum.
Signed-off-by: default avatarSteven Rostedt <srostedt@redhat.com>
Cc: Steven Rostedt <srostedt@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: john stultz <johnstul@us.ibm.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent a83bc47c
...@@ -3,6 +3,9 @@ ...@@ -3,6 +3,9 @@
* *
* Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
* *
* Updates and enhancements:
* Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <srostedt@redhat.com>
*
* Based on code by: * Based on code by:
* Ingo Molnar <mingo@redhat.com> * Ingo Molnar <mingo@redhat.com>
* Guillaume Chazarain <guichaz@gmail.com> * Guillaume Chazarain <guichaz@gmail.com>
...@@ -32,6 +35,11 @@ ...@@ -32,6 +35,11 @@
#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
#define MULTI_SHIFT 15
/* Max is double, Min is 1/2 */
#define MAX_MULTI (2LL << MULTI_SHIFT)
#define MIN_MULTI (1LL << (MULTI_SHIFT-1))
struct sched_clock_data { struct sched_clock_data {
/* /*
* Raw spinlock - this is a special case: this might be called * Raw spinlock - this is a special case: this might be called
...@@ -45,6 +53,7 @@ struct sched_clock_data { ...@@ -45,6 +53,7 @@ struct sched_clock_data {
u64 tick_raw; u64 tick_raw;
u64 tick_gtod; u64 tick_gtod;
u64 clock; u64 clock;
s64 multi;
#ifdef CONFIG_NO_HZ #ifdef CONFIG_NO_HZ
int check_max; int check_max;
#endif #endif
...@@ -79,6 +88,7 @@ void sched_clock_init(void) ...@@ -79,6 +88,7 @@ void sched_clock_init(void)
scd->tick_raw = 0; scd->tick_raw = 0;
scd->tick_gtod = ktime_now; scd->tick_gtod = ktime_now;
scd->clock = ktime_now; scd->clock = ktime_now;
scd->multi = 1 << MULTI_SHIFT;
#ifdef CONFIG_NO_HZ #ifdef CONFIG_NO_HZ
scd->check_max = 1; scd->check_max = 1;
#endif #endif
...@@ -134,8 +144,13 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim ...@@ -134,8 +144,13 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim
WARN_ON_ONCE(!irqs_disabled()); WARN_ON_ONCE(!irqs_disabled());
min_clock = scd->tick_gtod + /*
(delta_jiffies ? delta_jiffies - 1 : 0) * TICK_NSEC; * At schedule tick the clock can be just under the gtod. We don't
* want to push it too prematurely.
*/
min_clock = scd->tick_gtod + (delta_jiffies * TICK_NSEC);
if (min_clock > TICK_NSEC)
min_clock -= TICK_NSEC / 2;
if (unlikely(delta < 0)) { if (unlikely(delta < 0)) {
clock++; clock++;
...@@ -149,6 +164,9 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim ...@@ -149,6 +164,9 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim
*/ */
max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC; max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC;
delta *= scd->multi;
delta >>= MULTI_SHIFT;
if (unlikely(clock + delta > max_clock) && check_max(scd)) { if (unlikely(clock + delta > max_clock) && check_max(scd)) {
if (clock < max_clock) if (clock < max_clock)
clock = max_clock; clock = max_clock;
...@@ -230,6 +248,7 @@ void sched_clock_tick(void) ...@@ -230,6 +248,7 @@ void sched_clock_tick(void)
{ {
struct sched_clock_data *scd = this_scd(); struct sched_clock_data *scd = this_scd();
unsigned long now_jiffies = jiffies; unsigned long now_jiffies = jiffies;
s64 mult, delta_gtod, delta_raw;
u64 now, now_gtod; u64 now, now_gtod;
if (unlikely(!sched_clock_running)) if (unlikely(!sched_clock_running))
...@@ -247,9 +266,23 @@ void sched_clock_tick(void) ...@@ -247,9 +266,23 @@ void sched_clock_tick(void)
* already observe 1 new jiffy; adding a new tick_gtod to that would * already observe 1 new jiffy; adding a new tick_gtod to that would
* increase the clock 2 jiffies. * increase the clock 2 jiffies.
*/ */
scd->tick_jiffies = now_jiffies; delta_gtod = now_gtod - scd->tick_gtod;
delta_raw = now - scd->tick_raw;
if ((long)delta_raw > 0) {
mult = delta_gtod << MULTI_SHIFT;
do_div(mult, delta_raw);
scd->multi = mult;
if (scd->multi > MAX_MULTI)
scd->multi = MAX_MULTI;
else if (scd->multi < MIN_MULTI)
scd->multi = MIN_MULTI;
} else
scd->multi = 1 << MULTI_SHIFT;
scd->tick_raw = now; scd->tick_raw = now;
scd->tick_gtod = now_gtod; scd->tick_gtod = now_gtod;
scd->tick_jiffies = now_jiffies;
__raw_spin_unlock(&scd->lock); __raw_spin_unlock(&scd->lock);
} }
...@@ -279,6 +312,7 @@ void sched_clock_idle_wakeup_event(u64 delta_ns) ...@@ -279,6 +312,7 @@ void sched_clock_idle_wakeup_event(u64 delta_ns)
__raw_spin_lock(&scd->lock); __raw_spin_lock(&scd->lock);
scd->prev_raw = now; scd->prev_raw = now;
scd->clock += delta_ns; scd->clock += delta_ns;
scd->multi = 1 << MULTI_SHIFT;
__raw_spin_unlock(&scd->lock); __raw_spin_unlock(&scd->lock);
touch_softlockup_watchdog(); touch_softlockup_watchdog();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment