Commit cf7c9c17 authored by Daniel Thompson's avatar Daniel Thompson Committed by Ingo Molnar

timers, sched/clock: Optimize cache line usage

Currently sched_clock(), a very hot code path, is not optimized
to minimise its cache profile. In particular:

  1. cd is not ____cacheline_aligned,

  2. struct clock_data does not distinguish between hotpath and
     coldpath data, reducing locality of reference in the hotpath,

  3. Some hotpath data is missing from struct clock_data and is marked
     __read_mostly (which more or less guarantees it will not share a
     cache line with cd).

This patch corrects these problems by extracting all hotpath
data into a separate structure and using ____cacheline_aligned
to ensure the hotpath uses a single (64 byte) cache line.
Signed-off-by: default avatarDaniel Thompson <daniel.thompson@linaro.org>
Signed-off-by: default avatarJohn Stultz <john.stultz@linaro.org>
Reviewed-by: default avatarStephen Boyd <sboyd@codeaurora.org>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will.deacon@arm.com>
Link: http://lkml.kernel.org/r/1427397806-20889-3-git-send-email-john.stultz@linaro.orgSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 8710e914
...@@ -18,28 +18,59 @@ ...@@ -18,28 +18,59 @@
#include <linux/seqlock.h> #include <linux/seqlock.h>
#include <linux/bitops.h> #include <linux/bitops.h>
struct clock_data { /**
ktime_t wrap_kt; * struct clock_read_data - data required to read from sched_clock
*
* @epoch_ns: sched_clock value at last update
* @epoch_cyc: Clock cycle value at last update
* @sched_clock_mask: Bitmask for two's complement subtraction of non 64bit
* clocks
* @read_sched_clock: Current clock source (or dummy source when suspended)
* @mult: Multipler for scaled math conversion
* @shift: Shift value for scaled math conversion
* @suspended: Flag to indicate if the clock is suspended (stopped)
*
* Care must be taken when updating this structure; it is read by
* some very hot code paths. It occupies <=48 bytes and, when combined
* with the seqcount used to synchronize access, comfortably fits into
* a 64 byte cache line.
*/
struct clock_read_data {
u64 epoch_ns; u64 epoch_ns;
u64 epoch_cyc; u64 epoch_cyc;
seqcount_t seq; u64 sched_clock_mask;
unsigned long rate; u64 (*read_sched_clock)(void);
u32 mult; u32 mult;
u32 shift; u32 shift;
bool suspended; bool suspended;
}; };
/**
* struct clock_data - all data needed for sched_clock (including
* registration of a new clock source)
*
* @seq: Sequence counter for protecting updates.
* @read_data: Data required to read from sched_clock.
* @wrap_kt: Duration for which clock can run before wrapping
* @rate: Tick rate of the registered clock
* @actual_read_sched_clock: Registered clock read function
*
* The ordering of this structure has been chosen to optimize cache
* performance. In particular seq and read_data (combined) should fit
* into a single 64 byte cache line.
*/
struct clock_data {
seqcount_t seq;
struct clock_read_data read_data;
ktime_t wrap_kt;
unsigned long rate;
};
static struct hrtimer sched_clock_timer; static struct hrtimer sched_clock_timer;
static int irqtime = -1; static int irqtime = -1;
core_param(irqtime, irqtime, int, 0400); core_param(irqtime, irqtime, int, 0400);
static struct clock_data cd = {
.mult = NSEC_PER_SEC / HZ,
};
static u64 __read_mostly sched_clock_mask;
static u64 notrace jiffy_sched_clock_read(void) static u64 notrace jiffy_sched_clock_read(void)
{ {
/* /*
...@@ -49,7 +80,10 @@ static u64 notrace jiffy_sched_clock_read(void) ...@@ -49,7 +80,10 @@ static u64 notrace jiffy_sched_clock_read(void)
return (u64)(jiffies - INITIAL_JIFFIES); return (u64)(jiffies - INITIAL_JIFFIES);
} }
static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read; static struct clock_data cd ____cacheline_aligned = {
.read_data = { .mult = NSEC_PER_SEC / HZ,
.read_sched_clock = jiffy_sched_clock_read, },
};
static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift)
{ {
...@@ -60,15 +94,16 @@ unsigned long long notrace sched_clock(void) ...@@ -60,15 +94,16 @@ unsigned long long notrace sched_clock(void)
{ {
u64 cyc, res; u64 cyc, res;
unsigned long seq; unsigned long seq;
struct clock_read_data *rd = &cd.read_data;
do { do {
seq = raw_read_seqcount_begin(&cd.seq); seq = raw_read_seqcount_begin(&cd.seq);
res = cd.epoch_ns; res = rd->epoch_ns;
if (!cd.suspended) { if (!rd->suspended) {
cyc = read_sched_clock(); cyc = rd->read_sched_clock();
cyc = (cyc - cd.epoch_cyc) & sched_clock_mask; cyc = (cyc - rd->epoch_cyc) & rd->sched_clock_mask;
res += cyc_to_ns(cyc, cd.mult, cd.shift); res += cyc_to_ns(cyc, rd->mult, rd->shift);
} }
} while (read_seqcount_retry(&cd.seq, seq)); } while (read_seqcount_retry(&cd.seq, seq));
...@@ -83,16 +118,17 @@ static void notrace update_sched_clock(void) ...@@ -83,16 +118,17 @@ static void notrace update_sched_clock(void)
unsigned long flags; unsigned long flags;
u64 cyc; u64 cyc;
u64 ns; u64 ns;
struct clock_read_data *rd = &cd.read_data;
cyc = read_sched_clock(); cyc = rd->read_sched_clock();
ns = cd.epoch_ns + ns = rd->epoch_ns +
cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, cyc_to_ns((cyc - rd->epoch_cyc) & rd->sched_clock_mask,
cd.mult, cd.shift); rd->mult, rd->shift);
raw_local_irq_save(flags); raw_local_irq_save(flags);
raw_write_seqcount_begin(&cd.seq); raw_write_seqcount_begin(&cd.seq);
cd.epoch_ns = ns; rd->epoch_ns = ns;
cd.epoch_cyc = cyc; rd->epoch_cyc = cyc;
raw_write_seqcount_end(&cd.seq); raw_write_seqcount_end(&cd.seq);
raw_local_irq_restore(flags); raw_local_irq_restore(flags);
} }
...@@ -111,6 +147,7 @@ void __init sched_clock_register(u64 (*read)(void), int bits, ...@@ -111,6 +147,7 @@ void __init sched_clock_register(u64 (*read)(void), int bits,
u32 new_mult, new_shift; u32 new_mult, new_shift;
unsigned long r; unsigned long r;
char r_unit; char r_unit;
struct clock_read_data *rd = &cd.read_data;
if (cd.rate > rate) if (cd.rate > rate)
return; return;
...@@ -129,17 +166,18 @@ void __init sched_clock_register(u64 (*read)(void), int bits, ...@@ -129,17 +166,18 @@ void __init sched_clock_register(u64 (*read)(void), int bits,
/* update epoch for new counter and update epoch_ns from old counter*/ /* update epoch for new counter and update epoch_ns from old counter*/
new_epoch = read(); new_epoch = read();
cyc = read_sched_clock(); cyc = rd->read_sched_clock();
ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, ns = rd->epoch_ns +
cd.mult, cd.shift); cyc_to_ns((cyc - rd->epoch_cyc) & rd->sched_clock_mask,
rd->mult, rd->shift);
raw_write_seqcount_begin(&cd.seq); raw_write_seqcount_begin(&cd.seq);
read_sched_clock = read; rd->read_sched_clock = read;
sched_clock_mask = new_mask; rd->sched_clock_mask = new_mask;
cd.mult = new_mult; rd->mult = new_mult;
cd.shift = new_shift; rd->shift = new_shift;
cd.epoch_cyc = new_epoch; rd->epoch_cyc = new_epoch;
cd.epoch_ns = ns; rd->epoch_ns = ns;
raw_write_seqcount_end(&cd.seq); raw_write_seqcount_end(&cd.seq);
r = rate; r = rate;
...@@ -171,7 +209,7 @@ void __init sched_clock_postinit(void) ...@@ -171,7 +209,7 @@ void __init sched_clock_postinit(void)
* If no sched_clock function has been provided at that point, * If no sched_clock function has been provided at that point,
* make it the final one one. * make it the final one one.
*/ */
if (read_sched_clock == jiffy_sched_clock_read) if (cd.read_data.read_sched_clock == jiffy_sched_clock_read)
sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ);
update_sched_clock(); update_sched_clock();
...@@ -187,17 +225,21 @@ void __init sched_clock_postinit(void) ...@@ -187,17 +225,21 @@ void __init sched_clock_postinit(void)
static int sched_clock_suspend(void) static int sched_clock_suspend(void)
{ {
struct clock_read_data *rd = &cd.read_data;
update_sched_clock(); update_sched_clock();
hrtimer_cancel(&sched_clock_timer); hrtimer_cancel(&sched_clock_timer);
cd.suspended = true; rd->suspended = true;
return 0; return 0;
} }
static void sched_clock_resume(void) static void sched_clock_resume(void)
{ {
cd.epoch_cyc = read_sched_clock(); struct clock_read_data *rd = &cd.read_data;
rd->epoch_cyc = rd->read_sched_clock();
hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL);
cd.suspended = false; rd->suspended = false;
} }
static struct syscore_ops sched_clock_ops = { static struct syscore_ops sched_clock_ops = {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment