Commit 19d23dbf authored by Frederic Weisbecker's avatar Frederic Weisbecker Committed by Ingo Molnar

sched/irqtime: Consolidate accounting synchronization with u64_stats API

The irqtime accounting currently implement its own ad hoc implementation
of u64_stats API. Lets rather consolidate it with the appropriate
library.
Signed-off-by: default avatarFrederic Weisbecker <fweisbec@gmail.com>
Reviewed-by: default avatarRik van Riel <riel@redhat.com>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Wanpeng Li <wanpeng.li@hotmail.com>
Link: http://lkml.kernel.org/r/1474849761-12678-5-git-send-email-fweisbec@gmail.comSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 68107df5
...@@ -23,10 +23,8 @@ ...@@ -23,10 +23,8 @@
* task when irq is in progress while we read rq->clock. That is a worthy * task when irq is in progress while we read rq->clock. That is a worthy
* compromise in place of having locks on each irq in account_system_time. * compromise in place of having locks on each irq in account_system_time.
*/ */
DEFINE_PER_CPU(u64, cpu_hardirq_time); DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
DEFINE_PER_CPU(u64, cpu_softirq_time);
static DEFINE_PER_CPU(u64, irq_start_time);
static int sched_clock_irqtime; static int sched_clock_irqtime;
void enable_sched_clock_irqtime(void) void enable_sched_clock_irqtime(void)
...@@ -39,16 +37,13 @@ void disable_sched_clock_irqtime(void) ...@@ -39,16 +37,13 @@ void disable_sched_clock_irqtime(void)
sched_clock_irqtime = 0; sched_clock_irqtime = 0;
} }
#ifndef CONFIG_64BIT
DEFINE_PER_CPU(seqcount_t, irq_time_seq);
#endif /* CONFIG_64BIT */
/* /*
* Called before incrementing preempt_count on {soft,}irq_enter * Called before incrementing preempt_count on {soft,}irq_enter
* and before decrementing preempt_count on {soft,}irq_exit. * and before decrementing preempt_count on {soft,}irq_exit.
*/ */
void irqtime_account_irq(struct task_struct *curr) void irqtime_account_irq(struct task_struct *curr)
{ {
struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
s64 delta; s64 delta;
int cpu; int cpu;
...@@ -56,10 +51,10 @@ void irqtime_account_irq(struct task_struct *curr) ...@@ -56,10 +51,10 @@ void irqtime_account_irq(struct task_struct *curr)
return; return;
cpu = smp_processor_id(); cpu = smp_processor_id();
delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time); delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
__this_cpu_add(irq_start_time, delta); irqtime->irq_start_time += delta;
irq_time_write_begin(); u64_stats_update_begin(&irqtime->sync);
/* /*
* We do not account for softirq time from ksoftirqd here. * We do not account for softirq time from ksoftirqd here.
* We want to continue accounting softirq time to ksoftirqd thread * We want to continue accounting softirq time to ksoftirqd thread
...@@ -67,11 +62,11 @@ void irqtime_account_irq(struct task_struct *curr) ...@@ -67,11 +62,11 @@ void irqtime_account_irq(struct task_struct *curr)
* that do not consume any time, but still wants to run. * that do not consume any time, but still wants to run.
*/ */
if (hardirq_count()) if (hardirq_count())
__this_cpu_add(cpu_hardirq_time, delta); irqtime->hardirq_time += delta;
else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
__this_cpu_add(cpu_softirq_time, delta); irqtime->softirq_time += delta;
irq_time_write_end(); u64_stats_update_end(&irqtime->sync);
} }
EXPORT_SYMBOL_GPL(irqtime_account_irq); EXPORT_SYMBOL_GPL(irqtime_account_irq);
...@@ -79,9 +74,10 @@ static cputime_t irqtime_account_hi_update(cputime_t maxtime) ...@@ -79,9 +74,10 @@ static cputime_t irqtime_account_hi_update(cputime_t maxtime)
{ {
u64 *cpustat = kcpustat_this_cpu->cpustat; u64 *cpustat = kcpustat_this_cpu->cpustat;
cputime_t irq_cputime; cputime_t irq_cputime;
u64 nsecs;
irq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_hardirq_time)) - nsecs = __this_cpu_read(cpu_irqtime.hardirq_time);
cpustat[CPUTIME_IRQ]; irq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_IRQ];
irq_cputime = min(irq_cputime, maxtime); irq_cputime = min(irq_cputime, maxtime);
cpustat[CPUTIME_IRQ] += irq_cputime; cpustat[CPUTIME_IRQ] += irq_cputime;
...@@ -92,9 +88,10 @@ static cputime_t irqtime_account_si_update(cputime_t maxtime) ...@@ -92,9 +88,10 @@ static cputime_t irqtime_account_si_update(cputime_t maxtime)
{ {
u64 *cpustat = kcpustat_this_cpu->cpustat; u64 *cpustat = kcpustat_this_cpu->cpustat;
cputime_t softirq_cputime; cputime_t softirq_cputime;
u64 nsecs;
softirq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_softirq_time)) - nsecs = __this_cpu_read(cpu_irqtime.softirq_time);
cpustat[CPUTIME_SOFTIRQ]; softirq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_SOFTIRQ];
softirq_cputime = min(softirq_cputime, maxtime); softirq_cputime = min(softirq_cputime, maxtime);
cpustat[CPUTIME_SOFTIRQ] += softirq_cputime; cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/sched/sysctl.h> #include <linux/sched/sysctl.h>
#include <linux/sched/rt.h> #include <linux/sched/rt.h>
#include <linux/u64_stats_sync.h>
#include <linux/sched/deadline.h> #include <linux/sched/deadline.h>
#include <linux/binfmts.h> #include <linux/binfmts.h>
#include <linux/mutex.h> #include <linux/mutex.h>
...@@ -1735,52 +1736,28 @@ static inline void nohz_balance_exit_idle(unsigned int cpu) { } ...@@ -1735,52 +1736,28 @@ static inline void nohz_balance_exit_idle(unsigned int cpu) { }
#endif #endif
#ifdef CONFIG_IRQ_TIME_ACCOUNTING #ifdef CONFIG_IRQ_TIME_ACCOUNTING
struct irqtime {
u64 hardirq_time;
u64 softirq_time;
u64 irq_start_time;
struct u64_stats_sync sync;
};
DECLARE_PER_CPU(u64, cpu_hardirq_time); DECLARE_PER_CPU(struct irqtime, cpu_irqtime);
DECLARE_PER_CPU(u64, cpu_softirq_time);
#ifndef CONFIG_64BIT
DECLARE_PER_CPU(seqcount_t, irq_time_seq);
static inline void irq_time_write_begin(void)
{
__this_cpu_inc(irq_time_seq.sequence);
smp_wmb();
}
static inline void irq_time_write_end(void)
{
smp_wmb();
__this_cpu_inc(irq_time_seq.sequence);
}
static inline u64 irq_time_read(int cpu) static inline u64 irq_time_read(int cpu)
{ {
u64 irq_time; struct irqtime *irqtime = &per_cpu(cpu_irqtime, cpu);
unsigned seq; unsigned int seq;
u64 total;
do { do {
seq = read_seqcount_begin(&per_cpu(irq_time_seq, cpu)); seq = __u64_stats_fetch_begin(&irqtime->sync);
irq_time = per_cpu(cpu_softirq_time, cpu) + total = irqtime->softirq_time + irqtime->hardirq_time;
per_cpu(cpu_hardirq_time, cpu); } while (__u64_stats_fetch_retry(&irqtime->sync, seq));
} while (read_seqcount_retry(&per_cpu(irq_time_seq, cpu), seq));
return irq_time;
}
#else /* CONFIG_64BIT */
static inline void irq_time_write_begin(void)
{
}
static inline void irq_time_write_end(void) return total;
{
}
static inline u64 irq_time_read(int cpu)
{
return per_cpu(cpu_softirq_time, cpu) + per_cpu(cpu_hardirq_time, cpu);
} }
#endif /* CONFIG_64BIT */
#endif /* CONFIG_IRQ_TIME_ACCOUNTING */ #endif /* CONFIG_IRQ_TIME_ACCOUNTING */
#ifdef CONFIG_CPU_FREQ #ifdef CONFIG_CPU_FREQ
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment