Commit 668802c2 authored by Waiman Long's avatar Waiman Long Committed by Thomas Gleixner

tick/broadcast: Reduce lock cacheline contention

It was observed that on an Intel x86 system without the ARAT (Always
running APIC timer) feature and with fairly large number of CPUs as
well as CPUs coming in and out of intel_idle frequently, the lock
contention on the tick_broadcast_lock can become significant.

To reduce contention, the lock is put into its own cacheline and all
the cpumask_var_t variables are put into the __read_mostly section.

Running the SP benchmark of the NAS Parallel Benchmarks on a 4-socket
16-core 32-thread Nehalam system, the performance number improved
from 3353.94 Mop/s to 3469.31 Mop/s when this patch was applied on
a 4.9.6 kernel.  This is a 3.4% improvement.
Signed-off-by: default avatarWaiman Long <longman@redhat.com>
Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Link: http://lkml.kernel.org/r/1485799063-20857-1-git-send-email-longman@redhat.comSigned-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent 9556ad6a
...@@ -649,11 +649,15 @@ static inline size_t cpumask_size(void) ...@@ -649,11 +649,15 @@ static inline size_t cpumask_size(void)
* used. Please use this_cpu_cpumask_var_t in those cases. The direct use * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
* of this_cpu_ptr() or this_cpu_read() will lead to failures when the * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
* other type of cpumask_var_t implementation is configured. * other type of cpumask_var_t implementation is configured.
*
* Please also note that __cpumask_var_read_mostly can be used to declare
* a cpumask_var_t variable itself (not its content) as read mostly.
*/ */
#ifdef CONFIG_CPUMASK_OFFSTACK #ifdef CONFIG_CPUMASK_OFFSTACK
typedef struct cpumask *cpumask_var_t; typedef struct cpumask *cpumask_var_t;
#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x) #define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
#define __cpumask_var_read_mostly __read_mostly
bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags); bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
...@@ -667,6 +671,7 @@ void free_bootmem_cpumask_var(cpumask_var_t mask); ...@@ -667,6 +671,7 @@ void free_bootmem_cpumask_var(cpumask_var_t mask);
typedef struct cpumask cpumask_var_t[1]; typedef struct cpumask cpumask_var_t[1];
#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x) #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
#define __cpumask_var_read_mostly
static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
{ {
......
...@@ -29,12 +29,13 @@ ...@@ -29,12 +29,13 @@
*/ */
static struct tick_device tick_broadcast_device; static struct tick_device tick_broadcast_device;
static cpumask_var_t tick_broadcast_mask; static cpumask_var_t tick_broadcast_mask __cpumask_var_read_mostly;
static cpumask_var_t tick_broadcast_on; static cpumask_var_t tick_broadcast_on __cpumask_var_read_mostly;
static cpumask_var_t tmpmask; static cpumask_var_t tmpmask __cpumask_var_read_mostly;
static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
static int tick_broadcast_forced; static int tick_broadcast_forced;
static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
#ifdef CONFIG_TICK_ONESHOT #ifdef CONFIG_TICK_ONESHOT
static void tick_broadcast_clear_oneshot(int cpu); static void tick_broadcast_clear_oneshot(int cpu);
static void tick_resume_broadcast_oneshot(struct clock_event_device *bc); static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);
...@@ -517,9 +518,9 @@ void tick_resume_broadcast(void) ...@@ -517,9 +518,9 @@ void tick_resume_broadcast(void)
#ifdef CONFIG_TICK_ONESHOT #ifdef CONFIG_TICK_ONESHOT
static cpumask_var_t tick_broadcast_oneshot_mask; static cpumask_var_t tick_broadcast_oneshot_mask __cpumask_var_read_mostly;
static cpumask_var_t tick_broadcast_pending_mask; static cpumask_var_t tick_broadcast_pending_mask __cpumask_var_read_mostly;
static cpumask_var_t tick_broadcast_force_mask; static cpumask_var_t tick_broadcast_force_mask __cpumask_var_read_mostly;
/* /*
* Exposed for debugging: see timer_list.c * Exposed for debugging: see timer_list.c
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment