Commit 20e9751b authored by Oleg Nesterov's avatar Oleg Nesterov Committed by Linus Torvalds

[PATCH] rcu: simplify/improve batch tuning

Kill a hard-to-calculate 'rsinterval' boot parameter and per-cpu
rcu_data.last_rs_qlen.  Instead, it adds adds a flag rcu_ctrlblk.signaled,
which records the fact that one of CPUs has sent a resched IPI since the
last rcu_start_batch().

Roughly speaking, we need two rcu_start_batch()s in order to move callbacks
from ->nxtlist to ->donelist.  This means that when ->qlen exceeds qhimark
and continues to grow, we should send a resched IPI, and then do it again
after we gone through a quiescent state.

On the other hand, if it was already sent, we don't need to do it again
when another CPU detects overflow of the queue.
Signed-off-by: default avatarOleg Nesterov <oleg@tv-sign.ru>
Acked-by: default avatarPaul E. McKenney <paulmck@us.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 4b6c2cca
...@@ -1357,10 +1357,6 @@ and is between 256 and 4096 characters. It is defined in the file ...@@ -1357,10 +1357,6 @@ and is between 256 and 4096 characters. It is defined in the file
rcu.qlowmark= [KNL,BOOT] Set threshold of queued rcu.qlowmark= [KNL,BOOT] Set threshold of queued
RCU callbacks below which batch limiting is re-enabled. RCU callbacks below which batch limiting is re-enabled.
rcu.rsinterval= [KNL,BOOT,SMP] Set the number of additional
RCU callbacks to queued before forcing reschedule
on all cpus.
rdinit= [KNL] rdinit= [KNL]
Format: <full_path> Format: <full_path>
Run specified binary instead of /init from the ramdisk, Run specified binary instead of /init from the ramdisk,
......
...@@ -66,6 +66,8 @@ struct rcu_ctrlblk { ...@@ -66,6 +66,8 @@ struct rcu_ctrlblk {
long completed; /* Number of the last completed batch */ long completed; /* Number of the last completed batch */
int next_pending; /* Is the next batch already waiting? */ int next_pending; /* Is the next batch already waiting? */
int signaled;
spinlock_t lock ____cacheline_internodealigned_in_smp; spinlock_t lock ____cacheline_internodealigned_in_smp;
cpumask_t cpumask; /* CPUs that need to switch in order */ cpumask_t cpumask; /* CPUs that need to switch in order */
/* for current batch to proceed. */ /* for current batch to proceed. */
...@@ -106,9 +108,6 @@ struct rcu_data { ...@@ -106,9 +108,6 @@ struct rcu_data {
long blimit; /* Upper limit on a processed batch */ long blimit; /* Upper limit on a processed batch */
int cpu; int cpu;
struct rcu_head barrier; struct rcu_head barrier;
#ifdef CONFIG_SMP
long last_rs_qlen; /* qlen during the last resched */
#endif
}; };
DECLARE_PER_CPU(struct rcu_data, rcu_data); DECLARE_PER_CPU(struct rcu_data, rcu_data);
......
...@@ -71,9 +71,6 @@ static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL}; ...@@ -71,9 +71,6 @@ static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL};
static int blimit = 10; static int blimit = 10;
static int qhimark = 10000; static int qhimark = 10000;
static int qlowmark = 100; static int qlowmark = 100;
#ifdef CONFIG_SMP
static int rsinterval = 1000;
#endif
static atomic_t rcu_barrier_cpu_count; static atomic_t rcu_barrier_cpu_count;
static DEFINE_MUTEX(rcu_barrier_mutex); static DEFINE_MUTEX(rcu_barrier_mutex);
...@@ -86,8 +83,8 @@ static void force_quiescent_state(struct rcu_data *rdp, ...@@ -86,8 +83,8 @@ static void force_quiescent_state(struct rcu_data *rdp,
int cpu; int cpu;
cpumask_t cpumask; cpumask_t cpumask;
set_need_resched(); set_need_resched();
if (unlikely(rdp->qlen - rdp->last_rs_qlen > rsinterval)) { if (unlikely(!rcp->signaled)) {
rdp->last_rs_qlen = rdp->qlen; rcp->signaled = 1;
/* /*
* Don't send IPI to itself. With irqs disabled, * Don't send IPI to itself. With irqs disabled,
* rdp->cpu is the current cpu. * rdp->cpu is the current cpu.
...@@ -301,6 +298,7 @@ static void rcu_start_batch(struct rcu_ctrlblk *rcp) ...@@ -301,6 +298,7 @@ static void rcu_start_batch(struct rcu_ctrlblk *rcp)
smp_mb(); smp_mb();
cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask); cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask);
rcp->signaled = 0;
} }
} }
...@@ -628,9 +626,6 @@ void synchronize_rcu(void) ...@@ -628,9 +626,6 @@ void synchronize_rcu(void)
module_param(blimit, int, 0); module_param(blimit, int, 0);
module_param(qhimark, int, 0); module_param(qhimark, int, 0);
module_param(qlowmark, int, 0); module_param(qlowmark, int, 0);
#ifdef CONFIG_SMP
module_param(rsinterval, int, 0);
#endif
EXPORT_SYMBOL_GPL(rcu_batches_completed); EXPORT_SYMBOL_GPL(rcu_batches_completed);
EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
EXPORT_SYMBOL_GPL(call_rcu); EXPORT_SYMBOL_GPL(call_rcu);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment