Commit 2ee3dc80 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney

rcu: Make RCU_FAST_NO_HZ use timer rather than hrtimer

The RCU_FAST_NO_HZ facility uses an hrtimer to wake up a CPU when
it is allowed to go into dyntick-idle mode, which is almost always
cancelled soon after.  This is not what hrtimers are good at, so
this commit switches to the timer wheel.
Reported-by: default avatarSteven Rostedt <rostedt@goodmis.org>
Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 2fdbb31b
...@@ -1980,9 +1980,7 @@ static void rcu_prepare_for_idle(int cpu) ...@@ -1980,9 +1980,7 @@ static void rcu_prepare_for_idle(int cpu)
static DEFINE_PER_CPU(int, rcu_dyntick_drain); static DEFINE_PER_CPU(int, rcu_dyntick_drain);
static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff); static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
static DEFINE_PER_CPU(struct hrtimer, rcu_idle_gp_timer); static DEFINE_PER_CPU(struct timer_list, rcu_idle_gp_timer);
static ktime_t rcu_idle_gp_wait; /* If some non-lazy callbacks. */
static ktime_t rcu_idle_lazy_gp_wait; /* If only lazy callbacks. */
/* /*
* Allow the CPU to enter dyntick-idle mode if either: (1) There are no * Allow the CPU to enter dyntick-idle mode if either: (1) There are no
...@@ -2051,10 +2049,9 @@ static bool rcu_cpu_has_nonlazy_callbacks(int cpu) ...@@ -2051,10 +2049,9 @@ static bool rcu_cpu_has_nonlazy_callbacks(int cpu)
* real work is done upon re-entry to idle, or by the next scheduling-clock * real work is done upon re-entry to idle, or by the next scheduling-clock
* interrupt should idle not be re-entered. * interrupt should idle not be re-entered.
*/ */
static enum hrtimer_restart rcu_idle_gp_timer_func(struct hrtimer *hrtp) static void rcu_idle_gp_timer_func(unsigned long unused)
{ {
trace_rcu_prep_idle("Timer"); trace_rcu_prep_idle("Timer");
return HRTIMER_NORESTART;
} }
/* /*
...@@ -2062,19 +2059,8 @@ static enum hrtimer_restart rcu_idle_gp_timer_func(struct hrtimer *hrtp) ...@@ -2062,19 +2059,8 @@ static enum hrtimer_restart rcu_idle_gp_timer_func(struct hrtimer *hrtp)
*/ */
static void rcu_prepare_for_idle_init(int cpu) static void rcu_prepare_for_idle_init(int cpu)
{ {
static int firsttime = 1; setup_timer(&per_cpu(rcu_idle_gp_timer, cpu),
struct hrtimer *hrtp = &per_cpu(rcu_idle_gp_timer, cpu); rcu_idle_gp_timer_func, 0);
hrtimer_init(hrtp, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
hrtp->function = rcu_idle_gp_timer_func;
if (firsttime) {
unsigned int upj = jiffies_to_usecs(RCU_IDLE_GP_DELAY);
rcu_idle_gp_wait = ns_to_ktime(upj * (u64)1000);
upj = jiffies_to_usecs(RCU_IDLE_LAZY_GP_DELAY);
rcu_idle_lazy_gp_wait = ns_to_ktime(upj * (u64)1000);
firsttime = 0;
}
} }
/* /*
...@@ -2084,7 +2070,7 @@ static void rcu_prepare_for_idle_init(int cpu) ...@@ -2084,7 +2070,7 @@ static void rcu_prepare_for_idle_init(int cpu)
*/ */
static void rcu_cleanup_after_idle(int cpu) static void rcu_cleanup_after_idle(int cpu)
{ {
hrtimer_cancel(&per_cpu(rcu_idle_gp_timer, cpu)); del_timer(&per_cpu(rcu_idle_gp_timer, cpu));
trace_rcu_prep_idle("Cleanup after idle"); trace_rcu_prep_idle("Cleanup after idle");
} }
...@@ -2141,11 +2127,11 @@ static void rcu_prepare_for_idle(int cpu) ...@@ -2141,11 +2127,11 @@ static void rcu_prepare_for_idle(int cpu)
per_cpu(rcu_dyntick_drain, cpu) = 0; per_cpu(rcu_dyntick_drain, cpu) = 0;
per_cpu(rcu_dyntick_holdoff, cpu) = jiffies; per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
if (rcu_cpu_has_nonlazy_callbacks(cpu)) if (rcu_cpu_has_nonlazy_callbacks(cpu))
hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu), mod_timer(&per_cpu(rcu_idle_gp_timer, cpu),
rcu_idle_gp_wait, HRTIMER_MODE_REL); jiffies + RCU_IDLE_GP_DELAY);
else else
hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu), mod_timer(&per_cpu(rcu_idle_gp_timer, cpu),
rcu_idle_lazy_gp_wait, HRTIMER_MODE_REL); jiffies + RCU_IDLE_LAZY_GP_DELAY);
return; /* Nothing more to do immediately. */ return; /* Nothing more to do immediately. */
} else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) { } else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) {
/* We have hit the limit, so time to give up. */ /* We have hit the limit, so time to give up. */
...@@ -2193,14 +2179,12 @@ static void rcu_prepare_for_idle(int cpu) ...@@ -2193,14 +2179,12 @@ static void rcu_prepare_for_idle(int cpu)
static void print_cpu_stall_fast_no_hz(char *cp, int cpu) static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
{ {
struct hrtimer *hrtp = &per_cpu(rcu_idle_gp_timer, cpu); struct timer_list *tltp = &per_cpu(rcu_idle_gp_timer, cpu);
sprintf(cp, "drain=%d %c timer=%lld", sprintf(cp, "drain=%d %c timer=%lu",
per_cpu(rcu_dyntick_drain, cpu), per_cpu(rcu_dyntick_drain, cpu),
per_cpu(rcu_dyntick_holdoff, cpu) == jiffies ? 'H' : '.', per_cpu(rcu_dyntick_holdoff, cpu) == jiffies ? 'H' : '.',
hrtimer_active(hrtp) timer_pending(tltp) ? tltp->expires - jiffies : -1);
? ktime_to_us(hrtimer_get_remaining(hrtp))
: -1);
} }
#else /* #ifdef CONFIG_RCU_FAST_NO_HZ */ #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment