Commit 2232c2d8 authored by Steven Rostedt's avatar Steven Rostedt Committed by Ingo Molnar

rcu: add support for dynamic ticks and preempt rcu

The PREEMPT-RCU can get stuck if a CPU goes idle and NO_HZ is set. The
idle CPU will not progress the RCU through its grace period and a
synchronize_rcu my get stuck. Without this patch I have a box that will
not boot when PREEMPT_RCU and NO_HZ are set. That same box boots fine
with this patch.

This patch comes from the -rt kernel where it has been tested for
several months.
Signed-off-by: default avatarSteven Rostedt <srostedt@redhat.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent c0f4133b
...@@ -109,6 +109,14 @@ static inline void account_system_vtime(struct task_struct *tsk) ...@@ -109,6 +109,14 @@ static inline void account_system_vtime(struct task_struct *tsk)
} }
#endif #endif
#if defined(CONFIG_PREEMPT_RCU) && defined(CONFIG_NO_HZ)
extern void rcu_irq_enter(void);
extern void rcu_irq_exit(void);
#else
# define rcu_irq_enter() do { } while (0)
# define rcu_irq_exit() do { } while (0)
#endif /* CONFIG_PREEMPT_RCU */
/* /*
* It is safe to do non-atomic ops on ->hardirq_context, * It is safe to do non-atomic ops on ->hardirq_context,
* because NMI handlers may not preempt and the ops are * because NMI handlers may not preempt and the ops are
...@@ -117,6 +125,7 @@ static inline void account_system_vtime(struct task_struct *tsk) ...@@ -117,6 +125,7 @@ static inline void account_system_vtime(struct task_struct *tsk)
*/ */
#define __irq_enter() \ #define __irq_enter() \
do { \ do { \
rcu_irq_enter(); \
account_system_vtime(current); \ account_system_vtime(current); \
add_preempt_count(HARDIRQ_OFFSET); \ add_preempt_count(HARDIRQ_OFFSET); \
trace_hardirq_enter(); \ trace_hardirq_enter(); \
...@@ -135,6 +144,7 @@ extern void irq_enter(void); ...@@ -135,6 +144,7 @@ extern void irq_enter(void);
trace_hardirq_exit(); \ trace_hardirq_exit(); \
account_system_vtime(current); \ account_system_vtime(current); \
sub_preempt_count(HARDIRQ_OFFSET); \ sub_preempt_count(HARDIRQ_OFFSET); \
rcu_irq_exit(); \
} while (0) } while (0)
/* /*
......
...@@ -160,5 +160,8 @@ extern void rcu_restart_cpu(int cpu); ...@@ -160,5 +160,8 @@ extern void rcu_restart_cpu(int cpu);
extern long rcu_batches_completed(void); extern long rcu_batches_completed(void);
extern long rcu_batches_completed_bh(void); extern long rcu_batches_completed_bh(void);
#define rcu_enter_nohz() do { } while (0)
#define rcu_exit_nohz() do { } while (0)
#endif /* __KERNEL__ */ #endif /* __KERNEL__ */
#endif /* __LINUX_RCUCLASSIC_H */ #endif /* __LINUX_RCUCLASSIC_H */
...@@ -82,5 +82,27 @@ extern struct rcupreempt_trace *rcupreempt_trace_cpu(int cpu); ...@@ -82,5 +82,27 @@ extern struct rcupreempt_trace *rcupreempt_trace_cpu(int cpu);
struct softirq_action; struct softirq_action;
#ifdef CONFIG_NO_HZ
DECLARE_PER_CPU(long, dynticks_progress_counter);
static inline void rcu_enter_nohz(void)
{
__get_cpu_var(dynticks_progress_counter)++;
WARN_ON(__get_cpu_var(dynticks_progress_counter) & 0x1);
mb();
}
static inline void rcu_exit_nohz(void)
{
mb();
__get_cpu_var(dynticks_progress_counter)++;
WARN_ON(!(__get_cpu_var(dynticks_progress_counter) & 0x1));
}
#else /* CONFIG_NO_HZ */
#define rcu_enter_nohz() do { } while (0)
#define rcu_exit_nohz() do { } while (0)
#endif /* CONFIG_NO_HZ */
#endif /* __KERNEL__ */ #endif /* __KERNEL__ */
#endif /* __LINUX_RCUPREEMPT_H */ #endif /* __LINUX_RCUPREEMPT_H */
...@@ -23,6 +23,10 @@ ...@@ -23,6 +23,10 @@
* to Suparna Bhattacharya for pushing me completely away * to Suparna Bhattacharya for pushing me completely away
* from atomic instructions on the read side. * from atomic instructions on the read side.
* *
* - Added handling of Dynamic Ticks
* Copyright 2007 - Paul E. Mckenney <paulmck@us.ibm.com>
* - Steven Rostedt <srostedt@redhat.com>
*
* Papers: http://www.rdrop.com/users/paulmck/RCU * Papers: http://www.rdrop.com/users/paulmck/RCU
* *
* Design Document: http://lwn.net/Articles/253651/ * Design Document: http://lwn.net/Articles/253651/
...@@ -409,6 +413,212 @@ static void __rcu_advance_callbacks(struct rcu_data *rdp) ...@@ -409,6 +413,212 @@ static void __rcu_advance_callbacks(struct rcu_data *rdp)
} }
} }
#ifdef CONFIG_NO_HZ
DEFINE_PER_CPU(long, dynticks_progress_counter) = 1;
static DEFINE_PER_CPU(long, rcu_dyntick_snapshot);
static DEFINE_PER_CPU(int, rcu_update_flag);
/**
* rcu_irq_enter - Called from Hard irq handlers and NMI/SMI.
*
* If the CPU was idle with dynamic ticks active, this updates the
* dynticks_progress_counter to let the RCU handling know that the
* CPU is active.
*/
void rcu_irq_enter(void)
{
int cpu = smp_processor_id();
if (per_cpu(rcu_update_flag, cpu))
per_cpu(rcu_update_flag, cpu)++;
/*
* Only update if we are coming from a stopped ticks mode
* (dynticks_progress_counter is even).
*/
if (!in_interrupt() &&
(per_cpu(dynticks_progress_counter, cpu) & 0x1) == 0) {
/*
* The following might seem like we could have a race
* with NMI/SMIs. But this really isn't a problem.
* Here we do a read/modify/write, and the race happens
* when an NMI/SMI comes in after the read and before
* the write. But NMI/SMIs will increment this counter
* twice before returning, so the zero bit will not
* be corrupted by the NMI/SMI which is the most important
* part.
*
* The only thing is that we would bring back the counter
* to a postion that it was in during the NMI/SMI.
* But the zero bit would be set, so the rest of the
* counter would again be ignored.
*
* On return from the IRQ, the counter may have the zero
* bit be 0 and the counter the same as the return from
* the NMI/SMI. If the state machine was so unlucky to
* see that, it still doesn't matter, since all
* RCU read-side critical sections on this CPU would
* have already completed.
*/
per_cpu(dynticks_progress_counter, cpu)++;
/*
* The following memory barrier ensures that any
* rcu_read_lock() primitives in the irq handler
* are seen by other CPUs to follow the above
* increment to dynticks_progress_counter. This is
* required in order for other CPUs to correctly
* determine when it is safe to advance the RCU
* grace-period state machine.
*/
smp_mb(); /* see above block comment. */
/*
* Since we can't determine the dynamic tick mode from
* the dynticks_progress_counter after this routine,
* we use a second flag to acknowledge that we came
* from an idle state with ticks stopped.
*/
per_cpu(rcu_update_flag, cpu)++;
/*
* If we take an NMI/SMI now, they will also increment
* the rcu_update_flag, and will not update the
* dynticks_progress_counter on exit. That is for
* this IRQ to do.
*/
}
}
/**
* rcu_irq_exit - Called from exiting Hard irq context.
*
* If the CPU was idle with dynamic ticks active, update the
* dynticks_progress_counter to put let the RCU handling be
* aware that the CPU is going back to idle with no ticks.
*/
void rcu_irq_exit(void)
{
int cpu = smp_processor_id();
/*
* rcu_update_flag is set if we interrupted the CPU
* when it was idle with ticks stopped.
* Once this occurs, we keep track of interrupt nesting
* because a NMI/SMI could also come in, and we still
* only want the IRQ that started the increment of the
* dynticks_progress_counter to be the one that modifies
* it on exit.
*/
if (per_cpu(rcu_update_flag, cpu)) {
if (--per_cpu(rcu_update_flag, cpu))
return;
/* This must match the interrupt nesting */
WARN_ON(in_interrupt());
/*
* If an NMI/SMI happens now we are still
* protected by the dynticks_progress_counter being odd.
*/
/*
* The following memory barrier ensures that any
* rcu_read_unlock() primitives in the irq handler
* are seen by other CPUs to preceed the following
* increment to dynticks_progress_counter. This
* is required in order for other CPUs to determine
* when it is safe to advance the RCU grace-period
* state machine.
*/
smp_mb(); /* see above block comment. */
per_cpu(dynticks_progress_counter, cpu)++;
WARN_ON(per_cpu(dynticks_progress_counter, cpu) & 0x1);
}
}
static void dyntick_save_progress_counter(int cpu)
{
per_cpu(rcu_dyntick_snapshot, cpu) =
per_cpu(dynticks_progress_counter, cpu);
}
static inline int
rcu_try_flip_waitack_needed(int cpu)
{
long curr;
long snap;
curr = per_cpu(dynticks_progress_counter, cpu);
snap = per_cpu(rcu_dyntick_snapshot, cpu);
smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
/*
* If the CPU remained in dynticks mode for the entire time
* and didn't take any interrupts, NMIs, SMIs, or whatever,
* then it cannot be in the middle of an rcu_read_lock(), so
* the next rcu_read_lock() it executes must use the new value
* of the counter. So we can safely pretend that this CPU
* already acknowledged the counter.
*/
if ((curr == snap) && ((curr & 0x1) == 0))
return 0;
/*
* If the CPU passed through or entered a dynticks idle phase with
* no active irq handlers, then, as above, we can safely pretend
* that this CPU already acknowledged the counter.
*/
if ((curr - snap) > 2 || (snap & 0x1) == 0)
return 0;
/* We need this CPU to explicitly acknowledge the counter flip. */
return 1;
}
static inline int
rcu_try_flip_waitmb_needed(int cpu)
{
long curr;
long snap;
curr = per_cpu(dynticks_progress_counter, cpu);
snap = per_cpu(rcu_dyntick_snapshot, cpu);
smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
/*
* If the CPU remained in dynticks mode for the entire time
* and didn't take any interrupts, NMIs, SMIs, or whatever,
* then it cannot have executed an RCU read-side critical section
* during that time, so there is no need for it to execute a
* memory barrier.
*/
if ((curr == snap) && ((curr & 0x1) == 0))
return 0;
/*
* If the CPU either entered or exited an outermost interrupt,
* SMI, NMI, or whatever handler, then we know that it executed
* a memory barrier when doing so. So we don't need another one.
*/
if (curr != snap)
return 0;
/* We need the CPU to execute a memory barrier. */
return 1;
}
#else /* !CONFIG_NO_HZ */
# define dyntick_save_progress_counter(cpu) do { } while (0)
# define rcu_try_flip_waitack_needed(cpu) (1)
# define rcu_try_flip_waitmb_needed(cpu) (1)
#endif /* CONFIG_NO_HZ */
/* /*
* Get here when RCU is idle. Decide whether we need to * Get here when RCU is idle. Decide whether we need to
* move out of idle state, and return non-zero if so. * move out of idle state, and return non-zero if so.
...@@ -447,8 +657,10 @@ rcu_try_flip_idle(void) ...@@ -447,8 +657,10 @@ rcu_try_flip_idle(void)
/* Now ask each CPU for acknowledgement of the flip. */ /* Now ask each CPU for acknowledgement of the flip. */
for_each_cpu_mask(cpu, rcu_cpu_online_map) for_each_cpu_mask(cpu, rcu_cpu_online_map) {
per_cpu(rcu_flip_flag, cpu) = rcu_flipped; per_cpu(rcu_flip_flag, cpu) = rcu_flipped;
dyntick_save_progress_counter(cpu);
}
return 1; return 1;
} }
...@@ -464,7 +676,8 @@ rcu_try_flip_waitack(void) ...@@ -464,7 +676,8 @@ rcu_try_flip_waitack(void)
RCU_TRACE_ME(rcupreempt_trace_try_flip_a1); RCU_TRACE_ME(rcupreempt_trace_try_flip_a1);
for_each_cpu_mask(cpu, rcu_cpu_online_map) for_each_cpu_mask(cpu, rcu_cpu_online_map)
if (per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) { if (rcu_try_flip_waitack_needed(cpu) &&
per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) {
RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1); RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1);
return 0; return 0;
} }
...@@ -509,8 +722,10 @@ rcu_try_flip_waitzero(void) ...@@ -509,8 +722,10 @@ rcu_try_flip_waitzero(void)
smp_mb(); /* ^^^^^^^^^^^^ */ smp_mb(); /* ^^^^^^^^^^^^ */
/* Call for a memory barrier from each CPU. */ /* Call for a memory barrier from each CPU. */
for_each_cpu_mask(cpu, rcu_cpu_online_map) for_each_cpu_mask(cpu, rcu_cpu_online_map) {
per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed; per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed;
dyntick_save_progress_counter(cpu);
}
RCU_TRACE_ME(rcupreempt_trace_try_flip_z2); RCU_TRACE_ME(rcupreempt_trace_try_flip_z2);
return 1; return 1;
...@@ -528,7 +743,8 @@ rcu_try_flip_waitmb(void) ...@@ -528,7 +743,8 @@ rcu_try_flip_waitmb(void)
RCU_TRACE_ME(rcupreempt_trace_try_flip_m1); RCU_TRACE_ME(rcupreempt_trace_try_flip_m1);
for_each_cpu_mask(cpu, rcu_cpu_online_map) for_each_cpu_mask(cpu, rcu_cpu_online_map)
if (per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) { if (rcu_try_flip_waitmb_needed(cpu) &&
per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) {
RCU_TRACE_ME(rcupreempt_trace_try_flip_me1); RCU_TRACE_ME(rcupreempt_trace_try_flip_me1);
return 0; return 0;
} }
......
...@@ -313,6 +313,7 @@ void irq_exit(void) ...@@ -313,6 +313,7 @@ void irq_exit(void)
/* Make sure that timer wheel updates are propagated */ /* Make sure that timer wheel updates are propagated */
if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched()) if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
tick_nohz_stop_sched_tick(); tick_nohz_stop_sched_tick();
rcu_irq_exit();
#endif #endif
preempt_enable_no_resched(); preempt_enable_no_resched();
} }
......
...@@ -282,6 +282,7 @@ void tick_nohz_stop_sched_tick(void) ...@@ -282,6 +282,7 @@ void tick_nohz_stop_sched_tick(void)
ts->idle_tick = ts->sched_timer.expires; ts->idle_tick = ts->sched_timer.expires;
ts->tick_stopped = 1; ts->tick_stopped = 1;
ts->idle_jiffies = last_jiffies; ts->idle_jiffies = last_jiffies;
rcu_enter_nohz();
} }
/* /*
...@@ -375,6 +376,8 @@ void tick_nohz_restart_sched_tick(void) ...@@ -375,6 +376,8 @@ void tick_nohz_restart_sched_tick(void)
return; return;
} }
rcu_exit_nohz();
/* Update jiffies first */ /* Update jiffies first */
select_nohz_load_balancer(0); select_nohz_load_balancer(0);
now = ktime_get(); now = ktime_get();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment