Commit a26ac245 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney

rcu: move TREE_RCU from softirq to kthread

If RCU priority boosting is to be meaningful, callback invocation must
be boosted in addition to preempted RCU readers.  Otherwise, in presence
of CPU real-time threads, the grace period ends, but the callbacks don't
get invoked.  If the callbacks don't get invoked, the associated memory
doesn't get freed, so the system is still subject to OOM.

But it is not reasonable to priority-boost RCU_SOFTIRQ, so this commit
moves the callback invocations to a kthread, which can be boosted easily.

Also add comments and properly synchronized all accesses to
rcu_cpu_kthread_task, as suggested by Lai Jiangshan.
Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: default avatarJosh Triplett <josh@joshtriplett.org>
parent 12f5f524
...@@ -836,7 +836,6 @@ Provides counts of softirq handlers serviced since boot time, for each cpu. ...@@ -836,7 +836,6 @@ Provides counts of softirq handlers serviced since boot time, for each cpu.
TASKLET: 0 0 0 290 TASKLET: 0 0 0 290
SCHED: 27035 26983 26971 26746 SCHED: 27035 26983 26971 26746
HRTIMER: 0 0 0 0 HRTIMER: 0 0 0 0
RCU: 1678 1769 2178 2250
1.3 IDE devices in /proc/ide 1.3 IDE devices in /proc/ide
......
...@@ -414,7 +414,6 @@ enum ...@@ -414,7 +414,6 @@ enum
TASKLET_SOFTIRQ, TASKLET_SOFTIRQ,
SCHED_SOFTIRQ, SCHED_SOFTIRQ,
HRTIMER_SOFTIRQ, HRTIMER_SOFTIRQ,
RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */
NR_SOFTIRQS NR_SOFTIRQS
}; };
......
...@@ -20,8 +20,7 @@ struct softirq_action; ...@@ -20,8 +20,7 @@ struct softirq_action;
softirq_name(BLOCK_IOPOLL), \ softirq_name(BLOCK_IOPOLL), \
softirq_name(TASKLET), \ softirq_name(TASKLET), \
softirq_name(SCHED), \ softirq_name(SCHED), \
softirq_name(HRTIMER), \ softirq_name(HRTIMER))
softirq_name(RCU))
/** /**
* irq_handler_entry - called immediately before the irq action handler * irq_handler_entry - called immediately before the irq action handler
......
This diff is collapsed.
...@@ -111,6 +111,7 @@ struct rcu_node { ...@@ -111,6 +111,7 @@ struct rcu_node {
/* elements that need to drain to allow the */ /* elements that need to drain to allow the */
/* current expedited grace period to */ /* current expedited grace period to */
/* complete (only for TREE_PREEMPT_RCU). */ /* complete (only for TREE_PREEMPT_RCU). */
unsigned long wakemask; /* CPUs whose kthread needs to be awakened. */
unsigned long qsmaskinit; unsigned long qsmaskinit;
/* Per-GP initial value for qsmask & expmask. */ /* Per-GP initial value for qsmask & expmask. */
unsigned long grpmask; /* Mask to apply to parent qsmask. */ unsigned long grpmask; /* Mask to apply to parent qsmask. */
...@@ -134,6 +135,13 @@ struct rcu_node { ...@@ -134,6 +135,13 @@ struct rcu_node {
/* if there is no such task. If there */ /* if there is no such task. If there */
/* is no current expedited grace period, */ /* is no current expedited grace period, */
/* then there can cannot be any such task. */ /* then there can cannot be any such task. */
struct task_struct *node_kthread_task;
/* kthread that takes care of this rcu_node */
/* structure, for example, awakening the */
/* per-CPU kthreads as needed. */
wait_queue_head_t node_wq;
/* Wait queue on which to park the per-node */
/* kthread. */
} ____cacheline_internodealigned_in_smp; } ____cacheline_internodealigned_in_smp;
/* /*
......
...@@ -1206,7 +1206,7 @@ static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff); ...@@ -1206,7 +1206,7 @@ static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
* *
* Because it is not legal to invoke rcu_process_callbacks() with irqs * Because it is not legal to invoke rcu_process_callbacks() with irqs
* disabled, we do one pass of force_quiescent_state(), then do a * disabled, we do one pass of force_quiescent_state(), then do a
* raise_softirq() to cause rcu_process_callbacks() to be invoked later. * invoke_rcu_kthread() to cause rcu_process_callbacks() to be invoked later.
* The per-cpu rcu_dyntick_drain variable controls the sequencing. * The per-cpu rcu_dyntick_drain variable controls the sequencing.
*/ */
int rcu_needs_cpu(int cpu) int rcu_needs_cpu(int cpu)
...@@ -1257,7 +1257,7 @@ int rcu_needs_cpu(int cpu) ...@@ -1257,7 +1257,7 @@ int rcu_needs_cpu(int cpu)
/* If RCU callbacks are still pending, RCU still needs this CPU. */ /* If RCU callbacks are still pending, RCU still needs this CPU. */
if (c) if (c)
raise_softirq(RCU_SOFTIRQ); invoke_rcu_kthread();
return c; return c;
} }
......
...@@ -58,7 +58,7 @@ DEFINE_PER_CPU(struct task_struct *, ksoftirqd); ...@@ -58,7 +58,7 @@ DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
char *softirq_to_name[NR_SOFTIRQS] = { char *softirq_to_name[NR_SOFTIRQS] = {
"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL", "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
"TASKLET", "SCHED", "HRTIMER", "RCU" "TASKLET", "SCHED", "HRTIMER"
}; };
/* /*
......
...@@ -2187,7 +2187,6 @@ static const struct flag flags[] = { ...@@ -2187,7 +2187,6 @@ static const struct flag flags[] = {
{ "TASKLET_SOFTIRQ", 6 }, { "TASKLET_SOFTIRQ", 6 },
{ "SCHED_SOFTIRQ", 7 }, { "SCHED_SOFTIRQ", 7 },
{ "HRTIMER_SOFTIRQ", 8 }, { "HRTIMER_SOFTIRQ", 8 },
{ "RCU_SOFTIRQ", 9 },
{ "HRTIMER_NORESTART", 0 }, { "HRTIMER_NORESTART", 0 },
{ "HRTIMER_RESTART", 1 }, { "HRTIMER_RESTART", 1 },
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment