Commit 6563de9d authored by Paul E. McKenney's avatar Paul E. McKenney

rcu: Abstract the dynticks momentary-idle operation

This commit is the first step towards full abstraction of all accesses to
the ->dynticks counter, implementing the previously open-coded atomic add
of two in a new rcu_dynticks_momentary_idle() function.  This abstraction
will ease changes to the ->dynticks counter operation.

Note that this commit gets rid of the smp_mb__before_atomic() and the
smp_mb__after_atomic() calls that were previously present.  The reason
that this is OK from a memory-ordering perspective is that the atomic
operation is now atomic_add_return(), which, as a value-returning atomic,
guarantees full ordering.
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: default avatarJosh Triplett <josh@joshtriplett.org>
parent 52d7e48b
...@@ -281,6 +281,19 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { ...@@ -281,6 +281,19 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
}; };
/*
* Do a double-increment of the ->dynticks counter to emulate a
* momentary idle-CPU quiescent state.
*/
static void rcu_dynticks_momentary_idle(void)
{
struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
int special = atomic_add_return(2, &rdtp->dynticks);
/* It is illegal to call this from idle state. */
WARN_ON_ONCE(!(special & 0x1));
}
DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr); DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr); EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
...@@ -300,7 +313,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr); ...@@ -300,7 +313,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
static void rcu_momentary_dyntick_idle(void) static void rcu_momentary_dyntick_idle(void)
{ {
struct rcu_data *rdp; struct rcu_data *rdp;
struct rcu_dynticks *rdtp;
int resched_mask; int resched_mask;
struct rcu_state *rsp; struct rcu_state *rsp;
...@@ -327,10 +339,7 @@ static void rcu_momentary_dyntick_idle(void) ...@@ -327,10 +339,7 @@ static void rcu_momentary_dyntick_idle(void)
* quiescent state, with no need for this CPU to do anything * quiescent state, with no need for this CPU to do anything
* further. * further.
*/ */
rdtp = this_cpu_ptr(&rcu_dynticks); rcu_dynticks_momentary_idle();
smp_mb__before_atomic(); /* Earlier stuff before QS. */
atomic_add(2, &rdtp->dynticks); /* QS. */
smp_mb__after_atomic(); /* Later stuff after QS. */
break; break;
} }
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment