Commit c1ab99d6 authored by Paul E. McKenney's avatar Paul E. McKenney

rcu/nocb: Use build-time no-CBs check in rcu_core()

Currently, rcu_core() invokes rcu_segcblist_is_offloaded() each time it
needs to know whether the current CPU is a no-CBs CPU.  Given that it is
not possible to change the no-CBs status of a CPU after boot, and given
that it is not possible to even have no-CBs CPUs in CONFIG_RCU_NOCB_CPU=n
kernels, this repeated runtime invocation wastes CPU.  This commit
therefore created a const on-stack variable to allow this check to be
done only once per rcu_core() invocation.
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.ibm.com>
parent ec5ef87b
...@@ -2302,6 +2302,8 @@ static __latent_entropy void rcu_core(void) ...@@ -2302,6 +2302,8 @@ static __latent_entropy void rcu_core(void)
unsigned long flags; unsigned long flags;
struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
struct rcu_node *rnp = rdp->mynode; struct rcu_node *rnp = rdp->mynode;
const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
rcu_segcblist_is_offloaded(&rdp->cblist);
if (cpu_is_offline(smp_processor_id())) if (cpu_is_offline(smp_processor_id()))
return; return;
...@@ -2321,8 +2323,7 @@ static __latent_entropy void rcu_core(void) ...@@ -2321,8 +2323,7 @@ static __latent_entropy void rcu_core(void)
/* No grace period and unregistered callbacks? */ /* No grace period and unregistered callbacks? */
if (!rcu_gp_in_progress() && if (!rcu_gp_in_progress() &&
rcu_segcblist_is_enabled(&rdp->cblist) && rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) {
!rcu_segcblist_is_offloaded(&rdp->cblist)) {
local_irq_save(flags); local_irq_save(flags);
if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
rcu_accelerate_cbs_unlocked(rnp, rdp); rcu_accelerate_cbs_unlocked(rnp, rdp);
...@@ -2332,8 +2333,7 @@ static __latent_entropy void rcu_core(void) ...@@ -2332,8 +2333,7 @@ static __latent_entropy void rcu_core(void)
rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
/* If there are callbacks ready, invoke them. */ /* If there are callbacks ready, invoke them. */
if (!rcu_segcblist_is_offloaded(&rdp->cblist) && if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) &&
rcu_segcblist_ready_cbs(&rdp->cblist) &&
likely(READ_ONCE(rcu_scheduler_fully_active))) likely(READ_ONCE(rcu_scheduler_fully_active)))
rcu_do_batch(rdp); rcu_do_batch(rdp);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment