Commit 96fd20cf authored by Ingo Molnar's avatar Ingo Molnar

Merge branch 'rcu/urgent' of...

Merge branch 'rcu/urgent' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu into core/rcu

Pull RCU updates from Paul E. McKenney:

 "This series greatly reduces the performance degradation of Tree SRCU
  on a CPU-hotplug stress test.  The effect was not subtle:  Mike Galbraith
  measured Classic SRCU at 55 seconds and Tree SRCU at more than 16 -minutes-
  for this test.  Mike collected ftrace data that showed that Classic SRCU
  was auto-expediting invocations of synchronize_srcu() that found SRCU
  completely idle.  This series therefore adds this auto-expedite capability
  to Tree SRCU, bringing the performance shortfall to less than ten seconds,
  which is a great improvement over the initial shortfall of 15 minutes."
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parents d160a727 22607d66
......@@ -3779,6 +3779,14 @@
spia_pedr=
spia_peddr=
srcutree.exp_holdoff [KNL]
Specifies how many nanoseconds must elapse
since the end of the last SRCU grace period for
a given srcu_struct until the next normal SRCU
grace period will be considered for automatic
expediting. Set to zero to disable automatic
expediting.
stacktrace [FTRACE]
Enabled the stack tracer on boot up.
......
......@@ -98,4 +98,18 @@ void synchronize_srcu_expedited(struct srcu_struct *sp);
void srcu_barrier(struct srcu_struct *sp);
unsigned long srcu_batches_completed(struct srcu_struct *sp);
static inline void srcutorture_get_gp_data(enum rcutorture_type test_type,
struct srcu_struct *sp, int *flags,
unsigned long *gpnum,
unsigned long *completed)
{
if (test_type != SRCU_FLAVOR)
return;
*flags = 0;
*completed = sp->completed;
*gpnum = *completed;
if (sp->batch_queue.head || sp->batch_check0.head || sp->batch_check0.head)
(*gpnum)++;
}
#endif
......@@ -78,4 +78,16 @@ static inline unsigned long srcu_batches_completed(struct srcu_struct *sp)
return 0;
}
static inline void srcutorture_get_gp_data(enum rcutorture_type test_type,
struct srcu_struct *sp, int *flags,
unsigned long *gpnum,
unsigned long *completed)
{
if (test_type != SRCU_FLAVOR)
return;
*flags = 0;
*completed = sp->srcu_gp_seq;
*gpnum = *completed;
}
#endif
......@@ -43,10 +43,13 @@ struct srcu_data {
spinlock_t lock ____cacheline_internodealigned_in_smp;
struct rcu_segcblist srcu_cblist; /* List of callbacks.*/
unsigned long srcu_gp_seq_needed; /* Furthest future GP needed. */
unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */
bool srcu_cblist_invoking; /* Invoking these CBs? */
struct delayed_work work; /* Context for CB invoking. */
struct rcu_head srcu_barrier_head; /* For srcu_barrier() use. */
struct srcu_node *mynode; /* Leaf srcu_node. */
unsigned long grpmask; /* Mask for leaf srcu_node */
/* ->srcu_data_have_cbs[]. */
int cpu;
struct srcu_struct *sp;
};
......@@ -59,6 +62,9 @@ struct srcu_node {
unsigned long srcu_have_cbs[4]; /* GP seq for children */
/* having CBs, but only */
/* is > ->srcu_gq_seq. */
unsigned long srcu_data_have_cbs[4]; /* Which srcu_data structs */
/* have CBs for given GP? */
unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */
struct srcu_node *srcu_parent; /* Next up in tree. */
int grplo; /* Least CPU for node. */
int grphi; /* Biggest CPU for node. */
......@@ -77,7 +83,8 @@ struct srcu_struct {
unsigned int srcu_idx; /* Current rdr array element. */
unsigned long srcu_gp_seq; /* Grace-period seq #. */
unsigned long srcu_gp_seq_needed; /* Latest gp_seq needed. */
atomic_t srcu_exp_cnt; /* # ongoing expedited GPs. */
unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */
unsigned long srcu_last_gp_end; /* Last GP end timestamp (ns) */
struct srcu_data __percpu *sda; /* Per-CPU srcu_data array. */
unsigned long srcu_barrier_seq; /* srcu_barrier seq #. */
struct mutex srcu_barrier_mutex; /* Serialize barrier ops. */
......@@ -136,4 +143,8 @@ void synchronize_srcu_expedited(struct srcu_struct *sp);
void srcu_barrier(struct srcu_struct *sp);
unsigned long srcu_batches_completed(struct srcu_struct *sp);
void srcutorture_get_gp_data(enum rcutorture_type test_type,
struct srcu_struct *sp, int *flags,
unsigned long *gpnum, unsigned long *completed);
#endif
......@@ -1360,12 +1360,14 @@ rcu_torture_stats_print(void)
cur_ops->stats();
if (rtcv_snap == rcu_torture_current_version &&
rcu_torture_current != NULL) {
int __maybe_unused flags;
unsigned long __maybe_unused gpnum;
unsigned long __maybe_unused completed;
int __maybe_unused flags = 0;
unsigned long __maybe_unused gpnum = 0;
unsigned long __maybe_unused completed = 0;
rcutorture_get_gp_data(cur_ops->ttype,
&flags, &gpnum, &completed);
srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp,
&flags, &gpnum, &completed);
wtp = READ_ONCE(writer_task);
pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx\n",
rcu_torture_writer_state_getname(),
......
This diff is collapsed.
......@@ -704,15 +704,11 @@ void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
default:
break;
}
if (rsp != NULL) {
if (rsp == NULL)
return;
*flags = READ_ONCE(rsp->gp_flags);
*gpnum = READ_ONCE(rsp->gpnum);
*completed = READ_ONCE(rsp->completed);
return;
}
*flags = 0;
*gpnum = 0;
*completed = 0;
}
EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment