Commit 7b40066c authored by Mathieu Desnoyers's avatar Mathieu Desnoyers Committed by Steven Rostedt (VMware)

tracepoint: Use rcu get state and cond sync for static call updates

State transitions from 1->0->1 and N->2->1 callbacks require RCU
synchronization. Rather than performing the RCU synchronization every
time the state change occurs, which is quite slow when many tracepoints
are registered in batch, instead keep a snapshot of the RCU state on the
most recent transitions which belong to a chain, and conditionally wait
for a grace period on the last transition of the chain if one g.p. has
not elapsed since the last snapshot.

This applies to both RCU and SRCU.

This brings the performance regression caused by commit 231264d6
("Fix: tracepoint: static call function vs data state mismatch") back to
what it was originally.

Before this commit:

  # trace-cmd start -e all
  # time trace-cmd start -p nop

  real	0m10.593s
  user	0m0.017s
  sys	0m0.259s

After this commit:

  # trace-cmd start -e all
  # time trace-cmd start -p nop

  real	0m0.878s
  user	0m0.000s
  sys	0m0.103s

Link: https://lkml.kernel.org/r/20210805192954.30688-1-mathieu.desnoyers@efficios.com
Link: https://lore.kernel.org/io-uring/4ebea8f0-58c9-e571-fd30-0ce4f6f09c70@samba.org/

Cc: stable@vger.kernel.org
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Stefan Metzmacher <metze@samba.org>
Fixes: 231264d6 ("Fix: tracepoint: static call function vs data state mismatch")
Signed-off-by: default avatarMathieu Desnoyers <mathieu.desnoyers@efficios.com>
Reviewed-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Signed-off-by: default avatarSteven Rostedt (VMware) <rostedt@goodmis.org>
parent 231264d6
...@@ -28,6 +28,44 @@ extern tracepoint_ptr_t __stop___tracepoints_ptrs[]; ...@@ -28,6 +28,44 @@ extern tracepoint_ptr_t __stop___tracepoints_ptrs[];
DEFINE_SRCU(tracepoint_srcu); DEFINE_SRCU(tracepoint_srcu);
EXPORT_SYMBOL_GPL(tracepoint_srcu); EXPORT_SYMBOL_GPL(tracepoint_srcu);
enum tp_transition_sync {
TP_TRANSITION_SYNC_1_0_1,
TP_TRANSITION_SYNC_N_2_1,
_NR_TP_TRANSITION_SYNC,
};
struct tp_transition_snapshot {
unsigned long rcu;
unsigned long srcu;
bool ongoing;
};
/* Protected by tracepoints_mutex */
static struct tp_transition_snapshot tp_transition_snapshot[_NR_TP_TRANSITION_SYNC];
static void tp_rcu_get_state(enum tp_transition_sync sync)
{
struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync];
/* Keep the latest get_state snapshot. */
snapshot->rcu = get_state_synchronize_rcu();
snapshot->srcu = start_poll_synchronize_srcu(&tracepoint_srcu);
snapshot->ongoing = true;
}
static void tp_rcu_cond_sync(enum tp_transition_sync sync)
{
struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync];
if (!snapshot->ongoing)
return;
cond_synchronize_rcu(snapshot->rcu);
if (!poll_state_synchronize_srcu(&tracepoint_srcu, snapshot->srcu))
synchronize_srcu(&tracepoint_srcu);
snapshot->ongoing = false;
}
/* Set to 1 to enable tracepoint debug output */ /* Set to 1 to enable tracepoint debug output */
static const int tracepoint_debug; static const int tracepoint_debug;
...@@ -311,6 +349,11 @@ static int tracepoint_add_func(struct tracepoint *tp, ...@@ -311,6 +349,11 @@ static int tracepoint_add_func(struct tracepoint *tp,
*/ */
switch (nr_func_state(tp_funcs)) { switch (nr_func_state(tp_funcs)) {
case TP_FUNC_1: /* 0->1 */ case TP_FUNC_1: /* 0->1 */
/*
* Make sure new static func never uses old data after a
* 1->0->1 transition sequence.
*/
tp_rcu_cond_sync(TP_TRANSITION_SYNC_1_0_1);
/* Set static call to first function */ /* Set static call to first function */
tracepoint_update_call(tp, tp_funcs); tracepoint_update_call(tp, tp_funcs);
/* Both iterator and static call handle NULL tp->funcs */ /* Both iterator and static call handle NULL tp->funcs */
...@@ -325,10 +368,15 @@ static int tracepoint_add_func(struct tracepoint *tp, ...@@ -325,10 +368,15 @@ static int tracepoint_add_func(struct tracepoint *tp,
* Requires ordering between RCU assign/dereference and * Requires ordering between RCU assign/dereference and
* static call update/call. * static call update/call.
*/ */
rcu_assign_pointer(tp->funcs, tp_funcs); fallthrough;
break;
case TP_FUNC_N: /* N->N+1 (N>1) */ case TP_FUNC_N: /* N->N+1 (N>1) */
rcu_assign_pointer(tp->funcs, tp_funcs); rcu_assign_pointer(tp->funcs, tp_funcs);
/*
* Make sure static func never uses incorrect data after a
* N->...->2->1 (N>1) transition sequence.
*/
if (tp_funcs[0].data != old[0].data)
tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
break; break;
default: default:
WARN_ON_ONCE(1); WARN_ON_ONCE(1);
...@@ -372,24 +420,23 @@ static int tracepoint_remove_func(struct tracepoint *tp, ...@@ -372,24 +420,23 @@ static int tracepoint_remove_func(struct tracepoint *tp,
/* Both iterator and static call handle NULL tp->funcs */ /* Both iterator and static call handle NULL tp->funcs */
rcu_assign_pointer(tp->funcs, NULL); rcu_assign_pointer(tp->funcs, NULL);
/* /*
* Make sure new func never uses old data after a 1->0->1 * Make sure new static func never uses old data after a
* transition sequence. * 1->0->1 transition sequence.
* Considering that transition 0->1 is the common case
* and don't have rcu-sync, issue rcu-sync after
* transition 1->0 to break that sequence by waiting for
* readers to be quiescent.
*/ */
tracepoint_synchronize_unregister(); tp_rcu_get_state(TP_TRANSITION_SYNC_1_0_1);
break; break;
case TP_FUNC_1: /* 2->1 */ case TP_FUNC_1: /* 2->1 */
rcu_assign_pointer(tp->funcs, tp_funcs); rcu_assign_pointer(tp->funcs, tp_funcs);
/* /*
* On 2->1 transition, RCU sync is needed before setting * Make sure static func never uses incorrect data after a
* static call to first callback, because the observer * N->...->2->1 (N>2) transition sequence. If the first
* may have loaded any prior tp->funcs after the last one * element's data has changed, then force the synchronization
* associated with an rcu-sync. * to prevent current readers that have loaded the old data
* from calling the new function.
*/ */
tracepoint_synchronize_unregister(); if (tp_funcs[0].data != old[0].data)
tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
tp_rcu_cond_sync(TP_TRANSITION_SYNC_N_2_1);
/* Set static call to first function */ /* Set static call to first function */
tracepoint_update_call(tp, tp_funcs); tracepoint_update_call(tp, tp_funcs);
break; break;
...@@ -397,6 +444,12 @@ static int tracepoint_remove_func(struct tracepoint *tp, ...@@ -397,6 +444,12 @@ static int tracepoint_remove_func(struct tracepoint *tp,
fallthrough; fallthrough;
case TP_FUNC_N: case TP_FUNC_N:
rcu_assign_pointer(tp->funcs, tp_funcs); rcu_assign_pointer(tp->funcs, tp_funcs);
/*
* Make sure static func never uses incorrect data after a
* N->...->2->1 (N>2) transition sequence.
*/
if (tp_funcs[0].data != old[0].data)
tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
break; break;
default: default:
WARN_ON_ONCE(1); WARN_ON_ONCE(1);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment