Commit 38ff667b authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar

perf_counter: fix update_userpage()

It just occured to me it is possible to have multiple contending
updates of the userpage (mmap information vs overflow vs counter).
This would break the seqlock logic.

It appear the arch code uses this from NMI context, so we cannot
possibly serialize its use, therefore separate the data_head update
from it and let it return to its original use.

The arch code needs to make sure there are no contending callers by
disabling the counter before using it -- powerpc appears to do this
nicely.
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: default avatarPaul Mackerras <paulus@samba.org>
Orig-LKML-Reference: <20090330171023.241410660@chello.nl>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 925d519a
...@@ -160,10 +160,45 @@ struct perf_counter_hw_event { ...@@ -160,10 +160,45 @@ struct perf_counter_hw_event {
struct perf_counter_mmap_page { struct perf_counter_mmap_page {
__u32 version; /* version number of this structure */ __u32 version; /* version number of this structure */
__u32 compat_version; /* lowest version this is compat with */ __u32 compat_version; /* lowest version this is compat with */
/*
* Bits needed to read the hw counters in user-space.
*
* The index and offset should be read atomically using the seqlock:
*
* __u32 seq, index;
* __s64 offset;
*
* again:
* rmb();
* seq = pc->lock;
*
* if (unlikely(seq & 1)) {
* cpu_relax();
* goto again;
* }
*
* index = pc->index;
* offset = pc->offset;
*
* rmb();
* if (pc->lock != seq)
* goto again;
*
* After this, index contains architecture specific counter index + 1,
* so that 0 means unavailable, offset contains the value to be added
* to the result of the raw timer read to obtain this counter's value.
*/
__u32 lock; /* seqlock for synchronization */ __u32 lock; /* seqlock for synchronization */
__u32 index; /* hardware counter identifier */ __u32 index; /* hardware counter identifier */
__s64 offset; /* add to hardware counter value */ __s64 offset; /* add to hardware counter value */
/*
* Control data for the mmap() data buffer.
*
* User-space reading this value should issue an rmb(), on SMP capable
* platforms, after reading this value -- see perf_counter_wakeup().
*/
__u32 data_head; /* head in the data section */ __u32 data_head; /* head in the data section */
}; };
......
...@@ -1316,10 +1316,22 @@ static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) ...@@ -1316,10 +1316,22 @@ static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
return err; return err;
} }
static void __perf_counter_update_userpage(struct perf_counter *counter, /*
struct perf_mmap_data *data) * Callers need to ensure there can be no nesting of this function, otherwise
* the seqlock logic goes bad. We can not serialize this because the arch
* code calls this from NMI context.
*/
void perf_counter_update_userpage(struct perf_counter *counter)
{ {
struct perf_counter_mmap_page *userpg = data->user_page; struct perf_mmap_data *data;
struct perf_counter_mmap_page *userpg;
rcu_read_lock();
data = rcu_dereference(counter->data);
if (!data)
goto unlock;
userpg = data->user_page;
/* /*
* Disable preemption so as to not let the corresponding user-space * Disable preemption so as to not let the corresponding user-space
...@@ -1333,20 +1345,10 @@ static void __perf_counter_update_userpage(struct perf_counter *counter, ...@@ -1333,20 +1345,10 @@ static void __perf_counter_update_userpage(struct perf_counter *counter,
if (counter->state == PERF_COUNTER_STATE_ACTIVE) if (counter->state == PERF_COUNTER_STATE_ACTIVE)
userpg->offset -= atomic64_read(&counter->hw.prev_count); userpg->offset -= atomic64_read(&counter->hw.prev_count);
userpg->data_head = atomic_read(&data->head);
smp_wmb(); smp_wmb();
++userpg->lock; ++userpg->lock;
preempt_enable(); preempt_enable();
} unlock:
void perf_counter_update_userpage(struct perf_counter *counter)
{
struct perf_mmap_data *data;
rcu_read_lock();
data = rcu_dereference(counter->data);
if (data)
__perf_counter_update_userpage(counter, data);
rcu_read_unlock(); rcu_read_unlock();
} }
...@@ -1547,7 +1549,13 @@ void perf_counter_wakeup(struct perf_counter *counter) ...@@ -1547,7 +1549,13 @@ void perf_counter_wakeup(struct perf_counter *counter)
data = rcu_dereference(counter->data); data = rcu_dereference(counter->data);
if (data) { if (data) {
(void)atomic_xchg(&data->wakeup, POLL_IN); (void)atomic_xchg(&data->wakeup, POLL_IN);
__perf_counter_update_userpage(counter, data); /*
* Ensure all data writes are issued before updating the
* user-space data head information. The matching rmb()
* will be in userspace after reading this value.
*/
smp_wmb();
data->user_page->data_head = atomic_read(&data->head);
} }
rcu_read_unlock(); rcu_read_unlock();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment