fix hang with "trace.py --max-events #

Currently running "trace.py --max-events #" (shortname "trace.py -M #")
will hang like below:

  -bash-4.2$ sudo ./trace.py -M 2 'SyS_futex'
  PID     TID     COMM            FUNC
  137727  138229  IOThreadPool0   SyS_futex
  138250  151288  OBC Dispatcher  SyS_futex
  ^C^Z
  [1]+  Stopped                 sudo ./trace.py -M 2 'SyS_futex'
  -bash-4.2$

The hang happens in perf_reader_free.

Commit cd5d4a6c ("fix a race condition between perf_reader
munmap and read") fixed a race condition by adding some coordination
between perf_reader_event_read and perf_reader_free.

In this case, however, the callback function inside perf_reader_event_read
never returns and actually calling exit(). This is because
the maximum number of perf_events have been received. The exit()
is calling BPF object cleanup() which calls perf_reader_free to
free the ring buffer. perf_reader_free got stuck since it thinks
perf_reader_event_reader did not finish yet.

To fix this, a checking for thread_id is added so that
perf_reader_free will proceed without locking if the perf_reader_read
tid is the same as perf_reader_free since this signals that
the callback function calls/triggers perf_reader_free.

After the fix,

  -bash-4.2$ sudo ./trace.py -M 2 'SyS_futex'
  PID     TID     COMM            FUNC
  137727  138178  load-monitor    SyS_futex
  6359    6440    load-monitor    SyS_futex
  -bash-4.2$
Reported-by: Teng Qin <qinteng@fb.com>
Signed-off-by: Yonghong Song <yhs@fb.com>

src/cc/perf_reader.c

@@ -20,8 +20,10 @@
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
+#include <syscall.h>
 #include <sys/ioctl.h>
 #include <sys/mman.h>
+#include <sys/types.h>
 #include <unistd.h>
 #include <linux/perf_event.h>
 
@@ -43,6 +45,7 @@ struct perf_reader {
   size_t buf_size;
   void *base;
   int rb_use_state;
+  pid_t rb_read_tid;
   int page_size;
   int page_cnt;
   int fd;
@@ -70,7 +73,12 @@ struct perf_reader * perf_reader_new(perf_reader_cb cb,
 void perf_reader_free(void *ptr) {
   if (ptr) {
     struct perf_reader *reader = ptr;
-    while (!__sync_bool_compare_and_swap(&reader->rb_use_state, RB_NOT_USED, RB_USED_IN_MUNMAP));
+    pid_t tid = syscall(__NR_gettid);
+    while (!__sync_bool_compare_and_swap(&reader->rb_use_state, RB_NOT_USED, RB_USED_IN_MUNMAP)) {
+      // If the same thread, it is called from call back handler, no locking needed
+      if (tid == reader->rb_read_tid)
+        break;
+    }
     munmap(reader->base, reader->page_size * (reader->page_cnt + 1));
     if (reader->fd >= 0) {
       ioctl(reader->fd, PERF_EVENT_IOC_DISABLE, 0);
@@ -228,6 +236,7 @@ void perf_reader_event_read(struct perf_reader *reader) {
   uint8_t *sentinel = (uint8_t *)reader->base + buffer_size + reader->page_size;
   uint8_t *begin, *end;
 
+  reader->rb_read_tid = syscall(__NR_gettid);
   if (!__sync_bool_compare_and_swap(&reader->rb_use_state, RB_NOT_USED, RB_USED_IN_READ))
     return;
 
@@ -280,6 +289,8 @@ void perf_reader_event_read(struct perf_reader *reader) {
     write_data_tail(perf_header, perf_header->data_tail + e->size);
   }
   reader->rb_use_state = RB_NOT_USED;
+  __sync_synchronize();
+  reader->rb_read_tid = 0;
 }
 
 int perf_reader_poll(int num_readers, struct perf_reader **readers, int timeout) {