Handling multiple concurrent probe users.

Event naming pattern changed to $(eventname)_bcc_$(pid)
Detect /sys/kernel/debug/tracing/instances in bpf_attach_probe,
if it exist, then will per-instance event, if failed the make global
event instance as same as before.

src/cc/BPF.cc

@@ -151,6 +151,7 @@ StatusTuple BPF::attach_kprobe(const std::string& kernel_func,
                                pid_t pid, int cpu, int group_fd,
                                perf_reader_cb cb, void* cb_cookie) {
   std::string probe_event = get_kprobe_event(kernel_func, attach_type);
+  probe_event += "_bcc_" + std::to_string((long)getpid());
   if (kprobes_.find(probe_event) != kprobes_.end())
     return StatusTuple(-1, "kprobe %s already attached", probe_event.c_str());
 
@@ -495,7 +496,7 @@ StatusTuple BPF::detach_kprobe_event(const std::string& event,
   }
   TRY2(unload_func(attr.func));
   std::string detach_event = "-:kprobes/" + event;
-  if (bpf_detach_kprobe(detach_event.c_str()) < 0)
+  if (bpf_detach_kprobe(detach_event.c_str(), event.c_str()) < 0)
     return StatusTuple(-1, "Unable to detach kprobe %s", event.c_str());
   return StatusTuple(0);
 }
@@ -508,7 +509,7 @@ StatusTuple BPF::detach_uprobe_event(const std::string& event,
   }
   TRY2(unload_func(attr.func));
   std::string detach_event = "-:uprobes/" + event;
-  if (bpf_detach_uprobe(detach_event.c_str()) < 0)
+  if (bpf_detach_uprobe(detach_event.c_str(), event.c_str()) < 0)
     return StatusTuple(-1, "Unable to detach uprobe %s", event.c_str());
   return StatusTuple(0);
 }

src/cc/libbpf.c

@@ -35,6 +35,8 @@
 #include <sys/resource.h>
 #include <unistd.h>
 #include <stdbool.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 
 #include "libbpf.h"
 #include "perf_reader.h"
@@ -364,10 +366,22 @@ static void * bpf_attach_probe(int progfd, const char *event,
   }
   close(kfd);
 
+  if (access("/sys/kernel/debug/tracing/instances", F_OK) != -1) {
+    snprintf(buf, sizeof(buf), "/sys/kernel/debug/tracing/instances/%s", event);
+    if (mkdir(buf, 0755) == -1) 
+      goto retry;
+    snprintf(buf, sizeof(buf), "/sys/kernel/debug/tracing/instances/%s/events/%ss/%s", event, event_type, event);
+    if (bpf_attach_tracing_event(progfd, buf, reader, pid, cpu, group_fd) == 0)
+	  goto out;
+    snprintf(buf, sizeof(buf), "/sys/kernel/debug/tracing/instances/%s", event);
+    rmdir(buf);
+  }
+retry:
   snprintf(buf, sizeof(buf), "/sys/kernel/debug/tracing/events/%ss/%s", event_type, event);
   if (bpf_attach_tracing_event(progfd, buf, reader, pid, cpu, group_fd) < 0)
     goto error;
 
+out:
   return reader;
 
 error:
@@ -389,7 +403,7 @@ void * bpf_attach_uprobe(int progfd, const char *event,
   return bpf_attach_probe(progfd, event, event_desc, "uprobe", pid, cpu, group_fd, cb, cb_cookie);
 }
 
-static int bpf_detach_probe(const char *event_desc, const char *event_type) {
+static int bpf_detach_probe(const char *event_desc, const char *event_type, const char *event) {
   int kfd;
 
   char buf[256];
@@ -406,16 +420,20 @@ static int bpf_detach_probe(const char *event_desc, const char *event_type) {
     return -1;
   }
   close(kfd);
+  snprintf(buf, sizeof(buf), "/sys/kernel/debug/tracing/instances/%s", event);
+  if (access(buf, F_OK) != -1) {
+    rmdir(buf);
+  }
 
   return 0;
 }
 
-int bpf_detach_kprobe(const char *event_desc) {
-  return bpf_detach_probe(event_desc, "kprobe");
+int bpf_detach_kprobe(const char *event_desc, const char *event) {
+  return bpf_detach_probe(event_desc, "kprobe", event);
 }
 
-int bpf_detach_uprobe(const char *event_desc) {
-  return bpf_detach_probe(event_desc, "uprobe");
+int bpf_detach_uprobe(const char *event_desc, const char *event) {
+  return bpf_detach_probe(event_desc, "uprobe", event);
 }
 
 void * bpf_attach_tracepoint(int progfd, const char *tp_category,

src/cc/libbpf.h

@@ -47,12 +47,12 @@ typedef void (*perf_reader_raw_cb)(void *cb_cookie, void *raw, int raw_size);
 void * bpf_attach_kprobe(int progfd, const char *event, const char *event_desc,
                          int pid, int cpu, int group_fd, perf_reader_cb cb,
                          void *cb_cookie);
-int bpf_detach_kprobe(const char *event_desc);
+int bpf_detach_kprobe(const char *event_desc, const char *event);
 
 void * bpf_attach_uprobe(int progfd, const char *event, const char *event_desc,
                          int pid, int cpu, int group_fd, perf_reader_cb cb,
                          void *cb_cookie);
-int bpf_detach_uprobe(const char *event_desc);
+int bpf_detach_uprobe(const char *event_desc, const char *event);
 
 void * bpf_attach_tracepoint(int progfd, const char *tp_category,
                              const char *tp_name, int pid, int cpu,

src/python/bcc/__init__.py

@@ -459,6 +459,7 @@ class BPF(object):
         self._check_probe_quota(1)
         fn = self.load_func(fn_name, BPF.KPROBE)
         ev_name = "p_" + event.replace("+", "_").replace(".", "_")
+        ev_name += "_bcc_" + str(os.getpid())
         desc = "p:kprobes/%s %s" % (ev_name, event)
         res = lib.bpf_attach_kprobe(fn.fd, ev_name.encode("ascii"),
                 desc.encode("ascii"), pid, cpu, group_fd,
@@ -476,7 +477,7 @@ class BPF(object):
             raise Exception("Kprobe %s is not attached" % event)
         lib.perf_reader_free(self.open_kprobes[ev_name])
         desc = "-:kprobes/%s" % ev_name
-        res = lib.bpf_detach_kprobe(desc.encode("ascii"))
+        res = lib.bpf_detach_kprobe(desc.encode("ascii"), ev_name.encode("ascii"))
         if res < 0:
             raise Exception("Failed to detach BPF from kprobe")
         self._del_kprobe(ev_name)
@@ -498,6 +499,7 @@ class BPF(object):
         self._check_probe_quota(1)
         fn = self.load_func(fn_name, BPF.KPROBE)
         ev_name = "r_" + event.replace("+", "_").replace(".", "_")
+        ev_name += "_bcc_" + str(os.getpid())
         desc = "r:kprobes/%s %s" % (ev_name, event)
         res = lib.bpf_attach_kprobe(fn.fd, ev_name.encode("ascii"),
                 desc.encode("ascii"), pid, cpu, group_fd,
@@ -515,7 +517,7 @@ class BPF(object):
             raise Exception("Kretprobe %s is not attached" % event)
         lib.perf_reader_free(self.open_kprobes[ev_name])
         desc = "-:kprobes/%s" % ev_name
-        res = lib.bpf_detach_kprobe(desc.encode("ascii"))
+        res = lib.bpf_detach_kprobe(desc.encode("ascii"), ev_name.encode("ascii"))
         if res < 0:
             raise Exception("Failed to detach BPF from kprobe")
         self._del_kprobe(ev_name)
@@ -1046,12 +1048,12 @@ class BPF(object):
             # non-string keys here include the perf_events reader
             if isinstance(k, str):
                 desc = "-:kprobes/%s" % k
-                lib.bpf_detach_kprobe(desc.encode("ascii"))
+                lib.bpf_detach_kprobe(desc.encode("ascii"), str(k).encode("ascii"))
             self._del_kprobe(k)
         for k, v in list(self.open_uprobes.items()):
             lib.perf_reader_free(v)
             desc = "-:uprobes/%s" % k
-            lib.bpf_detach_uprobe(desc.encode("ascii"))
+            lib.bpf_detach_uprobe(desc.encode("ascii"), str(k).encode("ascii"))
             self._del_uprobe(k)
         for k, v in self.open_tracepoints.items():
             lib.perf_reader_free(v)

src/python/bcc/libbcc.py

@@ -90,12 +90,12 @@ _RAW_CB_TYPE = ct.CFUNCTYPE(None, ct.py_object, ct.c_void_p, ct.c_int)
 lib.bpf_attach_kprobe.argtypes = [ct.c_int, ct.c_char_p, ct.c_char_p, ct.c_int,
         ct.c_int, ct.c_int, _CB_TYPE, ct.py_object]
 lib.bpf_detach_kprobe.restype = ct.c_int
-lib.bpf_detach_kprobe.argtypes = [ct.c_char_p]
+lib.bpf_detach_kprobe.argtypes = [ct.c_char_p, ct.c_char_p]
 lib.bpf_attach_uprobe.restype = ct.c_void_p
 lib.bpf_attach_uprobe.argtypes = [ct.c_int, ct.c_char_p, ct.c_char_p, ct.c_int,
         ct.c_int, ct.c_int, _CB_TYPE, ct.py_object]
 lib.bpf_detach_uprobe.restype = ct.c_int
-lib.bpf_detach_uprobe.argtypes = [ct.c_char_p]
+lib.bpf_detach_uprobe.argtypes = [ct.c_char_p, ct.c_char_p]
 lib.bpf_attach_tracepoint.restype = ct.c_void_p
 lib.bpf_attach_tracepoint.argtypes = [ct.c_int, ct.c_char_p, ct.c_char_p, ct.c_int,
         ct.c_int, ct.c_int, _CB_TYPE, ct.py_object]