Merge pull request #763 from goldshtn/enhanced-funccount

funccount: Generalized for uprobes, tracepoints, and USDT

man/man8/funccount.8

 .TH funccount 8  "2015-08-18" "USER COMMANDS"
 .SH NAME
-funccount \- Count kernel function calls matching a pattern. Uses Linux eBPF/bcc.
+funccount \- Count function, tracepoint, and USDT probe calls matching a pattern. Uses Linux eBPF/bcc.
 .SH SYNOPSIS
-.B funccount [\-h] [\-p PID] [\-i INTERVAL] [\-T] [\-r] pattern
+.B funccount [\-h] [\-p PID] [\-i INTERVAL] [\-T] [\-r] [\-d] pattern
 .SH DESCRIPTION
-This tool is a quick way to determine which kernel functions are being called,
+This tool is a quick way to determine which functions are being called,
 and at what rate. It uses in-kernel eBPF maps to count function calls.
 
-WARNING: This uses dynamic tracing of (what can be many) kernel functions, an
+WARNING: This uses dynamic tracing of (what can be many) functions, an
 activity that has had issues on some kernel versions (risk of panics or
 freezes). Test, and know what you are doing, before use.
 
@@ -32,6 +32,9 @@ Include timestamps on output.
 .TP
 \-r
 Use regular expressions for the search pattern.
+.TP
+\-d
+Print the BPF program before starting (for debugging purposes).
 .SH EXAMPLES
 .TP
 Count kernel functions beginning with "vfs_", until Ctrl-C is hit:
@@ -53,19 +56,28 @@ Match kernel functions beginning with "vfs_", using regular expressions:
 Count vfs calls for process ID 181 only:
 #
 .B funccount \-p 181 'vfs_*'
-.SH FIELDS
 .TP
-ADDR
-Address of the instruction pointer that was traced (only useful if the FUNC column is suspicious and you would like to double check the translation).
+Count calls to the sched_fork tracepoint, indicating a fork() performed:
+#
+.B funccount t:sched:sched_fork
+.TP
+Count all GC USDT probes in the Node process:
+#
+.B funccount -p 185 u:node:gc*
+.TP
+Count all malloc() calls in libc:
+#
+.B funccount c:malloc
+.SH FIELDS
 .TP
 FUNC
-Kernel function name
+Function name
 .TP
 COUNT
 Number of calls while tracing
 .SH OVERHEAD
-This traces kernel functions and maintains in-kernel counts, which
-are asynchronously copied to user-space. While the rate of kernel calls
+This traces functions and maintains in-kernel counts, which
+are asynchronously copied to user-space. While the rate of calls
 be very high (>1M/sec), this is a relatively efficient way to trace these
 events, and so the overhead is expected to be small for normal workloads.
 Measure in a test environment before use.
@@ -81,6 +93,8 @@ Linux
 .SH STABILITY
 Unstable - in development.
 .SH AUTHOR
-Brendan Gregg
+Brendan Gregg, Sasha Goldshtein
 .SH SEE ALSO
+stackcount(8)
+funclatency(8)
 vfscount(8)

src/python/bcc/__init__.py

@@ -376,7 +376,8 @@ class BPF(object):
                     % (dev, errstr))
         fn.sock = sock
 
-    def _get_kprobe_functions(self, event_re):
+    @staticmethod
+    def get_kprobe_functions(event_re):
         with open("%s/../kprobes/blacklist" % TRACEFS) as blacklist_file:
             blacklist = set([line.rstrip().split()[1] for line in
                     blacklist_file])
@@ -386,7 +387,6 @@ class BPF(object):
                 fn = line.rstrip().split()[0]
                 if re.match(event_re, fn) and fn not in blacklist:
                     fns.append(fn)
-        self._check_probe_quota(len(fns))
         return fns
 
     def _check_probe_quota(self, num_new_probes):
@@ -409,7 +409,9 @@ class BPF(object):
 
         # allow the caller to glob multiple functions together
         if event_re:
-            for line in self._get_kprobe_functions(event_re):
+            matches = BPF.get_kprobe_functions(event_re)
+            self._check_probe_quota(len(matches))
+            for line in matches:
                 try:
                     self.attach_kprobe(event=line, fn_name=fn_name, pid=pid,
                             cpu=cpu, group_fd=group_fd)
@@ -448,7 +450,7 @@ class BPF(object):
 
         # allow the caller to glob multiple functions together
         if event_re:
-            for line in self._get_kprobe_functions(event_re):
+            for line in BPF.get_kprobe_functions(event_re):
                 try:
                     self.attach_kretprobe(event=line, fn_name=fn_name, pid=pid,
                             cpu=cpu, group_fd=group_fd)
@@ -531,7 +533,8 @@ class BPF(object):
         res = lib.bcc_procutils_which_so(libname.encode("ascii"))
         return res if res is None else res.decode()
 
-    def _get_tracepoints(self, tp_re):
+    @staticmethod
+    def get_tracepoints(tp_re):
         results = []
         events_dir = os.path.join(TRACEFS, "events")
         for category in os.listdir(events_dir):
@@ -570,7 +573,7 @@ class BPF(object):
         """
 
         if tp_re:
-            for tp in self._get_tracepoints(tp_re):
+            for tp in BPF.get_tracepoints(tp_re):
                 self.attach_tracepoint(tp=tp, fn_name=fn_name, pid=pid,
                                        cpu=cpu, group_fd=group_fd)
             return
@@ -615,7 +618,13 @@ class BPF(object):
         del self.open_uprobes[name]
         _num_open_probes -= 1
 
-    def _get_user_functions(self, name, sym_re):
+    @staticmethod
+    def get_user_functions(name, sym_re):
+        return set([name for (name, _) in
+                    BPF.get_user_functions_and_addresses(name, sym_re)])
+
+    @staticmethod
+    def get_user_addresses(name, sym_re):
         """
         We are returning addresses here instead of symbol names because it
         turns out that the same name may appear multiple times with different
@@ -624,10 +633,15 @@ class BPF(object):
         it makes sense to return the unique set of addresses that are mapped to
         a symbol that matches the provided regular expression.
         """
+        return set([address for (_, address) in
+                    BPF.get_user_functions_and_addresses(name, sym_re)])
+
+    @staticmethod
+    def get_user_functions_and_addresses(name, sym_re):
         addresses = []
         def sym_cb(sym_name, addr):
-            if re.match(sym_re, sym_name) and addr not in addresses:
-                addresses.append(addr)
+            if re.match(sym_re, sym_name):
+                addresses.append((sym_name, addr))
             return 0
 
         res = lib.bcc_foreach_symbol(name, _SYM_CB_TYPE(sym_cb))
@@ -660,7 +674,9 @@ class BPF(object):
         name = str(name)
 
         if sym_re:
-            for sym_addr in self._get_user_functions(name, sym_re):
+            addresses = BPF.get_user_addresses(name, sym_re)
+            self._check_probe_quota(len(addresses))
+            for sym_addr in addresses:
                 self.attach_uprobe(name=name, addr=sym_addr,
                                    fn_name=fn_name, pid=pid, cpu=cpu,
                                    group_fd=group_fd)
@@ -711,7 +727,7 @@ class BPF(object):
         """
 
         if sym_re:
-            for sym_addr in self._get_user_functions(name, sym_re):
+            for sym_addr in BPF.get_user_addresses(name, sym_re):
                 self.attach_uretprobe(name=name, addr=sym_addr,
                                       fn_name=fn_name, pid=pid, cpu=cpu,
                                       group_fd=group_fd)

tests/python/test_probe_count.py

@@ -67,6 +67,10 @@ class TestProbeQuota(TestCase):
         with self.assertRaises(Exception):
             self.b.attach_kprobe(event_re=".*", fn_name="count")
 
+    def test_uprobe_quota(self):
+        with self.assertRaises(Exception):
+            self.b.attach_uprobe(name="c", sym_re=".*", fn_name="count")
+
     def tearDown(self):
         self.b.cleanup()
 

tools/stackcount.py

@@ -96,10 +96,11 @@ class Probe(object):
                                        pid=self.pid or -1)
             self.matched = self.bpf.num_open_tracepoints()
         elif self.type == "u":
-            pass # Nothing to do -- attach already happened in `load`
+            pass    # Nothing to do -- attach already happened in `load`
 
         if self.matched == 0:
-            raise Exception("No functions matched by pattern %s" % self.pattern)
+            raise Exception("No functions matched by pattern %s" %
+                            self.pattern)
 
     def load(self):
         trace_count_text = """
@@ -142,11 +143,12 @@ BPF_STACK_TRACE(stack_traces, 1024);
             trace_count_text = trace_count_text.replace('GET_PID',
                                         'bpf_get_current_pid_tgid() >> 32')
         else:
-            trace_count_text = trace_count_text.replace('GET_PID', '0xffffffff')
+            trace_count_text = trace_count_text.replace(
+                                    'GET_PID', '0xffffffff')
 
         stack_flags = 'BPF_F_REUSE_STACKID'
         if not self.is_kernel_probe():
-            stack_flags += '| BPF_F_USER_STACK' # can't do both U *and* K
+            stack_flags += '| BPF_F_USER_STACK'     # can't do both U *and* K
         trace_count_text = trace_count_text.replace('STACK_FLAGS', stack_flags)
 
         self.usdt = None
@@ -173,22 +175,22 @@ BPF_STACK_TRACE(stack_traces, 1024);
 
         if debug:
             print(bpf_text)
-        self.bpf = BPF(text=bpf_text, usdt_contexts=
-                      [self.usdt] if self.usdt else [])
+        self.bpf = BPF(text=bpf_text,
+                       usdt_contexts=[self.usdt] if self.usdt else [])
 
 class Tool(object):
     def __init__(self):
         examples = """examples:
-    ./stackcount submit_bio          # count kernel stack traces for submit_bio
-    ./stackcount -s ip_output        # show symbol offsets
-    ./stackcount -sv ip_output       # show offsets and raw addresses (verbose)
-    ./stackcount 'tcp_send*'         # count stacks for funcs matching tcp_send*
-    ./stackcount -r '^tcp_send.*'    # same as above, using regular expressions
-    ./stackcount -Ti 5 ip_output     # output every 5 seconds, with timestamps
-    ./stackcount -p 185 ip_output    # count ip_output stacks for PID 185 only
-    ./stackcount -p 185 c:malloc     # count stacks for malloc in PID 185
-    ./stackcount t:sched:sched_fork  # count stacks for the sched_fork tracepoint
-    ./stackcount -p 185 u:node:*     # count stacks for all USDT probes in node
+    ./stackcount submit_bio         # count kernel stack traces for submit_bio
+    ./stackcount -s ip_output       # show symbol offsets
+    ./stackcount -sv ip_output      # show offsets and raw addresses (verbose)
+    ./stackcount 'tcp_send*'        # count stacks for funcs matching tcp_send*
+    ./stackcount -r '^tcp_send.*'   # same as above, using regular expressions
+    ./stackcount -Ti 5 ip_output    # output every 5 seconds, with timestamps
+    ./stackcount -p 185 ip_output   # count ip_output stacks for PID 185 only
+    ./stackcount -p 185 c:malloc    # count stacks for malloc in PID 185
+    ./stackcount t:sched:sched_fork # count stacks for sched_fork tracepoint
+    ./stackcount -p 185 u:node:*    # count stacks for all USDT probes in node
         """
         parser = argparse.ArgumentParser(
             description="Count events and their stack traces",
@@ -287,4 +289,3 @@ if __name__ == "__main__":
             traceback.print_exc()
         elif sys.exc_info()[0] is not SystemExit:
             print(sys.exc_info()[1])
-