Merge branch 'master' into master

CMakeLists.txt

@@ -68,7 +68,12 @@ if(NOT DEFINED BCC_KERNEL_MODULES_SUFFIX)
 endif()
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall")
+# iterate over all available directories in LLVM_INCLUDE_DIRS to
+# generate a correctly tokenized list of parameters
+foreach(ONE_LLVM_INCLUDE_DIR ${LLVM_INCLUDE_DIRS})
+  set(CXX_ISYSTEM_DIRS "${CXX_ISYSTEM_DIRS} -isystem ${ONE_LLVM_INCLUDE_DIR}")
+endforeach()
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall ${CXX_ISYSTEM_DIRS}")
 endif()
 
 add_subdirectory(examples)

CONTRIBUTING-SCRIPTS.md

@@ -15,7 +15,7 @@ More detail for each below.
 
 ## Examples
 
-These are grouped into subdirectories (networking, tracing). Your example can either be a Python program with embedded C (eg, tracing/strlen_count.py), or separate Python and C files (eg, tracing/bitehist.*).
+These are grouped into subdirectories (networking, tracing). Your example can either be a Python program with embedded C (eg, tracing/strlen_count.py), or separate Python and C files (eg, tracing/vfsreadlat.*).
 
 As said earlier: keep it short, neat, and documented (code comments).
 

README.md

@@ -120,6 +120,7 @@ Examples:
 - tools/[tcpconnect](tools/tcpconnect.py): Trace TCP active connections (connect()). [Examples](tools/tcpconnect_example.txt).
 - tools/[tcpconnlat](tools/tcpconnlat.py): Trace TCP active connection latency (connect()). [Examples](tools/tcpconnlat_example.txt).
 - tools/[tcpretrans](tools/tcpretrans.py): Trace TCP retransmits and TLPs. [Examples](tools/tcpretrans_example.txt).
+- tools/[tcptop](tools/tcptop.py): Summarize TCP send/recv throughput by host. Top for TCP. [Examples](tools/tcptop_example.txt).
 - tools/[tplist](tools/tplist.py): Display kernel tracepoints or USDT probes and their formats. [Examples](tools/tplist_example.txt).
 - tools/[trace](tools/trace.py): Trace arbitrary functions, with filters. [Examples](tools/trace_example.txt)
 - tools/[vfscount](tools/vfscount.py) tools/[vfscount.c](tools/vfscount.c): Count VFS calls. [Examples](tools/vfscount_example.txt).

examples/networking/distributed_bridge/tunnel.c

@@ -29,7 +29,8 @@ int handle_ingress(struct __sk_buff *skb) {
   struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));
 
   struct bpf_tunnel_key tkey = {};
-  bpf_skb_get_tunnel_key(skb, &tkey, sizeof(tkey), 0);
+  bpf_skb_get_tunnel_key(skb, &tkey,
+      offsetof(struct bpf_tunnel_key, remote_ipv6[1]), 0);
 
   int *ifindex = vni2if.lookup(&tkey.tunnel_id);
   if (ifindex) {
@@ -63,7 +64,8 @@ int handle_egress(struct __sk_buff *skb) {
     u32 zero = 0;
     tkey.tunnel_id = dst_host->tunnel_id;
     tkey.remote_ipv4 = dst_host->remote_ipv4;
-    bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey), 0);
+    bpf_skb_set_tunnel_key(skb, &tkey,
+        offsetof(struct bpf_tunnel_key, remote_ipv6[1]), 0);
     lock_xadd(&dst_host->tx_pkts, 1);
   } else {
     struct bpf_tunnel_key tkey = {};
@@ -73,7 +75,8 @@ int handle_egress(struct __sk_buff *skb) {
       return 1;
     tkey.tunnel_id = dst_host->tunnel_id;
     tkey.remote_ipv4 = dst_host->remote_ipv4;
-    bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey), 0);
+    bpf_skb_set_tunnel_key(skb, &tkey,
+        offsetof(struct bpf_tunnel_key, remote_ipv6[1]), 0);
   }
   bpf_clone_redirect(skb, cfg->tunnel_ifindex, 0/*egress*/);
   return 1;

examples/networking/distributed_bridge/tunnel_mesh.c

@@ -19,7 +19,8 @@ BPF_TABLE("hash", int, struct tunnel_key, if2tunkey, 1024);
 int handle_ingress(struct __sk_buff *skb) {
   struct bpf_tunnel_key tkey = {};
   struct tunnel_key key;
-  bpf_skb_get_tunnel_key(skb, &tkey, sizeof(tkey), 0);
+  bpf_skb_get_tunnel_key(skb, &tkey,
+      offsetof(struct bpf_tunnel_key, remote_ipv6[1]), 0);
 
   key.tunnel_id = tkey.tunnel_id;
   key.remote_ipv4 = tkey.remote_ipv4;
@@ -57,7 +58,8 @@ int handle_egress(struct __sk_buff *skb) {
   if (key_p) {
     tkey.tunnel_id = key_p->tunnel_id;
     tkey.remote_ipv4 = key_p->remote_ipv4;
-    bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey), 0);
+    bpf_skb_set_tunnel_key(skb, &tkey,
+        offsetof(struct bpf_tunnel_key, remote_ipv6[1]), 0);
     bpf_clone_redirect(skb, cfg->tunnel_ifindex, 0/*egress*/);
   }
   return 1;

man/man8/argdist.8

@@ -92,10 +92,9 @@ The expression(s) to capture.
 These are the values that are assigned to the histogram or raw event collection.
 You may use the parameters directly, or valid C expressions that involve the
 parameters, such as "size % 10".
-Tracepoints may access a special structure called "tp" that is formatted according
-to the tracepoint format (which you can obtain using tplist). For example, the
-block:block_rq_complete tracepoint can access tp.nr_sector. You may also use the
-members of the "tp" struct directly, e.g. "nr_sector" instead of "tp.nr_sector".
+Tracepoints may access a special structure called "args" that is formatted
+according to the tracepoint format (which you can obtain using tplist).
+For example, the block:block_rq_complete tracepoint can access args->nr_sector.
 USDT probes may access the arguments defined by the tracing program in the 
 special arg1, arg2, ... variables. To obtain their types, use the tplist tool.
 Return probes can use the argument values received by the

man/man8/filelife.8

@@ -9,9 +9,9 @@ on who deleted the file, the file age, and the file name. The intent is to
 provide information on short-lived files, for debugging or performance
 analysis.
 
-This works by tracing the kernel vfs_create() and vfs_delete() functions using
-dynamic tracing, and will need updating to match any changes to these
-functions.
+This works by tracing the kernel vfs_create() and vfs_delete() functions (and
+maybe more, see the source) using dynamic tracing, and will need updating to
+match any changes to these functions.
 
 This makes use of a Linux 4.5 feature (bpf_perf_event_output());
 for kernels older than 4.5, see the version under tools/old,

man/man8/tcptop.8

+.TH tcptop 8  "2016-09-13" "USER COMMANDS"
+.SH NAME
+tcptop \- Summarize TCP send/recv throughput by host. Top for TCP.
+.SH SYNOPSIS
+.B tcptop [\-h] [\-C] [\-S] [\-p PID] [interval] [count]
+.SH DESCRIPTION
+This is top for TCP sessions.
+
+This summarizes TCP send/receive Kbytes by host, and prints a summary that
+refreshes, along other system-wide metrics.
+
+This uses dynamic tracing of kernel TCP send/receive functions, and will
+need to be updated to match kernel changes.
+
+The traced TCP functions are usually called at a lower rate than
+per-packet functions, and therefore have lower overhead. The traced data is
+summarized in-kernel using a BPF map to further reduce overhead. At very high
+TCP event rates, the overhead may still be measurable. See the OVERHEAD
+section for more details.
+
+Since this uses BPF, only the root user can use this tool.
+.SH REQUIREMENTS
+CONFIG_BPF and bcc.
+.SH OPTIONS
+.TP
+\-h
+Print USAGE message.
+.TP
+\-C
+Don't clear the screen.
+.TP
+\-S
+Don't print the system summary line (load averages).
+.TP
+\-p PID
+Trace this PID only.
+.TP
+interval
+Interval between updates, seconds (default 1).
+.TP
+count
+Number of interval summaries (default is many).
+.SH EXAMPLES
+.TP
+Summarize TCP throughput by active sessions, 1 second refresh:
+#
+.B tcptop
+.TP
+Don't clear the screen (rolling output), and 5 second summaries:
+#
+.B tcptop \-C 5
+.TP
+Trace PID 181 only, and don't clear the screen:
+#
+.B tcptop \-Cp 181
+.SH FIELDS
+.TP
+loadavg:
+The contents of /proc/loadavg
+.TP
+PID
+Process ID.
+.TP
+COMM
+Process name.
+.TP
+LADDR
+Local address (IPv4), and TCP port
+.TP
+RADDR
+Remote address (IPv4), and TCP port
+.TP
+LADDR6
+Source address (IPv6), and TCP port
+.TP
+RADDR6
+Destination address (IPv6), and TCP port
+.TP
+RX_KB
+Received Kbytes
+.TP
+TX_KB
+Transmitted Kbytes
+.SH OVERHEAD
+This traces all send/receives in TCP, high in the TCP/IP stack (close to the
+application) which are usually called at a lower rate than per-packet
+functions, lowering overhead. It also summarizes data in-kernel to further
+reduce overhead. These techniques help, but there may still be measurable
+overhead at high send/receive rates, eg, ~13% of one CPU at 100k events/sec.
+use funccount to count the kprobes in the tool to find out this rate, as the
+overhead is relative to the rate. Some sample production servers tested found
+total TCP event rates of 4k to 15k per second, and the CPU overhead at these
+rates ranged from 0.5% to 2.0% of one CPU. If your send/receive rate is low
+(eg, <1000/sec) then the overhead is expected to be negligible; Test in a lab
+environment first.
+.SH SOURCE
+This is from bcc.
+.IP
+https://github.com/iovisor/bcc
+.PP
+Also look in the bcc distribution for a companion _examples.txt file containing
+example usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Brendan Gregg
+.SH INSPIRATION
+top(1) by William LeFebvre
+.SH SEE ALSO
+tcpconnect(8), tcpaccept(8)

man/man8/trace.8

@@ -93,11 +93,9 @@ format specifier replacements may be any C expressions, and may refer to the
 same special keywords as in the predicate (arg1, arg2, etc.).
 
 In tracepoints, both the predicate and the arguments may refer to the tracepoint
-format structure, which is stored in the special "tp" variable. For example, the
-block:block_rq_complete tracepoint can print or filter by tp.nr_sector. To 
-discover the format of your tracepoint, use the tplist tool. Note that you can
-also use the members of the "tp" struct directly, e.g "nr_sector" instead of
-"tp.nr_sector".
+format structure, which is stored in the special "args" variable. For example, the
+block:block_rq_complete tracepoint can print or filter by args->nr_sector. To 
+discover the format of your tracepoint, use the tplist tool. 
 
 In USDT probes, the arg1, ..., argN variables refer to the probe's arguments.
 To determine which arguments your probe has, use the tplist tool.
@@ -126,7 +124,7 @@ Trace returns from the readline function in bash and print the return value as a
 .TP
 Trace the block:block_rq_complete tracepoint and print the number of sectors completed:
 #
-.B trace 't:block:block_rq_complete """%d sectors"", nr_sector'
+.B trace 't:block:block_rq_complete """%d sectors"", args->nr_sector'
 .TP
 Trace the pthread_create USDT probe from the pthread library and print the address of the thread's start function:
 #

src/cc/CMakeLists.txt

@@ -63,7 +63,7 @@ target_link_libraries(bcc-static b_frontend clang_frontend bcc-loader-static ${c
 
 install(TARGETS bcc-shared LIBRARY COMPONENT libbcc
   DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES bpf_common.h bpf_module.h bcc_syms.h ../libbpf.h COMPONENT libbcc
+install(FILES bpf_common.h bpf_module.h bcc_syms.h libbpf.h COMPONENT libbcc
   DESTINATION include/bcc)
 install(DIRECTORY compat/linux/ COMPONENT libbcc
   DESTINATION include/bcc/compat/linux

src/cc/bcc_usdt.h

@@ -26,8 +26,23 @@ void *bcc_usdt_new_frompid(int pid);
 void *bcc_usdt_new_frompath(const char *path);
 void bcc_usdt_close(void *usdt);
 
+struct bcc_usdt {
+    const char *provider;
+    const char *name;
+    const char *bin_path;
+    uint64_t semaphore;
+    int num_locations;
+    int num_arguments;
+};
+
+typedef void (*bcc_usdt_cb)(struct bcc_usdt *);
+void bcc_usdt_foreach(void *usdt, bcc_usdt_cb callback);
+
 int bcc_usdt_enable_probe(void *, const char *, const char *);
 const char *bcc_usdt_genargs(void *);
+const char *bcc_usdt_get_probe_argctype(
+  void *ctx, const char* probe_name, const int arg_index
+);
 
 typedef void (*bcc_usdt_uprobe_cb)(const char *, const char *, uint64_t, int);
 void bcc_usdt_foreach_uprobe(void *usdt, bcc_usdt_uprobe_cb callback);

src/cc/bpf_common.cc

@@ -13,8 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "cc/bpf_module.h"
-#include "cc/bpf_common.h"
+#include "bpf_common.h"
+#include "bpf_module.h"
 
 extern "C" {
 void * bpf_module_create_b(const char *filename, const char *proto_filename, unsigned flags) {

src/cc/usdt.cc

@@ -24,6 +24,7 @@
 #include "bcc_proc.h"
 #include "usdt.h"
 #include "vendor/tinyformat.hpp"
+#include "bcc_usdt.h"
 
 namespace USDT {
 
@@ -255,6 +256,19 @@ bool Context::enable_probe(const std::string &probe_name,
   return p && p->enable(fn_name);
 }
 
+void Context::each(each_cb callback) {
+  for (const auto &probe : probes_) {
+    struct bcc_usdt info = {0};
+    info.provider = probe->provider().c_str();
+    info.bin_path = probe->bin_path().c_str();
+    info.name = probe->name().c_str();
+    info.semaphore = probe->semaphore();
+    info.num_locations = probe->num_locations();
+    info.num_arguments = probe->num_arguments();
+    callback(&info);
+  }
+}
+
 void Context::each_uprobe(each_uprobe_cb callback) {
   for (auto &p : probes_) {
     if (!p->enabled())
@@ -288,7 +302,6 @@ Context::~Context() {
 }
 
 extern "C" {
-#include "bcc_usdt.h"
 
 void *bcc_usdt_new_frompid(int pid) {
   USDT::Context *ctx = new USDT::Context(pid);
@@ -331,6 +344,19 @@ const char *bcc_usdt_genargs(void *usdt) {
   return storage_.c_str();
 }
 
+const char *bcc_usdt_get_probe_argctype(
+  void *ctx, const char* probe_name, const int arg_index
+) {
+  USDT::Probe *p = static_cast<USDT::Context *>(ctx)->get(probe_name);
+  std::string res = p ? p->get_arg_ctype(arg_index) : "";
+  return res.c_str();
+}
+
+void bcc_usdt_foreach(void *usdt, bcc_usdt_cb callback) {
+  USDT::Context *ctx = static_cast<USDT::Context *>(usdt);
+  ctx->each(callback);
+}
+
 void bcc_usdt_foreach_uprobe(void *usdt, bcc_usdt_uprobe_cb callback) {
   USDT::Context *ctx = static_cast<USDT::Context *>(usdt);
   ctx->each_uprobe(callback);

src/cc/usdt.h

@@ -23,6 +23,8 @@
 #include "syms.h"
 #include "vendor/optional.hpp"
 
+struct bcc_usdt;
+
 namespace USDT {
 
 using std::experimental::optional;
@@ -148,9 +150,13 @@ public:
 
   size_t num_locations() const { return locations_.size(); }
   size_t num_arguments() const { return locations_.front().arguments_.size(); }
+  uint64_t semaphore()   const { return semaphore_; }
 
   uint64_t address(size_t n = 0) const { return locations_[n].address_; }
   bool usdt_getarg(std::ostream &stream);
+  std::string get_arg_ctype(int arg_index) {
+    return largest_arg_type(arg_index);
+  }
 
   bool need_enable() const { return semaphore_ != 0x0; }
   bool enable(const std::string &fn_name);
@@ -194,6 +200,9 @@ public:
   bool enable_probe(const std::string &probe_name, const std::string &fn_name);
   bool generate_usdt_args(std::ostream &stream);
 
+  typedef void (*each_cb)(struct bcc_usdt *);
+  void each(each_cb callback);
+
   typedef void (*each_uprobe_cb)(const char *, const char *, uint64_t, int);
   void each_uprobe(each_uprobe_cb callback);
 };

src/python/bcc/__init__.py

@@ -27,7 +27,6 @@ basestring = (unicode if sys.version_info[0] < 3 else str)
 
 from .libbcc import lib, _CB_TYPE, bcc_symbol
 from .table import Table
-from .tracepoint import Tracepoint
 from .perf import Perf
 from .usyms import ProcessSymbols
 
@@ -120,9 +119,9 @@ class BPF(object):
         return filename
 
     @staticmethod
-    def _find_exe(cls, bin_path):
+    def find_exe(bin_path):
         """
-        _find_exe(bin_path)
+        find_exe(bin_path)
 
         Traverses the PATH environment variable, looking for the first
         directory that contains an executable file named bin_path, and
@@ -149,7 +148,7 @@ class BPF(object):
         return None
 
     def __init__(self, src_file="", hdr_file="", text=None, cb=None, debug=0,
-            cflags=[], usdt=None):
+            cflags=[], usdt_contexts=[]):
         """Create a a new BPF module with the given source code.
 
         Note:
@@ -179,7 +178,15 @@ class BPF(object):
         self.tables = {}
         cflags_array = (ct.c_char_p * len(cflags))()
         for i, s in enumerate(cflags): cflags_array[i] = s.encode("ascii")
-        if usdt and text: text = usdt.get_text() + text
+        if text:
+            for usdt_context in usdt_contexts:
+                usdt_text = usdt_context.get_text()
+                if usdt_text is None:
+                    raise Exception("can't generate USDT probe arguments; " +
+                                    "possible cause is missing pid when a " +
+                                    "probe in a shared object has multiple " +
+                                    "locations")
+                text = usdt_context.get_text() + text
 
         if text:
             self.module = lib.bpf_module_create_c_from_string(text.encode("ascii"),
@@ -197,7 +204,8 @@ class BPF(object):
         if not self.module:
             raise Exception("Failed to compile BPF module %s" % src_file)
 
-        if usdt: usdt.attach_uprobes(self)
+        for usdt_context in usdt_contexts:
+            usdt_context.attach_uprobes(self)
 
         # If any "kprobe__" or "tracepoint__" prefixed functions were defined,
         # they will be loaded and attached here.

src/python/bcc/libbcc.py

@@ -157,7 +157,26 @@ lib.bcc_usdt_enable_probe.argtypes = [ct.c_void_p, ct.c_char_p, ct.c_char_p]
 lib.bcc_usdt_genargs.restype = ct.c_char_p
 lib.bcc_usdt_genargs.argtypes = [ct.c_void_p]
 
-_USDT_CB = ct.CFUNCTYPE(None, ct.c_char_p, ct.c_char_p, ct.c_ulonglong, ct.c_int)
+lib.bcc_usdt_get_probe_argctype.restype = ct.c_char_p
+lib.bcc_usdt_get_probe_argctype.argtypes = [ct.c_void_p, ct.c_char_p, ct.c_int]
+
+class bcc_usdt(ct.Structure):
+    _fields_ = [
+            ('provider', ct.c_char_p),
+            ('name', ct.c_char_p),
+            ('bin_path', ct.c_char_p),
+            ('semaphore', ct.c_ulonglong),
+            ('num_locations', ct.c_int),
+            ('num_arguments', ct.c_int),
+        ]
+
+_USDT_CB = ct.CFUNCTYPE(None, ct.POINTER(bcc_usdt))
+
+lib.bcc_usdt_foreach.restype = None
+lib.bcc_usdt_foreach.argtypes = [ct.c_void_p, _USDT_CB]
+
+_USDT_PROBE_CB = ct.CFUNCTYPE(None, ct.c_char_p, ct.c_char_p,
+                              ct.c_ulonglong, ct.c_int)
 
 lib.bcc_usdt_foreach_uprobe.restype = None
-lib.bcc_usdt_foreach_uprobe.argtypes = [ct.c_void_p, _USDT_CB]
+lib.bcc_usdt_foreach_uprobe.argtypes = [ct.c_void_p, _USDT_PROBE_CB]

src/python/bcc/tracepoint.py

-# Copyright 2016 Sasha Goldshtein
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import ctypes as ct
-import multiprocessing
-import os
-import re
-
-class Tracepoint(object):
-        enabled_tracepoints = []
-        trace_root = "/sys/kernel/debug/tracing"
-        event_root = os.path.join(trace_root, "events")
-
-        @classmethod
-        def _any_tracepoints_enabled(cls):
-                return len(cls.enabled_tracepoints) > 0
-
-        @classmethod
-        def generate_decl(cls):
-                if not cls._any_tracepoints_enabled():
-                        return ""
-                return "\nBPF_HASH(__trace_di, u64, u64);\n"
-
-        @classmethod
-        def generate_entry_probe(cls):
-                if not cls._any_tracepoints_enabled():
-                        return ""
-                return """
-int __trace_entry_update(struct pt_regs *ctx)
-{
-        u64 tid = bpf_get_current_pid_tgid();
-        u64 val = PT_REGS_PARM1(ctx);
-        __trace_di.update(&tid, &val);
-        return 0;
-}
-"""
-
-        def __init__(self, category, event, tp_id):
-                self.category = category
-                self.event = event
-                self.tp_id = tp_id
-                self._retrieve_struct_fields()
-
-        def _retrieve_struct_fields(self):
-                self.struct_fields = []
-                format_lines = Tracepoint.get_tpoint_format(self.category,
-                                                            self.event)
-                for line in format_lines:
-                        match = re.search(r'field:([^;]*);.*size:(\d+);', line)
-                        if match is None:
-                                continue
-                        parts = match.group(1).split()
-                        field_name = parts[-1:][0]
-                        field_type = " ".join(parts[:-1])
-                        field_size = int(match.group(2))
-                        if "__data_loc" in field_type:
-                                continue
-                        if field_name.startswith("common_"):
-                                continue
-                        self.struct_fields.append((field_type, field_name))
-
-        def _generate_struct_fields(self):
-                text = ""
-                for field_type, field_name in self.struct_fields:
-                        text += "        %s %s;\n" % (field_type, field_name)
-                return text
-
-        def generate_struct(self):
-                self.struct_name = self.event + "_trace_entry"
-                return """
-struct %s {
-        u64 __do_not_use__;
-%s
-};
-                """ % (self.struct_name, self._generate_struct_fields())
-
-        def _generate_struct_locals(self):
-                text = ""
-                for field_type, field_name in self.struct_fields:
-                        if field_type == "char" and field_name.endswith(']'):
-                                # Special case for 'char whatever[N]', should
-                                # be assigned to a 'char *'
-                                field_type = "char *"
-                                field_name = re.sub(r'\[\d+\]$', '', field_name)
-                        text += "        %s %s = tp.%s;\n" % (
-                                        field_type, field_name, field_name)
-                return text
-
-        def generate_get_struct(self):
-                return """
-        u64 tid = bpf_get_current_pid_tgid();
-        u64 *di = __trace_di.lookup(&tid);
-        if (di == 0) { return 0; }
-        struct %s tp = {};
-        bpf_probe_read(&tp, sizeof(tp), (void *)*di);
-%s
-                """ % (self.struct_name, self._generate_struct_locals())
-
-        @classmethod
-        def enable_tracepoint(cls, category, event):
-                tp_id = cls.get_tpoint_id(category, event)
-                if tp_id == -1:
-                        raise ValueError("no such tracepoint found: %s:%s" %
-                                         (category, event))
-                Perf.perf_event_open(tp_id, ptype=Perf.PERF_TYPE_TRACEPOINT)
-                new_tp = Tracepoint(category, event, tp_id)
-                cls.enabled_tracepoints.append(new_tp)
-                return new_tp
-
-        @staticmethod
-        def get_tpoint_id(category, event):
-                evt_dir = os.path.join(Tracepoint.event_root, category, event)
-                try:
-                        return int(
-                          open(os.path.join(evt_dir, "id")).read().strip())
-                except:
-                        return -1
-
-        @staticmethod
-        def get_tpoint_format(category, event):
-                evt_dir = os.path.join(Tracepoint.event_root, category, event)
-                try:
-                        return open(os.path.join(evt_dir, "format")).readlines()
-                except:
-                        return ""
-
-        @classmethod
-        def attach(cls, bpf):
-                if cls._any_tracepoints_enabled():
-                        bpf.attach_kprobe(event="tracing_generic_entry_update",
-                                          fn_name="__trace_entry_update")

src/python/bcc/usdt.py

@@ -12,11 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .libbcc import lib, _USDT_CB
+from .libbcc import lib, _USDT_CB, _USDT_PROBE_CB
+
+class USDTProbe(object):
+    def __init__(self, usdt):
+        self.provider = usdt.provider
+        self.name = usdt.name
+        self.bin_path = usdt.bin_path
+        self.semaphore = usdt.semaphore
+        self.num_locations = usdt.num_locations
+        self.num_arguments = usdt.num_arguments
+
+    def __str__(self):
+        return "%s %s:%s [sema 0x%x]\n  %d location(s)\n  %d argument(s)" % \
+               (self.bin_path, self.provider, self.name, self.semaphore,
+                self.num_locations, self.num_arguments)
+
+    def short_name(self):
+        return "%s:%s" % (self.provider, self.name)
 
 class USDT(object):
     def __init__(self, pid=None, path=None):
-        if pid:
+        if pid and pid != -1:
             self.pid = pid
             self.context = lib.bcc_usdt_new_frompid(pid)
             if self.context == None:
@@ -26,20 +43,40 @@ class USDT(object):
             self.context = lib.bcc_usdt_new_frompath(path)
             if self.context == None:
                 raise Exception("USDT failed to instrument path %s" % path)
+        else:
+            raise Exception("either a pid or a binary path must be specified")
 
     def enable_probe(self, probe, fn_name):
         if lib.bcc_usdt_enable_probe(self.context, probe, fn_name) != 0:
-            raise Exception("failed to enable probe '%s'" % probe)
+            raise Exception(("failed to enable probe '%s'; a possible cause " +
+                            "can be that the probe requires a pid to enable") %
+                            probe)
 
     def get_text(self):
         return lib.bcc_usdt_genargs(self.context)
 
+    def get_probe_arg_ctype(self, probe_name, arg_index):
+        return lib.bcc_usdt_get_probe_argctype(
+            self.context, probe_name, arg_index)
+
+    def enumerate_probes(self):
+        probes = []
+        def _add_probe(probe):
+            probes.append(USDTProbe(probe.contents))
+
+        lib.bcc_usdt_foreach(self.context, _USDT_CB(_add_probe))
+        return probes
+
+    # This is called by the BPF module's __init__ when it realizes that there
+    # is a USDT context and probes need to be attached.
     def attach_uprobes(self, bpf):
         probes = []
         def _add_probe(binpath, fn_name, addr, pid):
             probes.append((binpath, fn_name, addr, pid))
 
-        lib.bcc_usdt_foreach_uprobe(self.context, _USDT_CB(_add_probe))
+        lib.bcc_usdt_foreach_uprobe(self.context, _USDT_PROBE_CB(_add_probe))
 
         for (binpath, fn_name, addr, pid) in probes:
-            bpf.attach_uprobe(name=binpath, fn_name=fn_name, addr=addr, pid=pid)
+            bpf.attach_uprobe(name=binpath, fn_name=fn_name,
+                              addr=addr, pid=pid)
+

tests/cc/test_usdt_probes.cc

@@ -56,15 +56,6 @@ TEST_CASE("test finding a probe in our own process", "[usdt]") {
 }
 #endif  // HAVE_SDT_HEADER
 
-static size_t countsubs(const std::string &str, const std::string &sub) {
-  size_t count = 0;
-  for (size_t offset = str.find(sub); offset != std::string::npos;
-       offset = str.find(sub, offset + sub.length())) {
-    ++count;
-  }
-  return count;
-}
-
 class ChildProcess {
   pid_t pid_;
 

tools/CMakeLists.txt

 file(GLOB C_FILES *.c)
 file(GLOB PY_FILES *.py)
 file(GLOB TXT_FILES *.txt)
+list(REMOVE_ITEM TXT_FILES "CMakeLists.txt")
 foreach(FIL ${PY_FILES})
   get_filename_component(FIL_WE ${FIL} NAME_WE)
   install(PROGRAMS ${FIL} DESTINATION share/bcc/tools RENAME ${FIL_WE})

tools/argdist_example.txt

@@ -10,7 +10,7 @@ various functions.
 For example, suppose you want to find what allocation sizes are common in
 your application:
 
-# ./argdist -p 2420 -C 'p:c:malloc(size_t size):size_t:size'
+# ./argdist -p 2420 -c -C 'p:c:malloc(size_t size):size_t:size'
 [01:42:29]
 p:c:malloc(size_t size):size_t:size
         COUNT      EVENT
@@ -43,7 +43,7 @@ probed and its value was 16, repeatedly.
 Now, suppose you wanted a histogram of buffer sizes passed to the write()
 function across the system:
 
-# ./argdist -H 'p:c:write(int fd, void *buf, size_t len):size_t:len'
+# ./argdist -c -H 'p:c:write(int fd, void *buf, size_t len):size_t:len'
 [01:45:22]
 p:c:write(int fd, void *buf, size_t len):size_t:len
      len                 : count     distribution
@@ -81,7 +81,7 @@ bytes, medium writes of 32-63 bytes, and larger writes of 64-127 bytes.
 But these are writes across the board -- what if you wanted to focus on writes
 to STDOUT?
 
-# ./argdist -H 'p:c:write(int fd, void *buf, size_t len):size_t:len:fd==1'
+# ./argdist -c -H 'p:c:write(int fd, void *buf, size_t len):size_t:len:fd==1'
 [01:47:17]
 p:c:write(int fd, void *buf, size_t len):size_t:len:fd==1
      len                 : count     distribution
@@ -232,7 +232,7 @@ multiple microseconds per byte.
 You could also group results by more than one field. For example, __kmalloc
 takes an additional flags parameter that describes how to allocate memory:
 
-# ./argdist -C 'p::__kmalloc(size_t size, gfp_t flags):gfp_t,size_t:flags,size'
+# ./argdist -c -C 'p::__kmalloc(size_t size, gfp_t flags):gfp_t,size_t:flags,size'
 [03:42:29]
 p::__kmalloc(size_t size, gfp_t flags):gfp_t,size_t:flags,size
         COUNT      EVENT
@@ -264,29 +264,23 @@ certain kinds of allocations or visually group them together.
 
 argdist also has basic support for kernel tracepoints. It is sometimes more
 convenient to use tracepoints because they are documented and don't vary a lot
-between kernel versions like function signatures tend to. For example, let's
-trace the net:net_dev_start_xmit tracepoint and print the interface name that
-is transmitting:
+between kernel versions. For example, let's trace the net:net_dev_start_xmit
+tracepoint and print out the protocol field from the tracepoint structure:
 
-# argdist -C 't:net:net_dev_start_xmit(void *a, void *b, struct net_device *c):char*:c->name' -n 2
-[05:01:10]
-t:net:net_dev_start_xmit(void *a, void *b, struct net_device *c):char*:c->name
+# argdist -C 't:net:net_dev_start_xmit():u16:args->protocol'
+[13:01:49]
+t:net:net_dev_start_xmit():u16:args->protocol
         COUNT      EVENT
-        4          c->name = eth0
-[05:01:11]
-t:net:net_dev_start_xmit(void *a, void *b, struct net_device *c):char*:c->name
-        COUNT      EVENT
-        6          c->name = lo
-        92         c->name = eth0
+        8          args->protocol = 2048
+^C
 
-Note that to determine the necessary function signature you need to look at the
-TP_PROTO declaration in the kernel headers. For example, the net_dev_start_xmit
-tracepoint is defined in the include/trace/events/net.h header file.
+Note that to discover the format of the net:net_dev_start_xmit tracepoint, you
+use the tplist tool (tplist -v net:net_dev_start_xmit).
 
 Here's a final example that finds how many write() system calls are performed
 by each process on the system:
 
-# argdist -C 'p:c:write():int:$PID;write per process' -n 2
+# argdist -c -C 'p:c:write():int:$PID;write per process' -n 2
 [06:47:18]
 write by process
         COUNT      EVENT
@@ -305,7 +299,7 @@ USAGE message:
 
 # argdist -h
 usage: argdist [-h] [-p PID] [-z STRING_SIZE] [-i INTERVAL] [-n COUNT] [-v]
-                  [-T TOP] [-H [specifier [specifier ...]]]
+               [-c] [-T TOP] [-H [specifier [specifier ...]]]
                [-C [specifier [specifier ...]]] [-I [header [header ...]]]
 
 Trace a function and display a summary of its parameter values.
@@ -320,6 +314,7 @@ optional arguments:
   -n COUNT, --number COUNT
                         number of outputs
   -v, --verbose         print resulting BPF program code before executing
+  -c, --cumulative      do not clear histograms and freq counts at each interval
   -T TOP, --top TOP     number of top results to show (not applicable to
                         histograms)
   -H [specifier [specifier ...]], --histogram [specifier [specifier ...]]
@@ -387,10 +382,10 @@ argdist -C 'p:c:fork()#fork calls'
         Count fork() calls in libc across all processes
         Can also use funccount.py, which is easier and more flexible 
 
-argdist -H 't:block:block_rq_complete():u32:tp.nr_sector'
+argdist -H 't:block:block_rq_complete():u32:args->nr_sector'
         Print histogram of number of sectors in completing block I/O requests
 
-argdist -C 't:irq:irq_handler_entry():int:tp.irq'
+argdist -C 't:irq:irq_handler_entry():int:args->irq'
         Aggregate interrupts by interrupt request (IRQ)
 
 argdist -C 'u:pthread:pthread_start():u64:arg2' -p 1337

tools/filelife.py

@@ -82,13 +82,14 @@ int trace_unlink(struct pt_regs *ctx, struct inode *dir, struct dentry *dentry)
     delta = (bpf_ktime_get_ns() - *tsp) / 1000000;
     birth.delete(&dentry);
 
-    if (dentry->d_iname[0] == 0)
+    if (dentry->d_name.len == 0)
         return 0;
 
     if (bpf_get_current_comm(&data.comm, sizeof(data.comm)) == 0) {
         data.pid = pid;
         data.delta = delta;
-        bpf_probe_read(&data.fname, sizeof(data.fname), dentry->d_iname);
+        bpf_probe_read(&data.fname, sizeof(data.fname),
+            (void *)dentry->d_name.name);
     }
 
     events.perf_submit(ctx, &data, sizeof(data));
@@ -119,6 +120,9 @@ if debug:
 # initialize BPF
 b = BPF(text=bpf_text)
 b.attach_kprobe(event="vfs_create", fn_name="trace_create")
+# newer kernels (say, 4.8) may don't fire vfs_create, so record (or overwrite)
+# the timestamp in security_inode_create():
+b.attach_kprobe(event="security_inode_create", fn_name="trace_create")
 b.attach_kprobe(event="vfs_unlink", fn_name="trace_unlink")
 
 # header

tools/offcputime.py

@@ -40,6 +40,8 @@ examples = """examples:
     ./offcputime             # trace off-CPU stack time until Ctrl-C
     ./offcputime 5           # trace for 5 seconds only
     ./offcputime -f 5        # 5 seconds, and output in folded format
+    ./offcputime -m 1000     # trace only events that last more than 1000 usec.
+    ./offcputime -M 10000    # trace only events that last less than 10000 usec.
     ./offcputime -p 185      # only trace threads for PID 185
     ./offcputime -t 188      # only trace thread 188
     ./offcputime -u          # only trace user threads (no kernel)
@@ -78,6 +80,12 @@ parser.add_argument("--stack-storage-size", default=1024,
 parser.add_argument("duration", nargs="?", default=99999999,
     type=positive_nonzero_int,
     help="duration of trace, in seconds")
+parser.add_argument("-m", "--min-block-time", default=1,
+    type=positive_nonzero_int,
+    help="the amount of time in microseconds over which we store traces (default 1)")
+parser.add_argument("-M", "--max-block-time", default=(1<<64)-1,
+    type=positive_nonzero_int,
+    help="the amount of time in microseconds under which we store traces (default U64_MAX)")
 args = parser.parse_args()
 if args.pid and args.tgid:
     parser.error("specify only one of -p and -t")
@@ -93,7 +101,8 @@ bpf_text = """
 #include <uapi/linux/ptrace.h>
 #include <linux/sched.h>
 
-#define MINBLOCK_US	1
+#define MINBLOCK_US    MINBLOCK_US_VALUEULL
+#define MAXBLOCK_US    MAXBLOCK_US_VALUEULL
 
 struct key_t {
     u32 pid;
@@ -129,7 +138,7 @@ int oncpu(struct pt_regs *ctx, struct task_struct *prev) {
     u64 delta = bpf_ktime_get_ns() - *tsp;
     start.delete(&pid);
     delta = delta / 1000;
-    if (delta < MINBLOCK_US) {
+    if ((delta < MINBLOCK_US) || (delta > MAXBLOCK_US)) {
         return 0;
     }
 
@@ -170,6 +179,8 @@ bpf_text = bpf_text.replace('THREAD_FILTER', thread_filter)
 
 # set stack storage size
 bpf_text = bpf_text.replace('STACK_STORAGE_SIZE', str(args.stack_storage_size))
+bpf_text = bpf_text.replace('MINBLOCK_US_VALUE', str(args.min_block_time))
+bpf_text = bpf_text.replace('MAXBLOCK_US_VALUE', str(args.max_block_time))
 
 # handle stack args
 kernel_stack_get = "stack_traces.get_stackid(ctx, BPF_F_REUSE_STACKID)"

tools/tcptop.py

+#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
+#
+# tcptop    Summarize TCP send/recv throughput by host.
+#           For Linux, uses BCC, eBPF. Embedded C.
+#
+# USAGE: tcptop [-h] [-C] [-S] [-p PID] [interval [count]]
+#
+# This uses dynamic tracing of kernel functions, and will need to be updated
+# to match kernel changes.
+#
+# WARNING: This traces all send/receives at the TCP level, and while it
+# summarizes data in-kernel to reduce overhead, there may still be some
+# overhead at high TCP send/receive rates (eg, ~13% of one CPU at 100k TCP
+# events/sec. This is not the same as packet rate: funccount can be used to
+# count the kprobes below to find out the TCP rate). Test in a lab environment
+# first. If your send/receive rate is low (eg, <1k/sec) then the overhead is
+# expected to be negligible.
+#
+# ToDo: Fit output to screen size (top X only) in default (not -C) mode.
+#
+# Copyright 2016 Netflix, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License")
+#
+# 02-Sep-2016   Brendan Gregg   Created this.
+
+from __future__ import print_function
+from bcc import BPF
+import argparse
+from socket import inet_ntop, AF_INET, AF_INET6
+from struct import pack
+from time import sleep, strftime
+from subprocess import call
+import ctypes as ct
+
+# arguments
+examples = """examples:
+    ./tcptop           # trace TCP send/recv by host
+    ./tcptop -C        # don't clear the screen
+    ./tcptop -p 181    # only trace PID 181
+"""
+parser = argparse.ArgumentParser(
+    description="Summarize TCP send/recv throughput by host",
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    epilog=examples)
+parser.add_argument("-C", "--noclear", action="store_true",
+    help="don't clear the screen")
+parser.add_argument("-S", "--nosummary", action="store_true",
+    help="skip system summary line")
+parser.add_argument("-p", "--pid",
+    help="trace this PID only")
+parser.add_argument("interval", nargs="?", default=1,
+    help="output interval, in seconds (default 1)")
+parser.add_argument("count", nargs="?", default=99999999,
+    help="number of outputs")
+args = parser.parse_args()
+countdown = int(args.count)
+if args.interval and int(args.interval) == 0:
+    print("ERROR: interval 0. Exiting.")
+    exit()
+debug = 0
+
+# linux stats
+loadavg = "/proc/loadavg"
+
+# define BPF program
+bpf_text = """
+#include <uapi/linux/ptrace.h>
+#include <net/sock.h>
+#include <bcc/proto.h>
+
+struct ipv4_key_t {
+    u32 pid;
+    u32 saddr;
+    u32 daddr;
+    u16 lport;
+    u16 dport;
+};
+BPF_HASH(ipv4_send_bytes, struct ipv4_key_t);
+BPF_HASH(ipv4_recv_bytes, struct ipv4_key_t);
+
+struct ipv6_key_t {
+    u32 pid;
+    // workaround until unsigned __int128 support:
+    u64 saddr0;
+    u64 saddr1;
+    u64 daddr0;
+    u64 daddr1;
+    u16 lport;
+    u16 dport;
+};
+BPF_HASH(ipv6_send_bytes, struct ipv6_key_t);
+BPF_HASH(ipv6_recv_bytes, struct ipv6_key_t);
+
+int kprobe__tcp_sendmsg(struct pt_regs *ctx, struct sock *sk,
+    struct msghdr *msg, size_t size)
+{
+    u32 pid = bpf_get_current_pid_tgid();
+    FILTER
+    u16 dport = 0, family = sk->__sk_common.skc_family;
+    u64 *val, zero = 0;
+
+    if (family == AF_INET) {
+        struct ipv4_key_t ipv4_key = {.pid = pid};
+        ipv4_key.saddr = sk->__sk_common.skc_rcv_saddr;
+        ipv4_key.daddr = sk->__sk_common.skc_daddr;
+        ipv4_key.lport = sk->__sk_common.skc_num;
+        dport = sk->__sk_common.skc_dport;
+        ipv4_key.dport = ntohs(dport);
+        val = ipv4_send_bytes.lookup_or_init(&ipv4_key, &zero);
+        (*val) += size;
+
+    } else if (family == AF_INET6) {
+        struct ipv6_key_t ipv6_key = {.pid = pid};
+
+        bpf_probe_read(&ipv6_key.saddr0, sizeof(ipv6_key.saddr0),
+            &sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[0]);
+        bpf_probe_read(&ipv6_key.saddr1, sizeof(ipv6_key.saddr1),
+            &sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[2]);
+        bpf_probe_read(&ipv6_key.daddr0, sizeof(ipv6_key.daddr0),
+            &sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32[0]);
+        bpf_probe_read(&ipv6_key.daddr1, sizeof(ipv6_key.daddr1),
+            &sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32[2]);
+        ipv6_key.lport = sk->__sk_common.skc_num;
+        dport = sk->__sk_common.skc_dport;
+        ipv6_key.dport = ntohs(dport);
+        val = ipv6_send_bytes.lookup_or_init(&ipv6_key, &zero);
+        (*val) += size;
+    }
+    // else drop
+
+    return 0;
+}
+
+/*
+ * tcp_recvmsg() would be obvious to trace, but is less suitable because:
+ * - we'd need to trace both entry and return, to have both sock and size
+ * - misses tcp_read_sock() traffic
+ * we'd much prefer tracepoints once they are available.
+ */
+int kprobe__tcp_cleanup_rbuf(struct pt_regs *ctx, struct sock *sk, int copied)
+{
+    u32 pid = bpf_get_current_pid_tgid();
+    FILTER
+    u16 dport = 0, family = sk->__sk_common.skc_family;
+    u64 *val, zero = 0;
+
+    if (family == AF_INET) {
+        struct ipv4_key_t ipv4_key = {.pid = pid};
+        ipv4_key.saddr = sk->__sk_common.skc_rcv_saddr;
+        ipv4_key.daddr = sk->__sk_common.skc_daddr;
+        ipv4_key.lport = sk->__sk_common.skc_num;
+        dport = sk->__sk_common.skc_dport;
+        ipv4_key.dport = ntohs(dport);
+        val = ipv4_recv_bytes.lookup_or_init(&ipv4_key, &zero);
+        (*val) += copied;
+
+    } else if (family == AF_INET6) {
+        struct ipv6_key_t ipv6_key = {.pid = pid};
+        bpf_probe_read(&ipv6_key.saddr0, sizeof(ipv6_key.saddr0),
+            &sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[0]);
+        bpf_probe_read(&ipv6_key.saddr1, sizeof(ipv6_key.saddr1),
+            &sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[2]);
+        bpf_probe_read(&ipv6_key.daddr0, sizeof(ipv6_key.daddr0),
+            &sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32[0]);
+        bpf_probe_read(&ipv6_key.daddr1, sizeof(ipv6_key.daddr1),
+            &sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32[2]);
+        ipv6_key.lport = sk->__sk_common.skc_num;
+        dport = sk->__sk_common.skc_dport;
+        ipv6_key.dport = ntohs(dport);
+        val = ipv6_recv_bytes.lookup_or_init(&ipv6_key, &zero);
+        (*val) += copied;
+    }
+    // else drop
+
+    return 0;
+}
+"""
+
+# code substitutions
+if args.pid:
+    bpf_text = bpf_text.replace('FILTER',
+        'if (pid != %s) { return 0; }' % args.pid)
+else:
+    bpf_text = bpf_text.replace('FILTER', '')
+if debug:
+    print(bpf_text)
+
+def pid_to_comm(pid):
+    try:
+        comm = open("/proc/%d/comm" % pid, "r").read().rstrip()
+        return comm
+    except IOError:
+        return str(pid)
+
+# initialize BPF
+b = BPF(text=bpf_text)
+
+ipv4_send_bytes = b["ipv4_send_bytes"]
+ipv4_recv_bytes = b["ipv4_recv_bytes"]
+ipv6_send_bytes = b["ipv6_send_bytes"]
+ipv6_recv_bytes = b["ipv6_recv_bytes"]
+
+print('Tracing... Output every %s secs. Hit Ctrl-C to end' % args.interval)
+
+# output
+exiting = 0
+while (1):
+    try:
+        if args.interval:
+            sleep(int(args.interval))
+        else:
+            sleep(99999999)
+    except KeyboardInterrupt:
+        exiting = 1
+
+    # header
+    if args.noclear:
+        print()
+    else:
+        call("clear")
+    if not args.nosummary:
+        with open(loadavg) as stats:
+            print("%-8s loadavg: %s" % (strftime("%H:%M:%S"), stats.read()))
+
+    # IPv4:  build dict of all seen keys
+    keys = ipv4_recv_bytes
+    for k, v in ipv4_send_bytes.items():
+        if k not in keys:
+            keys[k] = v
+
+    if keys:
+        print("%-6s %-12s %-21s %-21s %6s %6s" % ("PID", "COMM",
+            "LADDR", "RADDR", "RX_KB", "TX_KB"))
+
+    # output
+    for k, v in reversed(sorted(keys.items(), key=lambda keys: keys[1].value)):
+        send_kbytes = 0
+        if k in ipv4_send_bytes:
+            send_kbytes = int(ipv4_send_bytes[k].value / 1024)
+        recv_kbytes = 0
+        if k in ipv4_recv_bytes:
+            recv_kbytes = int(ipv4_recv_bytes[k].value / 1024)
+
+        print("%-6d %-12.12s %-21s %-21s %6d %6d" % (k.pid,
+            pid_to_comm(k.pid),
+            inet_ntop(AF_INET, pack("I", k.saddr)) + ":" + str(k.lport),
+            inet_ntop(AF_INET, pack("I", k.daddr)) + ":" + str(k.dport),
+            recv_kbytes, send_kbytes))
+
+    ipv4_send_bytes.clear()
+    ipv4_recv_bytes.clear()
+
+    # IPv6: build dict of all seen keys
+    keys = ipv6_recv_bytes
+    for k, v in ipv6_send_bytes.items():
+        if k not in keys:
+            keys[k] = v
+
+    if keys:
+        # more than 80 chars, sadly.
+        print("\n%-6s %-12s %-32s %-32s %6s %6s" % ("PID", "COMM",
+            "LADDR6", "RADDR6", "RX_KB", "TX_KB"))
+
+    # output
+    for k, v in reversed(sorted(keys.items(), key=lambda keys: keys[1].value)):
+        send_kbytes = 0
+        if k in ipv6_send_bytes:
+            send_kbytes = int(ipv6_send_bytes[k].value / 1024)
+        recv_kbytes = 0
+        if k in ipv6_recv_bytes:
+            recv_kbytes = int(ipv6_recv_bytes[k].value / 1024)
+
+        print("%-6d %-12.12s %-32s %-32s %6d %6d" % (k.pid,
+            pid_to_comm(k.pid),
+            inet_ntop(AF_INET6, pack("QQ", k.saddr0, k.saddr1)) + ":" +
+            str(k.lport),
+            inet_ntop(AF_INET6, pack("QQ", k.daddr0, k.daddr1)) + ":" +
+            str(k.dport),
+            recv_kbytes, send_kbytes))
+
+    ipv6_send_bytes.clear()
+    ipv6_recv_bytes.clear()
+
+    countdown -= 1
+    if exiting or countdown == 0:
+        exit()

tools/tcptop_example.txt

+Demonstrations of tcptop, the Linux eBPF/bcc version.
+
+
+tcptop summarizes throughput by host and port. Eg:
+
+# tcptop
+Tracing... Output every 1 secs. Hit Ctrl-C to end
+<screen clears>
+19:46:24 loadavg: 1.86 2.67 2.91 3/362 16681
+
+PID    COMM         LADDR                 RADDR                  RX_KB  TX_KB
+16648  16648        100.66.3.172:22       100.127.69.165:6684        1      0
+16647  sshd         100.66.3.172:22       100.127.69.165:6684        0   2149
+14374  sshd         100.66.3.172:22       100.127.69.165:25219       0      0
+14458  sshd         100.66.3.172:22       100.127.69.165:7165        0      0
+
+PID    COMM         LADDR6                           RADDR6                            RX_KB  TX_KB
+16681  sshd         fe80::8a3:9dff:fed5:6b19:22      fe80::8a3:9dff:fed5:6b19:16606        1      1
+16679  ssh          fe80::8a3:9dff:fed5:6b19:16606   fe80::8a3:9dff:fed5:6b19:22           1      1
+16680  sshd         fe80::8a3:9dff:fed5:6b19:22      fe80::8a3:9dff:fed5:6b19:16606        0      0
+
+This example output shows two listings of TCP connections, for IPv4 and IPv6.
+If there is only traffic for one of these, then only one group is shown.
+
+The output in each listing is sorted by total throughput (send then receive),
+and when printed it is rounded (floor) to the nearest Kbyte. The example output
+shows PID 16647, sshd, transmitted 2149 Kbytes during the tracing interval.
+The other IPv4 sessions had such low throughput they rounded to zero.
+
+All TCP sessions, including over loopback, are included.
+
+The session with the process name (COMM) of 16648 is really a short-lived
+process with PID 16648 where we didn't catch the process name when printing
+the output. If this behavior is a serious issue for you, you can modify the
+tool's code to include bpf_get_current_comm() in the key structs, so that it's
+fetched during the event and will always be seen. I did it this way to start
+with, but it was measurably increasing the overhead of this tool, so I switched
+to the asynchronous model.
+
+The overhead is relative to TCP event rate (the rate of tcp_sendmsg() and
+tcp_recvmsg() or tcp_cleanup_rbuf()). Due to buffering, this should be lower
+than the packet rate. You can measure the rate of these using funccount.
+Some sample production servers tested found total rates of 4k to 15k per
+second. The CPU overhead at these rates ranged from 0.5% to 2.0% of one CPU.
+Maybe your workloads have higher rates and therefore higher overhead, or,
+lower rates.
+
+
+I much prefer not clearing the screen, so that historic output is in the
+scroll-back buffer, and patterns or intermittent issues can be better seen.
+You can do this with -C:
+
+# tcptop -C
+Tracing... Output every 1 secs. Hit Ctrl-C to end
+
+20:27:12 loadavg: 0.08 0.02 0.17 2/367 17342
+
+PID    COMM         LADDR                 RADDR                  RX_KB  TX_KB
+17287  17287        100.66.3.172:22       100.127.69.165:57585       3      1
+17286  sshd         100.66.3.172:22       100.127.69.165:57585       0      1
+14374  sshd         100.66.3.172:22       100.127.69.165:25219       0      0
+
+20:27:13 loadavg: 0.08 0.02 0.17 1/367 17342
+
+PID    COMM         LADDR                 RADDR                  RX_KB  TX_KB
+17286  sshd         100.66.3.172:22       100.127.69.165:57585       1   7761
+14374  sshd         100.66.3.172:22       100.127.69.165:25219       0      0
+
+20:27:14 loadavg: 0.08 0.02 0.17 2/365 17347
+
+PID    COMM         LADDR                 RADDR                  RX_KB  TX_KB
+17286  17286        100.66.3.172:22       100.127.69.165:57585       1   2501
+14374  sshd         100.66.3.172:22       100.127.69.165:25219       0      0
+
+20:27:15 loadavg: 0.07 0.02 0.17 2/367 17403
+
+PID    COMM         LADDR                 RADDR                  RX_KB  TX_KB
+17349  17349        100.66.3.172:22       100.127.69.165:10161       3      1
+17348  sshd         100.66.3.172:22       100.127.69.165:10161       0      1
+14374  sshd         100.66.3.172:22       100.127.69.165:25219       0      0
+
+20:27:16 loadavg: 0.07 0.02 0.17 1/367 17403
+
+PID    COMM         LADDR                 RADDR                  RX_KB  TX_KB
+17348  sshd         100.66.3.172:22       100.127.69.165:10161    3333      0
+14374  sshd         100.66.3.172:22       100.127.69.165:25219       0      0
+
+20:27:17 loadavg: 0.07 0.02 0.17 2/366 17409
+
+PID    COMM         LADDR                 RADDR                  RX_KB  TX_KB
+17348  17348        100.66.3.172:22       100.127.69.165:10161    6909      2
+
+You can disable the loadavg summary line with -S if needed.
+
+
+USAGE:
+
+# tcptop -h
+usage: tcptop.py [-h] [-C] [-S] [-p PID] [interval] [count]
+
+Summarize TCP send/recv throughput by host
+
+positional arguments:
+  interval           output interval, in seconds (default 1)
+  count              number of outputs
+
+optional arguments:
+  -h, --help         show this help message and exit
+  -C, --noclear      don't clear the screen
+  -S, --nosummary    skip system summary line
+  -p PID, --pid PID  trace this PID only
+
+examples:
+    ./tcptop           # trace TCP send/recv by host
+    ./tcptop -C        # don't clear the screen
+    ./tcptop -p 181    # only trace PID 181

tools/tplist.py

@@ -13,7 +13,7 @@ import os
 import re
 import sys
 
-from bcc import USDTReader
+from bcc import USDT
 
 trace_root = "/sys/kernel/debug/tracing"
 event_root = os.path.join(trace_root, "events")
@@ -21,7 +21,7 @@ event_root = os.path.join(trace_root, "events")
 parser = argparse.ArgumentParser(description=
                 "Display kernel tracepoints or USDT probes and their formats.",
                 formatter_class=argparse.RawDescriptionHelpFormatter)
-parser.add_argument("-p", "--pid", type=int, default=-1, help=
+parser.add_argument("-p", "--pid", type=int, default=None, help=
                 "List USDT probes in the specified process")
 parser.add_argument("-l", "--lib", default="", help=
                 "List USDT probes in the specified library or executable")
@@ -65,23 +65,23 @@ def print_tracepoints():
                                 print_tpoint(category, event)
 
 def print_usdt(pid, lib):
-        reader = USDTReader(bin_path=lib, pid=pid)
+        reader = USDT(path=lib, pid=pid)
         probes_seen = []
-        for probe in reader.probes:
-                probe_name = "%s:%s" % (probe.provider, probe.name)
+        for probe in reader.enumerate_probes():
+                probe_name = probe.short_name()
                 if not args.filter or fnmatch.fnmatch(probe_name, args.filter):
                         if probe_name in probes_seen:
                                 continue
                         probes_seen.append(probe_name)
                         if args.variables:
-                                print(probe.display_verbose())
+                                print(probe)
                         else:
                                 print("%s %s:%s" % (probe.bin_path,
                                                     probe.provider, probe.name))
 
 if __name__ == "__main__":
         try:
-                if args.pid != -1 or args.lib != "":
+                if args.pid or args.lib != "":
                         print_usdt(args.pid, args.lib)
                 else:
                         print_tracepoints()

tools/tplist_example.txt

@@ -17,25 +17,18 @@ $ tplist -l basic_usdt
 /home/vagrant/basic_usdt basic_usdt:loop_iter
 /home/vagrant/basic_usdt basic_usdt:end_main
 
-The loop_iter probe sounds interesting. What are the locations of that
-probe, and which variables are available?
+The loop_iter probe sounds interesting. How many arguments are available?
 
 $ tplist '*loop_iter' -l basic_usdt -v
 /home/vagrant/basic_usdt basic_usdt:loop_iter [sema 0x601036]
-  location 0x400550 raw args: -4@$42 8@%rax
-    4   signed bytes @ constant 42
-    8 unsigned bytes @ register %rax
-  location 0x40056f raw args: 8@-8(%rbp) 8@%rax
-    8 unsigned bytes @ -8(%rbp)
-    8 unsigned bytes @ register %rax
+  2 location(s)
+  2 argument(s)
 
 This output indicates that the loop_iter probe is used in two locations
-in the basic_usdt executable. The first location passes a constant value,
-42, to the probe. The second location passes a variable value located at
-an offset from the %rbp register. Don't worry -- you don't have to trace
-the register values yourself. The argdist and trace tools understand the
-probe format and can print out the arguments automatically -- you can
-refer to them as arg1, arg2, and so on.
+in the basic_usdt executable, and that it has two arguments. Fortunately,
+the argdist and trace tools understand the probe format and can print out
+the arguments automatically -- you can refer to them as arg1, arg2, and
+so on.
 
 Try to explore with some common libraries on your system and see if they
 contain UDST probes. Here are two examples you might find interesting:

tools/trace_example.txt

@@ -84,15 +84,15 @@ trace has also some basic support for kernel tracepoints. For example, let's
 trace the block:block_rq_complete tracepoint and print out the number of sectors
 transferred:
 
-# trace 't:block:block_rq_complete "sectors=%d", tp.nr_sector'
+# trace 't:block:block_rq_complete "sectors=%d", args->nr_sector'
 TIME     PID    COMM         FUNC             -
 01:23:51 0      swapper/0    block_rq_complete sectors=8
 01:23:55 10017  kworker/u64: block_rq_complete sectors=1
 01:23:55 0      swapper/0    block_rq_complete sectors=8
 ^C
 
-To discover the tracepoint structure format (which you can refer to as the "tp"
-variable), use the tplist tool. For example:
+To discover the tracepoint structure format (which you can refer to as the "args"
+pointer variable), use the tplist tool. For example:
 
 # tplist -v block:block_rq_complete
 block:block_rq_complete
@@ -102,7 +102,7 @@ block:block_rq_complete
     int errors;
     char rwbs[8];
 
-This output tells you that you can use "tp.dev", "tp.sector", etc. in your
+This output tells you that you can use "args->dev", "args->sector", etc. in your
 predicate and trace arguments.
 
 As a final example, let's trace open syscalls for a specific process. By 
@@ -169,7 +169,7 @@ trace 'r::__kmalloc (retval == 0) "kmalloc failed!"
         Trace returns from __kmalloc which returned a null pointer
 trace 'r:c:malloc (retval) "allocated = %p", retval
         Trace returns from malloc and print non-NULL allocated buffers
-trace 't:block:block_rq_complete "sectors=%d", tp.nr_sector'
+trace 't:block:block_rq_complete "sectors=%d", args->nr_sector'
         Trace the block_rq_complete kernel tracepoint and print # of tx sectors
 trace 'u:pthread:pthread_create (arg4 != 0)'
         Trace the USDT probe pthread_create when its 4th argument is non-zero