tplist: Print USDT locations and arguments (#734)

* cc: Add USDT location and argument reporting

libbcc now exposes USDT location and argument information using
two new APIs: `bcc_usdt_get_location` and `bcc_usdt_get_argument`.

* python: Retrieve USDT locations and arguments

Add wrappers in the libbcc.py file to access the new APIs for
retrieving USDT location and argument information. Also add
high-level classes in usdt.py to access this information and
format arguments and locations in a shape suitable for display.

* tplist: Print USDT locations and arguments

Add super-verbose mode (-vv) to tplist where it prints USDT locations
and arguments including full detail on registers, offsets, and global
identifier offsets.

man/man8/tplist.8

@@ -22,7 +22,8 @@ Display the USDT probes from the specified library or executable. If the librar
 or executable can be found in the standard paths, a full path is not required.
 .TP
 \-v
-Display the variables associated with the tracepoint or USDT probe.
+Increase the verbosity level. Can be used to display the variables, locations,
+and arguments of tracepoints and USDT probes.
 .TP
 [filter]
 A wildcard expression that specifies which tracepoints or probes to print.
@@ -45,6 +46,10 @@ $
 Print all USDT probes in process 4717 from the libc provider:
 $
 .B tplist -p 4717 'libc:*'
+.TP
+Print all the USDT probes in the node executable:
+$
+.B tplist -l node
 .SH SOURCE
 This is from bcc.
 .IP

src/cc/bcc_usdt.h

@@ -35,8 +35,32 @@ struct bcc_usdt {
     int num_arguments;
 };
 
+struct bcc_usdt_location {
+    uint64_t address;
+};
+
+#define BCC_USDT_ARGUMENT_NONE          0x0
+#define BCC_USDT_ARGUMENT_CONSTANT      0x1
+#define BCC_USDT_ARGUMENT_DEREF_OFFSET  0x2
+#define BCC_USDT_ARGUMENT_DEREF_IDENT   0x4
+#define BCC_USDT_ARGUMENT_REGISTER_NAME 0x8
+
+struct bcc_usdt_argument {
+    int size;
+    int valid;
+    int constant;
+    int deref_offset;
+    const char *deref_ident;
+    const char *register_name;
+};
+
 typedef void (*bcc_usdt_cb)(struct bcc_usdt *);
 void bcc_usdt_foreach(void *usdt, bcc_usdt_cb callback);
+int bcc_usdt_get_location(void *usdt, const char *probe_name,
+                          int index, struct bcc_usdt_location *location);
+int bcc_usdt_get_argument(void *usdt, const char *probe_name,
+                          int location_index, int argument_index,
+                          struct bcc_usdt_argument *argument);
 
 int bcc_usdt_enable_probe(void *, const char *, const char *);
 const char *bcc_usdt_genargs(void *);

src/cc/usdt.cc

@@ -28,7 +28,7 @@
 
 namespace USDT {
 
-Probe::Location::Location(uint64_t addr, const char *arg_fmt) : address_(addr) {
+Location::Location(uint64_t addr, const char *arg_fmt) : address_(addr) {
   ArgumentParser_x64 parser(arg_fmt);
   while (!parser.done()) {
     Argument arg;
@@ -274,7 +274,7 @@ void Context::each_uprobe(each_uprobe_cb callback) {
     if (!p->enabled())
       continue;
 
-    for (Probe::Location &loc : p->locations_) {
+    for (Location &loc : p->locations_) {
       callback(p->bin_path_.c_str(), p->attached_to_->c_str(), loc.address_,
                pid_.value_or(-1));
     }
@@ -357,6 +357,52 @@ void bcc_usdt_foreach(void *usdt, bcc_usdt_cb callback) {
   ctx->each(callback);
 }
 
+int bcc_usdt_get_location(void *usdt, const char *probe_name,
+                          int index, struct bcc_usdt_location *location) {
+    USDT::Context *ctx = static_cast<USDT::Context *>(usdt);
+    USDT::Probe *probe = ctx->get(probe_name);
+    if (!probe)
+        return -1;
+    if (index < 0 || (size_t)index >= probe->num_locations())
+        return -1;
+    location->address = probe->address(index);
+    return 0;
+}
+
+int bcc_usdt_get_argument(void *usdt, const char *probe_name,
+                          int location_index, int argument_index,
+                          struct bcc_usdt_argument *argument) {
+    USDT::Context *ctx = static_cast<USDT::Context *>(usdt);
+    USDT::Probe *probe = ctx->get(probe_name);
+    if (!probe)
+        return -1;
+    if (argument_index < 0 || (size_t)argument_index >= probe->num_arguments())
+        return -1;
+    if (location_index < 0 || (size_t)location_index >= probe->num_locations())
+        return -1;
+    auto const &location = probe->location(location_index);
+    auto const &arg = location.arguments_[argument_index];
+    argument->size = arg.arg_size();
+    argument->valid = BCC_USDT_ARGUMENT_NONE;
+    if (arg.constant()) {
+        argument->valid |= BCC_USDT_ARGUMENT_CONSTANT;
+        argument->constant = *(arg.constant());
+    }
+    if (arg.deref_offset()) {
+        argument->valid |= BCC_USDT_ARGUMENT_DEREF_OFFSET;
+        argument->deref_offset = *(arg.deref_offset());
+    }
+    if (arg.deref_ident()) {
+        argument->valid |= BCC_USDT_ARGUMENT_DEREF_IDENT;
+        argument->deref_ident = arg.deref_ident()->c_str();
+    }
+    if (arg.register_name()) {
+        argument->valid |= BCC_USDT_ARGUMENT_REGISTER_NAME;
+        argument->register_name = arg.register_name()->c_str();
+    }
+    return 0;
+}
+
 void bcc_usdt_foreach_uprobe(void *usdt, bcc_usdt_uprobe_cb callback) {
   USDT::Context *ctx = static_cast<USDT::Context *>(usdt);
   ctx->each_uprobe(callback);

src/cc/usdt.h

@@ -117,18 +117,18 @@ public:
   ArgumentParser_x64(const char *arg) : ArgumentParser(arg) {}
 };
 
+struct Location {
+  uint64_t address_;
+  std::vector<Argument> arguments_;
+  Location(uint64_t addr, const char *arg_fmt);
+};
+
 class Probe {
   std::string bin_path_;
   std::string provider_;
   std::string name_;
   uint64_t semaphore_;
 
-  struct Location {
-    uint64_t address_;
-    std::vector<Argument> arguments_;
-    Location(uint64_t addr, const char *arg_fmt);
-  };
-
   std::vector<Location> locations_;
 
   optional<int> pid_;
@@ -153,6 +153,7 @@ public:
   uint64_t semaphore()   const { return semaphore_; }
 
   uint64_t address(size_t n = 0) const { return locations_[n].address_; }
+  const Location &location(size_t n) const { return locations_[n]; }
   bool usdt_getarg(std::ostream &stream);
   std::string get_arg_ctype(int arg_index) {
     return largest_arg_type(arg_index);

src/python/bcc/libbcc.py

@@ -174,11 +174,41 @@ class bcc_usdt(ct.Structure):
             ('num_arguments', ct.c_int),
         ]
 
+class bcc_usdt_location(ct.Structure):
+    _fields_ = [
+            ('address', ct.c_ulonglong)
+        ]
+
+class BCC_USDT_ARGUMENT_FLAGS(object):
+    NONE = 0x0
+    CONSTANT = 0x1
+    DEREF_OFFSET = 0x2
+    DEREF_IDENT = 0x4
+    REGISTER_NAME = 0x8
+
+class bcc_usdt_argument(ct.Structure):
+    _fields_ = [
+            ('size', ct.c_int),
+            ('valid', ct.c_int),
+            ('constant', ct.c_int),
+            ('deref_offset', ct.c_int),
+            ('deref_ident', ct.c_char_p),
+            ('register_name', ct.c_char_p)
+        ]
+
 _USDT_CB = ct.CFUNCTYPE(None, ct.POINTER(bcc_usdt))
 
 lib.bcc_usdt_foreach.restype = None
 lib.bcc_usdt_foreach.argtypes = [ct.c_void_p, _USDT_CB]
 
+lib.bcc_usdt_get_location.restype = ct.c_int
+lib.bcc_usdt_get_location.argtypes = [ct.c_void_p, ct.c_char_p, ct.c_int,
+                                      ct.POINTER(bcc_usdt_location)]
+
+lib.bcc_usdt_get_argument.restype = ct.c_int
+lib.bcc_usdt_get_argument.argtypes = [ct.c_void_p, ct.c_char_p, ct.c_int,
+                                      ct.c_int, ct.POINTER(bcc_usdt_argument)]
+
 _USDT_PROBE_CB = ct.CFUNCTYPE(None, ct.c_char_p, ct.c_char_p,
                               ct.c_ulonglong, ct.c_int)
 

src/python/bcc/usdt.py

@@ -12,25 +12,100 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .libbcc import lib, _USDT_CB, _USDT_PROBE_CB
+import ctypes as ct
+from .libbcc import lib, _USDT_CB, _USDT_PROBE_CB, \
+                    bcc_usdt_location, bcc_usdt_argument, \
+                    BCC_USDT_ARGUMENT_FLAGS
+
+class USDTProbeArgument(object):
+    def __init__(self, argument):
+        self.signed = argument.size < 0
+        self.size = abs(argument.size)
+        self.valid = argument.valid
+        if self.valid & BCC_USDT_ARGUMENT_FLAGS.CONSTANT != 0:
+            self.constant = argument.constant
+        if self.valid & BCC_USDT_ARGUMENT_FLAGS.DEREF_OFFSET != 0:
+            self.deref_offset = argument.deref_offset
+        if self.valid & BCC_USDT_ARGUMENT_FLAGS.DEREF_IDENT != 0:
+            self.deref_ident = argument.deref_ident
+        if self.valid & BCC_USDT_ARGUMENT_FLAGS.REGISTER_NAME != 0:
+            self.register_name = argument.register_name
+
+    def _size_prefix(self):
+        return "%d %s bytes" % \
+                (self.size, "signed  " if self.signed else "unsigned")
+
+    def _format(self):
+        # This mimics the logic in cc/usdt_args.cc that gives meaning to the
+        # various argument settings. A change there will require a change here.
+        if self.valid & BCC_USDT_ARGUMENT_FLAGS.CONSTANT != 0:
+            return "%d" % self.constant
+        if self.valid & BCC_USDT_ARGUMENT_FLAGS.DEREF_OFFSET == 0:
+            return "%s" % self.register_name
+        if self.valid & BCC_USDT_ARGUMENT_FLAGS.DEREF_OFFSET != 0 and \
+           self.valid & BCC_USDT_ARGUMENT_FLAGS.DEREF_IDENT == 0:
+            sign = '+' if self.deref_offset >= 0 else '-'
+            return "*(%s %s %d)" % (self.register_name,
+                                    sign, abs(self.deref_offset))
+        if self.valid & BCC_USDT_ARGUMENT_FLAGS.DEREF_OFFSET != 0 and \
+           self.valid & BCC_USDT_ARGUMENT_FLAGS.DEREF_IDENT != 0 and \
+           self.valid & BCC_USDT_ARGUMENT_FLAGS.REGISTER_NAME != 0 and \
+           self.register_name == "ip":
+            sign = '+' if self.deref_offset >= 0 else '-'
+            return "*(&%s %s %d)" % (self.deref_ident,
+                                     sign, abs(self.deref_offset))
+        # If we got here, this is an unrecognized case. Doesn't mean it's
+        # necessarily bad, so just provide the raw data. It just means that
+        # other tools won't be able to work with this argument.
+        return "unrecognized argument format, flags %d" % self.valid
+
+    def __str__(self):
+        return "%s @ %s" % (self._size_prefix(), self._format())
+
+class USDTProbeLocation(object):
+    def __init__(self, probe, index, location):
+        self.probe = probe
+        self.index = index
+        self.num_arguments = probe.num_arguments
+        self.address = location.address
+
+    def __str__(self):
+        return "0x%x" % self.address
+
+    def get_argument(self, index):
+        arg = bcc_usdt_argument()
+        res = lib.bcc_usdt_get_argument(self.probe.context, self.probe.name,
+                                        self.index, index, ct.pointer(arg))
+        if res != 0:
+            raise Exception("error retrieving probe argument %d location %d" %
+                            (index, self.index))
+        return USDTProbeArgument(arg)
 
 class USDTProbe(object):
-    def __init__(self, usdt):
-        self.provider = usdt.provider
-        self.name = usdt.name
-        self.bin_path = usdt.bin_path
-        self.semaphore = usdt.semaphore
-        self.num_locations = usdt.num_locations
-        self.num_arguments = usdt.num_arguments
+    def __init__(self, context, probe):
+        self.context = context
+        self.provider = probe.provider
+        self.name = probe.name
+        self.bin_path = probe.bin_path
+        self.semaphore = probe.semaphore
+        self.num_locations = probe.num_locations
+        self.num_arguments = probe.num_arguments
 
     def __str__(self):
-        return "%s %s:%s [sema 0x%x]\n  %d location(s)\n  %d argument(s)" % \
-               (self.bin_path, self.provider, self.name, self.semaphore,
-                self.num_locations, self.num_arguments)
+        return "%s %s:%s [sema 0x%x]" % \
+               (self.bin_path, self.provider, self.name, self.semaphore)
 
     def short_name(self):
         return "%s:%s" % (self.provider, self.name)
 
+    def get_location(self, index):
+        loc = bcc_usdt_location()
+        res = lib.bcc_usdt_get_location(self.context, self.name,
+                                        index, ct.pointer(loc))
+        if res != 0:
+            raise Exception("error retrieving probe location %d" % index)
+        return USDTProbeLocation(self, index, loc)
+
 class USDT(object):
     def __init__(self, pid=None, path=None):
         if pid and pid != -1:
@@ -62,7 +137,7 @@ class USDT(object):
     def enumerate_probes(self):
         probes = []
         def _add_probe(probe):
-            probes.append(USDTProbe(probe.contents))
+            probes.append(USDTProbe(self.context, probe.contents))
 
         lib.bcc_usdt_foreach(self.context, _USDT_CB(_add_probe))
         return probes

tools/tplist.py

@@ -25,8 +25,8 @@ parser.add_argument("-p", "--pid", type=int, default=None, help=
                 "List USDT probes in the specified process")
 parser.add_argument("-l", "--lib", default="", help=
                 "List USDT probes in the specified library or executable")
-parser.add_argument("-v", dest="variables", action="store_true", help=
-                "Print the format (available variables)")
+parser.add_argument("-v", dest="verbosity", action="count", help=
+                "Increase verbosity level (print variables, arguments, etc.)")
 parser.add_argument(dest="filter", nargs="?", help=
                 "A filter that specifies which probes/tracepoints to print")
 args = parser.parse_args()
@@ -51,7 +51,7 @@ def print_tpoint(category, event):
         tpoint = "%s:%s" % (category, event)
         if not args.filter or fnmatch.fnmatch(tpoint, args.filter):
                 print(tpoint)
-                if args.variables:
+                if args.verbosity > 0:
                         print_tpoint_format(category, event)
 
 def print_tracepoints():
@@ -64,6 +64,25 @@ def print_tracepoints():
                         if os.path.isdir(evt_dir):
                                 print_tpoint(category, event)
 
+def print_usdt_argument_details(location):
+        for idx in xrange(0, location.num_arguments):
+                arg = location.get_argument(idx)
+                print("    argument #%d %s" % (idx, arg))
+
+def print_usdt_details(probe):
+        if args.verbosity > 0:
+                print(probe)
+                if args.verbosity > 1:
+                        for idx in xrange(0, probe.num_locations):
+                                loc = probe.get_location(idx)
+                                print("  location #%d %s" % (idx, loc))
+                                print_usdt_argument_details(loc)
+                else:
+                        print("  %d location(s)" % probe.num_locations)
+                        print("  %d argument(s)" % probe.num_arguments)
+        else:
+                print("%s %s:%s" % (probe.bin_path, probe.provider, probe.name))
+
 def print_usdt(pid, lib):
         reader = USDT(path=lib, pid=pid)
         probes_seen = []
@@ -73,11 +92,7 @@ def print_usdt(pid, lib):
                         if probe_name in probes_seen:
                                 continue
                         probes_seen.append(probe_name)
-                        if args.variables:
-                                print(probe)
-                        else:
-                                print("%s %s:%s" % (probe.bin_path,
-                                                    probe.provider, probe.name))
+                        print_usdt_details(probe)
 
 if __name__ == "__main__":
         try:

tools/tplist_example.txt

@@ -88,6 +88,31 @@ The dev, sector, nr_sector, etc. variables can now all be used in probes
 you specify with argdist or trace.
 
 
+For debugging USDT probes, it is sometimes useful to see the exact locations
+and arguments of the probes, including the registers or global variables from
+which their values are coming from. In super-verbose mode, tplist will print
+this information (note the -vv):
+
+$ tplist -vv -l c *alloc*
+/lib64/libc.so.6 libc:memory_malloc_retry [sema 0x0]
+  location #0 0x835c0
+    argument #0 8 unsigned bytes @ bp
+  location #1 0x83778
+    argument #0 8 unsigned bytes @ bp
+  location #2 0x85a50
+    argument #0 8 unsigned bytes @ bp
+/lib64/libc.so.6 libc:memory_realloc_retry [sema 0x0]
+  location #0 0x84b90
+    argument #0 8 unsigned bytes @ r13
+    argument #1 8 unsigned bytes @ bp
+  location #1 0x85cf0
+    argument #0 8 unsigned bytes @ r13
+    argument #1 8 unsigned bytes @ bp
+/lib64/libc.so.6 libc:memory_calloc_retry [sema 0x0]
+  location #0 0x850f0
+    argument #0 8 unsigned bytes @ bp
+
+
 USAGE message:
 
 $ tplist -h
@@ -102,5 +127,5 @@ optional arguments:
   -h, --help         show this help message and exit
   -p PID, --pid PID  List USDT probes in the specified process
   -l LIB, --lib LIB  List USDT probes in the specified library or executable
-  -v                 Print the format (available variables)
+  -v                 Increase verbosity level (print variables, arguments, etc.)