Merge pull request #1555 from myreg/trace-addr

Add option to print virtual address to trace.py

man/man8/trace.8

@@ -3,7 +3,7 @@
 trace \- Trace a function and print its arguments or return value, optionally evaluating a filter. Uses Linux eBPF/bcc.
 .SH SYNOPSIS
 .B trace [-h] [-b BUFFER_PAGES] [-p PID] [-L TID] [-v] [-Z STRING_SIZE] [-S]
-         [-M MAX_EVENTS] [-t] [-T] [-C] [-K] [-U] [-I header]
+         [-M MAX_EVENTS] [-t] [-T] [-C] [-K] [-U] [-a] [-I header]
          probe [probe ...]
 .SH DESCRIPTION
 trace probes functions you specify and displays trace messages if a particular
@@ -53,6 +53,8 @@ Print the kernel stack for each event.
 .TP
 \-U
 Print the user stack for each event.
+\-a
+Print virtual address in kernel and user stacks.
 .TP
 \-I header
 Additional header files to include in the BPF program. This is needed if your

tools/trace.py

@@ -4,7 +4,7 @@
 #               parameters, with an optional filter.
 #
 # usage: trace [-h] [-p PID] [-L TID] [-v] [-Z STRING_SIZE] [-S]
-#              [-M MAX_EVENTS] [-T] [-t] [-K] [-U] [-I header]
+#              [-M MAX_EVENTS] [-T] [-t] [-K] [-U] [-a] [-I header]
 #              probe [probe ...]
 #
 # Licensed under the Apache License, Version 2.0 (the "License")
@@ -31,6 +31,7 @@ class Probe(object):
         use_localtime = True
         time_field = False
         print_cpu = False
+        print_address = False
         tgid = -1
         pid = -1
         page_cnt = None
@@ -42,6 +43,7 @@ class Probe(object):
                 cls.use_localtime = not args.timestamp
                 cls.time_field = cls.print_time and (not cls.use_localtime)
                 cls.print_cpu = args.print_cpu
+                cls.print_address = args.address
                 cls.first_ts = BPF.monotonic_time()
                 cls.tgid = args.tgid or -1
                 cls.pid = args.pid or -1
@@ -474,13 +476,16 @@ BPF_PERF_OUTPUT(%s);
 
         def print_stack(self, bpf, stack_id, tgid):
             if stack_id < 0:
-                    print("        %d" % stack_id)
-                    return
+                print("        %d" % stack_id)
+                return
 
             stack = list(bpf.get_table(self.stacks_name).walk(stack_id))
             for addr in stack:
-                    print("        %s" % (bpf.sym(addr, tgid,
-                                         show_module=True, show_offset=True)))
+                print("        ", end="")
+                if Probe.print_address:
+                    print("%16x " % addr, end="")
+                print("%s" % (bpf.sym(addr, tgid,
+                                     show_module=True, show_offset=True)))
 
         def _format_message(self, bpf, tgid, values):
                 # Replace each %K with kernel sym and %U with user sym in tgid
@@ -645,6 +650,8 @@ trace -I 'linux/fs_struct.h' 'mntns_install "users = %d", $task->fs->users'
                   action="store_true", help="output kernel stack trace")
                 parser.add_argument("-U", "--user-stack",
                   action="store_true", help="output user stack trace")
+                parser.add_argument("-a", "--address", action="store_true",
+                  help="print virtual address in stacks")
                 parser.add_argument(metavar="probe", dest="probes", nargs="+",
                   help="probe specifier (see examples)")
                 parser.add_argument("-I", "--include", action="append",

tools/trace_example.txt

@@ -60,12 +60,25 @@ PID    COMM         FUNC             -
 2740   bash         readline         man ls
 ^C
 
-The special retval keywords stands for the function's return value, and can
+The special retval keyword stands for the function's return value, and can
 be used only in a retprobe, specified by the 'r' prefix. The next component
 of the probe is the library that contains the desired function. It's OK to
 specify executables too, as long as they can be found in the PATH. Or, you
 can specify the full path to the executable (e.g. "/usr/bin/bash").
 
+Sometimes it can be useful to see where in code the events happen. There are
+flags to print the kernel stack (-K), the user stack (-U) and optionally
+include the virtual address in the stacks as well (-a):
+
+# trace.py -U -a 'r::sys_futex "%d", retval'
+PID     TID     COMM            FUNC             -
+793922  793951  poller          sys_futex        0
+        7f6c72b6497a __lll_unlock_wake+0x1a [libpthread-2.23.so]
+              627fef folly::FunctionScheduler::run()+0x46f [router]
+        7f6c7345f171 execute_native_thread_routine+0x21 [libstdc++.so.6.0.21]
+        7f6c72b5b7a9 start_thread+0xd9 [libpthread-2.23.so]
+        7f6c7223fa7d clone+0x6d [libc-2.23.so]
+
 Multiple probes can be combined on the same command line. For example, let's
 trace failed read and write calls on the libc level, and include a time column:
 
@@ -225,7 +238,7 @@ size and is measured in pages. The value must be a power of two and defaults to
 USAGE message:
 
 usage: trace [-h] [-b BUFFER_PAGES] [-p PID] [-L TID] [-v] [-Z STRING_SIZE]
-             [-S] [-M MAX_EVENTS] [-t] [-T] [-K] [-U] [-I header]
+             [-S] [-M MAX_EVENTS] [-t] [-T] [-K] [-U] [-a] [-I header]
              probe [probe ...]
 
 Attach to functions and print trace messages.
@@ -251,6 +264,7 @@ optional arguments:
   -C, --print_cpu       print CPU id 
   -K, --kernel-stack    output kernel stack trace
   -U, --user-stack      output user stack trace
+  -a, --address         print virtual address in stacks
   -I header, --include header
                         additional header files to include in the BPF program
                         as either full path, or relative to current working directory,