support symbol resolution of short-lived process. (#2144)

New command line options have been added to tools/trace.py to support the 
new BUILD_ID stackmap. List of symbol files can be added to the script to 
resolve symbols from build id as reported by the kernel in the stack trace
Updated man page and added an example usage

man/man8/trace.8

@@ -2,7 +2,7 @@
 .SH NAME
 trace \- Trace a function and print its arguments or return value, optionally evaluating a filter. Uses Linux eBPF/bcc.
 .SH SYNOPSIS
-.B trace [-h] [-b BUFFER_PAGES] [-p PID] [-L TID] [-v] [-Z STRING_SIZE] [-S]
+.B trace [-h] [-b BUFFER_PAGES] [-p PID] [-L TID] [-v] [-Z STRING_SIZE] [-S] [-s SYM_FILE_LIST]
          [-M MAX_EVENTS] [-t] [-T] [-C] [-K] [-U] [-a] [-I header]
          probe [probe ...]
 .SH DESCRIPTION
@@ -28,9 +28,13 @@ Trace only functions in the thread TID.
 Display the generated BPF program, for debugging purposes.
 .TP
 \-z STRING_SIZE
-When collecting string arguments (of type char*), collect up to STRING_SIZE 
+When collecting string arguments (of type char*), collect up to STRING_SIZE
 characters. Longer strings will be truncated.
 .TP
+\-s SYM_FILE_LIST
+When collecting stack trace in build id format, use the coma separated list for
+symbol resolution.
+.TP
 \-S
 If set, trace messages from trace's own process. By default, this is off to
 avoid tracing storms -- for example, if you trace the write system call, and
@@ -177,6 +181,10 @@ Trace the pthread_create USDT probe from the pthread library and print the addre
 Trace the nanosleep system call and print the sleep duration in nanoseconds:
 #
 .B trace 'p::SyS_nanosleep(struct timespec *ts) "sleep for %lld ns", ts->tv_nsec'
+.TP
+Trace the inet_pton system call using build id mechanism and print the stack
+#
+.B trace -s /lib/x86_64-linux-gnu/libc.so.6,/bin/ping 'p:c:inet_pton' -U
 .SH SOURCE
 This is from bcc.
 .IP

tools/trace.py

@@ -4,7 +4,7 @@
 #               parameters, with an optional filter.
 #
 # usage: trace [-h] [-p PID] [-L TID] [-v] [-Z STRING_SIZE] [-S]
-#              [-M MAX_EVENTS] [-T] [-t] [-K] [-U] [-a] [-I header]
+#              [-M MAX_EVENTS] [-s SYMBOLFILES] [-T] [-t] [-K] [-U] [-a] [-I header]
 #              probe [probe ...]
 #
 # Licensed under the Apache License, Version 2.0 (the "License")
@@ -35,6 +35,7 @@ class Probe(object):
         tgid = -1
         pid = -1
         page_cnt = None
+        build_id_enabled = False
 
         @classmethod
         def configure(cls, args):
@@ -49,6 +50,7 @@ class Probe(object):
                 cls.pid = args.pid or -1
                 cls.page_cnt = args.buffer_pages
                 cls.bin_cmp = args.bin_cmp
+                cls.build_id_enabled = args.sym_file_list is not None
 
         def __init__(self, probe, string_size, kernel_stack, user_stack):
                 self.usdt = None
@@ -346,7 +348,9 @@ static inline bool %s(char const *ignored, uintptr_t str) {
                 self.events_name = "%s_events" % self.probe_name
                 self.struct_name = "%s_data_t" % self.probe_name
                 self.stacks_name = "%s_stacks" % self.probe_name
-                stack_table = "BPF_STACK_TRACE(%s, 1024);" % self.stacks_name \
+                stack_type = "BPF_STACK_TRACE" if self.build_id_enabled is False \
+                             else "BPF_STACK_TRACE_BUILDID"
+                stack_table = "%s(%s, 1024);" % (stack_type,self.stacks_name) \
                               if (self.kernel_stack or self.user_stack) else ""
                 data_fields = ""
                 for i, field_type in enumerate(self.types):
@@ -693,6 +697,10 @@ trace -I 'linux/fs_struct.h' 'mntns_install "users = %d", $task->fs->users'
                   help="print CPU id")
                 parser.add_argument("-B", "--bin_cmp", action="store_true",
                   help="allow to use STRCMP with binary values")
+                parser.add_argument('-s', "--sym_file_list", type=str, \
+                  metavar="SYM_FILE_LIST", dest="sym_file_list", \
+                  help="coma separated list of symbol files to use \
+                  for symbol resolution")
                 parser.add_argument("-K", "--kernel-stack",
                   action="store_true", help="output kernel stack trace")
                 parser.add_argument("-U", "--user-stack",
@@ -757,6 +765,9 @@ trace -I 'linux/fs_struct.h' 'mntns_install "users = %d", $task->fs->users'
                                 print(probe.usdt.get_text())
                         usdt_contexts.append(probe.usdt)
                 self.bpf = BPF(text=self.program, usdt_contexts=usdt_contexts)
+                if self.args.sym_file_list is not None:
+                  print("Note: Kernel bpf will report stack map with ip/build_id")
+                  map(lambda x: self.bpf.add_module(x), self.args.sym_file_list.split(','))
                 for probe in self.probes:
                         if self.args.verbose:
                                 print(probe)

tools/trace_example.txt

@@ -104,6 +104,19 @@ TIME     PID    COMM         FUNC             -
 01:23:55 0      swapper/0    block_rq_complete sectors=8
 ^C
 
+Suppose that you want to trace a system-call in a short-lived process, you can use
+the -s option to trace. The option is followed by list of libraries/executables to
+use for symbol resolution.
+# trace -s /lib/x86_64-linux-gnu/libc.so.6,/bin/ping 'p:c:inet_pton' -U
+Note: Kernel bpf will report stack map with ip/build_id
+PID     TID     COMM            FUNC
+4175    4175    ping            inet_pton
+        inet_pton+0x136340 [libc.so.6]
+        getaddrinfo+0xfb510 [libc.so.6]
+        _init+0x2a08 [ping]
+
+During the trace, 'ping -c1 google.com' was executed to obtain the above results
+
 To discover the tracepoint structure format (which you can refer to as the "args"
 pointer variable), use the tplist tool. For example:
 
@@ -268,6 +281,8 @@ optional arguments:
   -v, --verbose         print resulting BPF program code before executing
   -Z STRING_SIZE, --string-size STRING_SIZE
                         maximum size to read from strings
+  -s SYM_FILE_LIST      when collecting stack trace in build id format,
+                        use the coma separated list for symbol resolution
   -S, --include-self    do not filter trace's own pid from the trace
   -M MAX_EVENTS, --max-events MAX_EVENTS
                         number of events to print before quitting
@@ -325,4 +340,7 @@ trace -I 'net/sock.h' \\
         to 53 (DNS; 13568 in big endian order)
 trace -I 'linux/fs_struct.h' 'mntns_install "users = %d", $task->fs->users'
         Trace the number of users accessing the file system of the current task
+trace -s /lib/x86_64-linux-gnu/libc.so.6,/bin/ping 'p:c:inet_pton' -U
+        Trace inet_pton system call and use the specified libraries/executables for
+        symbol resolution.
 "