Merge pull request #967 from goldshtn/debuginfo

External debuginfo support and general symbols overhaul

docs/reference_guide.md

@@ -65,10 +65,9 @@ This guide is incomplete. If something feels missing, check the bcc and kernel s
         - [7. print_linear_hist()](#6-print_linear_hist)
     - [Helpers](#helpers)
         - [1. ksym()](#1-ksym)
-        - [2. ksymaddr()](#2-ksymaddr)
-        - [3. ksymname()](#3-ksymname)
-        - [4. sym()](#4-sym)
-        - [5. num_open_kprobes()](#5-num_open_kprobes)
+        - [2. ksymname()](#2-ksymname)
+        - [3. sym()](#3-sym)
+        - [4. num_open_kprobes()](#4-num_open_kprobes)
 
 - [BPF Errors](#bpf-errors)
     - [1. Invalid mem access](#1-invalid-mem-access)
@@ -1090,27 +1089,11 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=ksym+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=ksym+path%3Atools+language%3Apython&type=Code)
 
-### 2. ksymaddr()
-
-Syntax: ```BPF.ksymaddr(addr)```
-
-Translate a kernel memory address into a kernel function name plus the instruction offset as a hexadecimal number, which is returned as a string.
-
-Example:
-
-```Python
-print("kernel function+offset: " + b.ksymaddr(addr))
-```
-
-Examples in situ:
-[search /examples](https://github.com/iovisor/bcc/search?q=ksymaddr+path%3Aexamples+language%3Apython&type=Code),
-[search /tools](https://github.com/iovisor/bcc/search?q=ksymaddr+path%3Atools+language%3Apython&type=Code)
-
-### 3. ksymname()
+### 2. ksymname()
 
 Syntax: ```BPF.ksymname(name)```
 
-Translate a kernel name into an address. This is the reverse of ksymaddr. Returns -1 when the function name is unknown.
+Translate a kernel name into an address. This is the reverse of ksym. Returns -1 when the function name is unknown.
 
 Example:
 
@@ -1122,11 +1105,11 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=ksymname+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=ksymname+path%3Atools+language%3Apython&type=Code)
 
-### 4. sym()
+### 3. sym()
 
-Syntax: ```BPF.sym(addr, pid)```
+Syntax: ```BPF.sym(addr, pid, show_module=False, show_offset=False)```
 
-Translate a memory address into a function name for a pid, which is returned. A pid of less than zero will access the kernel symbol cache.
+Translate a memory address into a function name for a pid, which is returned. A pid of less than zero will access the kernel symbol cache. The `show_module` and `show_offset` parameters control whether the module in which the symbol lies should be displayed, and whether the instruction offset from the beginning of the symbol should be displayed. These extra parameters default to `False`.
 
 Example:
 
@@ -1138,9 +1121,9 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=sym+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=sym+path%3Atools+language%3Apython&type=Code)
 
-### 5. num_open_kprobes()
+### 4. num_open_kprobes()
 
-Syntax: ```BPF.num_open_probes()```
+Syntax: ```BPF.num_open_kprobes()```
 
 Returns the number of open k[ret]probes. Can be useful for scenarios where event_re is used while attaching and detaching probes. Excludes perf_events readers.
 

examples/tracing/mallocstacks.py

@@ -11,7 +11,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 
 from __future__ import print_function
-from bcc import BPF, ProcessSymbols
+from bcc import BPF
 from time import sleep
 import sys
 
@@ -43,8 +43,6 @@ int alloc_enter(struct pt_regs *ctx, size_t size) {
 b.attach_uprobe(name="c", sym="malloc", fn_name="alloc_enter", pid=pid)
 print("Attaching to malloc in pid %d, Ctrl+C to quit." % pid)
 
-decoder = ProcessSymbols(pid)
-
 # sleep until Ctrl-C
 try:
     sleep(99999999)
@@ -57,4 +55,4 @@ stack_traces = b.get_table("stack_traces")
 for k, v in reversed(sorted(calls.items(), key=lambda c: c[1].value)):
     print("%d bytes allocated at:" % v.value)
     for addr in stack_traces.walk(k.value):
-        print("\t%s (%x)" % (decoder.decode_addr(addr), addr))
+        print("\t%s" % b.sym(addr, pid, show_offset=True))

src/cc/bcc_elf.c

@@ -15,9 +15,14 @@
  */
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <string.h>
+#include <libgen.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
 
 #include <gelf.h>
 #include "bcc_elf.h"
@@ -196,20 +201,248 @@ static int listsymbols(Elf *e, bcc_elf_symcb callback, void *payload) {
   return 0;
 }
 
-int bcc_elf_foreach_sym(const char *path, bcc_elf_symcb callback,
-                        void *payload) {
+static Elf_Data * get_section_elf_data(Elf *e, const char *section_name) {
+  Elf_Scn *section = NULL;
+  GElf_Shdr header;
+  char *name;
+
+  size_t stridx;
+  if (elf_getshdrstrndx(e, &stridx) != 0)
+    return NULL;
+
+  while ((section = elf_nextscn(e, section)) != 0) {
+    if (!gelf_getshdr(section, &header))
+      continue;
+
+    name = elf_strptr(e, stridx, header.sh_name);
+    if (name && !strcmp(name, section_name)) {
+      return elf_getdata(section, NULL);
+    }
+  }
+
+  return NULL;
+}
+
+static int find_debuglink(Elf *e, char **debug_file, unsigned int *crc) {
+  Elf_Data *data = NULL;
+
+  *debug_file = NULL;
+  *crc = 0;
+
+  data = get_section_elf_data(e, ".gnu_debuglink");
+  if (!data || data->d_size <= 5)
+    return 0;
+
+  *debug_file = (char *)data->d_buf;
+  *crc = *(unsigned int*)((char *)data->d_buf + data->d_size - 4);
+
+  return *debug_file ? 1 : 0;
+}
+
+static int find_buildid(Elf *e, char *buildid) {
+  Elf_Data *data = get_section_elf_data(e, ".note.gnu.build-id");
+  if (data->d_size <= 16 || strcmp((char *)data->d_buf + 12, "GNU"))
+    return 0;
+
+  char *buf = (char *)data->d_buf + 16;
+  size_t length = data->d_size - 16;
+  for (size_t i = 0; i < length; ++i) {
+    sprintf(buildid + (i * 2), "%02hhx", buf[i]);
+  }
+
+  return 1;
+}
+
+// The CRC algorithm used by GNU debuglink. Taken from:
+//    https://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html
+static unsigned int gnu_debuglink_crc32(unsigned int crc,
+                                        char *buf, size_t len) {
+  static const unsigned int crc32_table[256] =
+  {
+    0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419,
+    0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4,
+    0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07,
+    0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
+    0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856,
+    0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
+    0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4,
+    0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+    0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3,
+    0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a,
+    0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599,
+    0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+    0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190,
+    0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f,
+    0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e,
+    0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+    0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed,
+    0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
+    0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3,
+    0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
+    0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a,
+    0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5,
+    0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010,
+    0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+    0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17,
+    0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6,
+    0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615,
+    0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
+    0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344,
+    0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
+    0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a,
+    0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+    0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1,
+    0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c,
+    0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
+    0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+    0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe,
+    0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31,
+    0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c,
+    0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+    0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b,
+    0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
+    0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1,
+    0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
+    0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278,
+    0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7,
+    0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66,
+    0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+    0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605,
+    0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8,
+    0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b,
+    0x2d02ef8d
+  };
+  char *end;
+
+  crc = ~crc & 0xffffffff;
+  for (end = buf + len; buf < end; ++buf)
+    crc = crc32_table[(crc ^ *buf) & 0xff] ^ (crc >> 8);
+  return ~crc & 0xffffffff;
+}
+
+static int verify_checksum(const char *file, unsigned int crc) {
+  struct stat st;
+  int fd;
+  void *buf;
+  unsigned int actual;
+
+  fd = open(file, O_RDONLY);
+  if (fd < 0)
+    return 0;
+
+  if (fstat(fd, &st) < 0)
+    return 0;
+
+  buf = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+  if (!buf) {
+    close(fd);
+    return 0;
+  }
+
+  actual = gnu_debuglink_crc32(0, buf, st.st_size);
+
+  munmap(buf, st.st_size);
+  close(fd);
+  return actual == crc;
+}
+
+static char *find_debug_via_debuglink(Elf *e, const char *binpath) {
+  char fullpath[PATH_MAX];
+  char *bindir = NULL;
+  char *res = NULL;
+  unsigned int crc;
+  char *name;  // the name of the debuginfo file
+
+  if (!find_debuglink(e, &name, &crc))
+    return NULL;
+
+  bindir = strdup(binpath);
+  bindir = dirname(bindir);
+
+  // Search for the file in 'binpath'
+  sprintf(fullpath, "%s/%s", bindir, name);
+  if (access(fullpath, F_OK) != -1) {
+    res = strdup(fullpath);
+    goto DONE;
+  }
+
+  // Search for the file in 'binpath'/.debug
+  sprintf(fullpath, "%s/.debug/%s", bindir, name);
+  if (access(fullpath, F_OK) != -1) {
+    res = strdup(fullpath);
+    goto DONE;
+  }
+
+  // Search for the file in the global debug directory /usr/lib/debug/'binpath'
+  sprintf(fullpath, "/usr/lib/debug%s/%s", bindir, name);
+  if (access(fullpath, F_OK) != -1) {
+    res = strdup(fullpath);
+    goto DONE;
+  }
+
+DONE:
+  free(bindir);
+  if (verify_checksum(res, crc))
+    return res;
+  return NULL;
+}
+
+static char *find_debug_via_buildid(Elf *e) {
+  char fullpath[PATH_MAX];
+  char buildid[128];  // currently 40 seems to be default, let's be safe
+
+  if (!find_buildid(e, buildid))
+    return NULL;
+
+  // Search for the file in the global debug directory with a sub-path:
+  //    mm/nnnnnn...nnnn.debug
+  // Where mm are the first two characters of the buildid, and nnnn are the
+  // rest of the build id, followed by .debug.
+  sprintf(fullpath, "/usr/lib/debug/.build-id/%c%c/%s.debug",
+          buildid[0], buildid[1], buildid + 2);
+  if (access(fullpath, F_OK) != -1) {
+    return strdup(fullpath);
+  }
+
+  return NULL;
+}
+
+static int foreach_sym_core(const char *path, bcc_elf_symcb callback,
+                            void *payload, int is_debug_file) {
   Elf *e;
   int fd, res;
+  char *debug_file;
 
   if (openelf(path, &e, &fd) < 0)
     return -1;
 
+  // If there is a separate debuginfo file, try to locate and read it, first
+  // using the build-id section, then using the debuglink section. These are
+  // also the rules that GDB folows.
+  // See: https://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html
+  if (!is_debug_file) {
+    // The is_debug_file argument helps avoid infinitely resolving debuginfo
+    // files for debuginfo files and so on.
+    debug_file = find_debug_via_buildid(e);
+    if (!debug_file)
+      debug_file = find_debug_via_debuglink(e, path);
+    if (debug_file) {
+      foreach_sym_core(debug_file, callback, payload, 1);
+      free(debug_file);
+    }
+  }
+
   res = listsymbols(e, callback, payload);
   elf_end(e);
   close(fd);
   return res;
 }
 
+int bcc_elf_foreach_sym(const char *path, bcc_elf_symcb callback,
+                        void *payload) {
+  return foreach_sym_core(path, callback, payload, 0);
+}
+
 static int loadaddr(Elf *e, uint64_t *addr) {
   size_t phnum, i;
 
@@ -268,11 +501,11 @@ int bcc_elf_is_shared_obj(const char *path) {
 
 int main(int argc, char *argv[])
 {
-	uint64_t addr;
-	if (bcc_elf_findsym(argv[1], argv[2], -1, STT_FUNC, &addr) < 0)
-		return -1;
+  uint64_t addr;
+  if (bcc_elf_findsym(argv[1], argv[2], -1, STT_FUNC, &addr) < 0)
+    return -1;
 
-	printf("%s: %p\n", argv[2], (void *)addr);
-	return 0;
+  printf("%s: %p\n", argv[2], (void *)addr);
+  return 0;
 }
 #endif

src/cc/bcc_syms.cc

@@ -62,7 +62,7 @@ bool KSyms::resolve_addr(uint64_t addr, struct bcc_symbol *sym) {
   auto it = std::upper_bound(syms_.begin(), syms_.end(), Symbol("", addr)) - 1;
   sym->name = (*it).name.c_str();
   sym->demangle_name = sym->name;
-  sym->module = "[kernel]";
+  sym->module = "kernel";
   sym->offset = addr - (*it).addr;
   return true;
 }
@@ -140,6 +140,11 @@ bool ProcSyms::resolve_name(const char *module, const char *name,
   return false;
 }
 
+ProcSyms::Module::Module(const char *name, uint64_t start, uint64_t end)
+  : name_(name), start_(start), end_(end) {
+  is_so_ = bcc_elf_is_shared_obj(name) == 1;
+}
+
 int ProcSyms::Module::_add_symbol(const char *symname, uint64_t start,
                                   uint64_t end, int flags, void *p) {
   Module *m = static_cast<Module *>(p);
@@ -148,10 +153,6 @@ int ProcSyms::Module::_add_symbol(const char *symname, uint64_t start,
   return 0;
 }
 
-bool ProcSyms::Module::is_so() const {
-  return strstr(name_.c_str(), ".so") != nullptr;
-}
-
 bool ProcSyms::Module::is_perf_map() const {
   return strstr(name_.c_str(), ".map") != nullptr;
 }
@@ -225,10 +226,10 @@ int bcc_symcache_resolve(void *resolver, uint64_t addr,
   return cache->resolve_addr(addr, sym) ? 0 : -1;
 }
 
-int bcc_symcache_resolve_name(void *resolver, const char *name,
-                              uint64_t *addr) {
+int bcc_symcache_resolve_name(void *resolver, const char *module,
+                              const char *name, uint64_t *addr) {
   SymbolCache *cache = static_cast<SymbolCache *>(resolver);
-  return cache->resolve_name(nullptr, name, addr) ? 0 : -1;
+  return cache->resolve_name(module, name, addr) ? 0 : -1;
 }
 
 void bcc_symcache_refresh(void *resolver) {

src/cc/bcc_syms.h

@@ -35,7 +35,8 @@ void *bcc_symcache_new(int pid);
 void bcc_free_symcache(void *symcache, int pid);
 
 int bcc_symcache_resolve(void *symcache, uint64_t addr, struct bcc_symbol *sym);
-int bcc_symcache_resolve_name(void *resolver, const char *name, uint64_t *addr);
+int bcc_symcache_resolve_name(void *resolver, const char *module,
+                              const char *name, uint64_t *addr);
 void bcc_symcache_refresh(void *resolver);
 
 int bcc_resolve_global_addr(int pid, const char *module, const uint64_t address,

src/cc/syms.h

@@ -79,18 +79,18 @@ class ProcSyms : SymbolCache {
   };
 
   struct Module {
-    Module(const char *name, uint64_t start, uint64_t end)
-        : name_(name), start_(start), end_(end) {}
+    Module(const char *name, uint64_t start, uint64_t end);
     std::string name_;
     uint64_t start_;
     uint64_t end_;
+    bool is_so_;
     std::unordered_set<std::string> symnames_;
     std::vector<Symbol> syms_;
 
     void load_sym_table();
     bool find_addr(uint64_t addr, struct bcc_symbol *sym);
     bool find_name(const char *symname, uint64_t *addr);
-    bool is_so() const;
+    bool is_so() const { return is_so_; }
     bool is_perf_map() const;
 
     static int _add_symbol(const char *symname, uint64_t start, uint64_t end,

src/python/bcc/__init__.py

@@ -27,7 +27,6 @@ basestring = (unicode if sys.version_info[0] < 3 else str)
 from .libbcc import lib, _CB_TYPE, bcc_symbol, _SYM_CB_TYPE
 from .table import Table
 from .perf import Perf
-from .usyms import ProcessSymbols
 from .utils import get_online_cpus
 
 _kprobe_limit = 1000
@@ -50,15 +49,29 @@ class SymbolCache(object):
         self.cache = lib.bcc_symcache_new(pid)
 
     def resolve(self, addr):
+        """
+        Return a tuple of the symbol (function), its offset from the beginning
+        of the function, and the module in which it lies. For example:
+            ("start_thread", 0x202, "/usr/lib/.../libpthread-2.24.so")
+        If the symbol cannot be found but we know which module it is in,
+        return the module name and the offset from the beginning of the
+        module. If we don't even know the module, return the absolute
+        address as the offset.
+        """
         sym = bcc_symbol()
         psym = ct.pointer(sym)
         if lib.bcc_symcache_resolve(self.cache, addr, psym) < 0:
-            return "[unknown]", 0
-        return sym.demangle_name.decode(), sym.offset
-
-    def resolve_name(self, name):
+            if sym.module and sym.offset:
+                return (None, sym.offset,
+                        ct.cast(sym.module, ct.c_char_p).value.decode())
+            return (None, addr, None)
+        return (sym.demangle_name.decode(), sym.offset,
+                ct.cast(sym.module, ct.c_char_p).value.decode())
+
+    def resolve_name(self, module, name):
         addr = ct.c_ulonglong()
-        if lib.bcc_symcache_resolve_name(self.cache, name, ct.pointer(addr)) < 0:
+        if lib.bcc_symcache_resolve_name(
+                    self.cache, module, name, ct.pointer(addr)) < 0:
             return -1
         return addr.value
 
@@ -968,53 +981,50 @@ class BPF(object):
         return BPF._sym_caches[pid]
 
     @staticmethod
-    def sym(addr, pid):
-        """sym(addr, pid)
+    def sym(addr, pid, show_module=False, show_offset=False):
+        """sym(addr, pid, show_module=False, show_offset=False)
 
         Translate a memory address into a function name for a pid, which is
-        returned.
+        returned. When show_module is True, the module name is also included.
+        When show_offset is True, the instruction offset as a hexadecimal
+        number is also included in the string.
+        
         A pid of less than zero will access the kernel symbol cache.
-        """
-        name, _ = BPF._sym_cache(pid).resolve(addr)
-        return name
-
-    @staticmethod
-    def symaddr(addr, pid):
-        """symaddr(addr, pid)
 
-        Translate a memory address into a function name plus the instruction
-        offset as a hexadecimal number, which is returned as a string.
-        A pid of less than zero will access the kernel symbol cache.
+        Example output when both show_module and show_offset are True:
+            "start_thread+0x202 [libpthread-2.24.so]"
+        
+        Example output when both show_module and show_offset are False:
+            "start_thread"
         """
-        name, offset = BPF._sym_cache(pid).resolve(addr)
-        return "%s+0x%x" % (name, offset)
+        name, offset, module = BPF._sym_cache(pid).resolve(addr)
+        offset = "+0x%x" % offset if show_offset and name is not None else ""
+        name = name or "[unknown]"
+        name = name + offset
+        module = " [%s]" % os.path.basename(module) if show_module else ""
+        return name + module
 
     @staticmethod
-    def ksym(addr):
+    def ksym(addr, show_module=False, show_offset=False):
         """ksym(addr)
 
         Translate a kernel memory address into a kernel function name, which is
-        returned.
-        """
-        return BPF.sym(addr, -1)
-
-    @staticmethod
-    def ksymaddr(addr):
-        """ksymaddr(addr)
+        returned. When show_module is True, the module name ("kernel") is also
+        included. When show_offset is true, the instruction offset as a 
+        hexadecimal number is also included in the string.
 
-        Translate a kernel memory address into a kernel function name plus the
-        instruction offset as a hexidecimal number, which is returned as a
-        string.
+        Example output when both show_module and show_offset are True:
+            "default_idle+0x0 [kernel]"
         """
-        return BPF.symaddr(addr, -1)
+        return BPF.sym(addr, -1, show_module, show_offset)
 
     @staticmethod
     def ksymname(name):
         """ksymname(name)
 
         Translate a kernel name into an address. This is the reverse of
-        ksymaddr. Returns -1 when the function name is unknown."""
-        return BPF._sym_cache(-1).resolve_name(name)
+        ksym. Returns -1 when the function name is unknown."""
+        return BPF._sym_cache(-1).resolve_name(None, name)
 
     def num_open_kprobes(self):
         """num_open_kprobes()

src/python/bcc/libbcc.py

@@ -154,7 +154,7 @@ lib.bcc_symcache_resolve.argtypes = [ct.c_void_p, ct.c_ulonglong, ct.POINTER(bcc
 
 lib.bcc_symcache_resolve_name.restype = ct.c_int
 lib.bcc_symcache_resolve_name.argtypes = [
-    ct.c_void_p, ct.c_char_p, ct.POINTER(ct.c_ulonglong)]
+    ct.c_void_p, ct.c_char_p, ct.c_char_p, ct.POINTER(ct.c_ulonglong)]
 
 lib.bcc_symcache_refresh.restype = None
 lib.bcc_symcache_refresh.argtypes = [ct.c_void_p]

src/python/bcc/usyms.py

-# Copyright 2016 Sasha Goldshtein
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import ctypes as ct
-from .libbcc import lib, bcc_symbol
-
-class ProcessSymbols(object):
-    def __init__(self, pid):
-        """
-        Initializes the process symbols store for the specified pid.
-        Call refresh_code_ranges() periodically if you anticipate changes
-        in the set of loaded libraries or their addresses.
-        """
-        self.cache = lib.bcc_symcache_new(pid)
-
-    def refresh_code_ranges(self):
-        lib.bcc_symcache_refresh(self.cache)
-
-    def decode_addr(self, addr):
-        """
-        Given an address, return the best symbolic representation of it.
-        If it doesn't fall in any module, return its hex string. If it
-        falls within a module but we don't have a symbol for it, return
-        the hex string and the module. If we do have a symbol for it,
-        return the symbol and the module, e.g. "readline+0x10 [bash]".
-        """
-        sym = bcc_symbol()
-        psym = ct.pointer(sym)
-        if lib.bcc_symcache_resolve(self.cache, addr, psym) < 0:
-            if sym.module and sym.offset:
-                return "0x%x [%s]" % (sym.offset, sym.module)
-            return "%x" % addr
-        return "%s+0x%x [%s]" % (sym.name, sym.offset, sym.module)

tests/python/CMakeLists.txt

@@ -36,6 +36,8 @@ add_test(NAME py_test_trace4 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   COMMAND ${TEST_WRAPPER} py_trace4 sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_trace4.py)
 add_test(NAME py_test_probe_count WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   COMMAND ${TEST_WRAPPER} py_probe_count sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_probe_count.py)
+add_test(NAME py_test_debuginfo WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND ${TEST_WRAPPER} py_test_debuginfo sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_debuginfo.py)
 add_test(NAME py_test_brb WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   COMMAND ${TEST_WRAPPER} py_brb_c sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_brb.py test_brb.c)
 add_test(NAME py_test_brb2 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -60,6 +62,5 @@ add_test(NAME py_test_utils WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   COMMAND ${TEST_WRAPPER} py_test_utils sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_utils.py)
 add_test(NAME py_test_percpu WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   COMMAND ${TEST_WRAPPER} py_test_percpu sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_percpu.py)
-
 add_test(NAME py_test_dump_func WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
   COMMAND ${TEST_WRAPPER} py_dump_func simple ${CMAKE_CURRENT_SOURCE_DIR}/test_dump_func.py)

tests/python/dummy.c

+#include <unistd.h>
+#include <stdio.h>
+
+static __attribute__((noinline)) int some_function(int x, int y) {
+	volatile int z = x + y;
+	return z;
+}
+
+int main() {
+	printf("%p\n", &some_function);
+	fflush(stdout);
+	printf("result = %d\n", some_function(42, 11));
+	sleep(1000);
+	return 0;
+}

tests/python/test_debuginfo.py

+#!/usr/bin/env python
+# Copyright (c) Sasha Goldshtein
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+import os
+import subprocess
+from bcc import SymbolCache
+from unittest import main, TestCase
+
+class Harness(TestCase):
+    def setUp(self):
+        self.build_command()
+        subprocess.check_output('objcopy --only-keep-debug dummy dummy.debug'
+                                .split())
+        self.debug_command()
+        subprocess.check_output('strip dummy'.split())
+        self.process = subprocess.Popen('./dummy', stdout=subprocess.PIPE)
+        # The process prints out the address of some symbol, which we then
+        # try to resolve in the test.
+        self.addr = int(self.process.stdout.readline().strip(), 16)
+        self.syms = SymbolCache(self.process.pid)
+
+    def tearDown(self):
+        self.process.kill()
+
+    def resolve_addr(self):
+        sym, offset, module = self.syms.resolve(self.addr)
+        self.assertEqual(sym, 'some_function')
+        self.assertEqual(offset, 0)
+        self.assertTrue(module[-5:] == 'dummy')
+
+    def resolve_name(self):
+        script_dir = os.path.dirname(os.path.realpath(__file__))
+        addr = self.syms.resolve_name(os.path.join(script_dir, 'dummy'),
+                                      'some_function')
+        self.assertEqual(addr, self.addr)
+        pass
+
+class TestDebuglink(Harness):
+    def build_command(self):
+        subprocess.check_output('gcc -o dummy dummy.c'.split())
+
+    def debug_command(self):
+        subprocess.check_output('objcopy --add-gnu-debuglink=dummy.debug dummy'
+                                .split())
+
+    def tearDown(self):
+        subprocess.check_output('rm dummy dummy.debug'.split())
+
+    def test_resolve_addr(self):
+        self.resolve_addr()
+
+    def test_resolve_name(self):
+        self.resolve_name()
+
+class TestBuildid(Harness):
+    def build_command(self):
+        subprocess.check_output(('gcc -o dummy -Xlinker ' + \
+               '--build-id=0x123456789abcdef0123456789abcdef012345678 dummy.c')
+               .split())
+
+    def debug_command(self):
+        subprocess.check_output('mkdir -p /usr/lib/debug/.build-id/12'.split())
+        subprocess.check_output(('mv dummy.debug /usr/lib/debug/.build-id' + \
+            '/12/3456789abcdef0123456789abcdef012345678.debug').split())
+
+    def tearDown(self):
+        subprocess.check_output('rm dummy'.split())
+        subprocess.check_output(('rm /usr/lib/debug/.build-id/12' +
+            '/3456789abcdef0123456789abcdef012345678.debug').split())
+
+    def test_resolve_name(self):
+        self.resolve_addr()
+
+    def test_resolve_addr(self):
+        self.resolve_name()
+
+if __name__ == "__main__":
+    main()

tools/memleak.py

@@ -11,31 +11,13 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 # Copyright (C) 2016 Sasha Goldshtein.
 
-from bcc import BPF, ProcessSymbols
+from bcc import BPF
 from time import sleep
 from datetime import datetime
 import argparse
 import subprocess
 import os
 
-class KStackDecoder(object):
-        def refresh(self):
-                pass
-
-        def __call__(self, addr):
-                return "%s [kernel] (%x)" % (BPF.ksym(addr), addr)
-
-class UStackDecoder(object):
-        def __init__(self, pid):
-                self.pid = pid
-                self.proc_sym = ProcessSymbols(pid)
-
-        def refresh(self):
-                self.proc_sym.refresh_code_ranges()
-
-        def __call__(self, addr):
-                return "%s (%x)" % (self.proc_sym.decode_addr(addr), addr)
-
 class Allocation(object):
     def __init__(self, stack, size):
         self.stack = stack
@@ -240,8 +222,6 @@ else:
         bpf_program.attach_kretprobe(event="__kmalloc", fn_name="alloc_exit")
         bpf_program.attach_kprobe(event="kfree", fn_name="free_enter")
 
-decoder = KStackDecoder() if kernel_trace else UStackDecoder(pid)
-
 def print_outstanding():
         print("[%s] Top %d stacks with outstanding allocations:" %
               (datetime.now().strftime("%H:%M:%S"), top_stacks))
@@ -256,8 +236,12 @@ def print_outstanding():
                 if info.stack_id in alloc_info:
                         alloc_info[info.stack_id].update(info.size)
                 else:
-                        stack = list(stack_traces.walk(info.stack_id, decoder))
-                        alloc_info[info.stack_id] = Allocation(stack,
+                        stack = list(stack_traces.walk(info.stack_id))
+                        combined = []
+                        for addr in stack:
+                                combined.append(bpf_program.sym(addr, pid,
+                                        show_module=True, show_offset=True))
+                        alloc_info[info.stack_id] = Allocation(combined,
                                                                info.size)
                 if args.show_allocs:
                         print("\taddr = %x size = %s" %
@@ -277,7 +261,6 @@ while True:
                         sleep(interval)
                 except KeyboardInterrupt:
                         exit()
-                decoder.refresh()
                 print_outstanding()
                 count_so_far += 1
                 if num_prints is not None and count_so_far >= num_prints:

tools/memleak_example.txt

@@ -10,13 +10,13 @@ For example:
 Attaching to malloc and free in pid 5193, Ctrl+C to quit.
 [11:16:33] Top 2 stacks with outstanding allocations:
         80 bytes in 5 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fd460ac2790) 
+                 main+0x6d [allocs]
+                 __libc_start_main+0xf0 [libc-2.21.so]
 
 [11:16:34] Top 2 stacks with outstanding allocations:
         160 bytes in 10 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fd460ac2790) 
+                 main+0x6d [allocs]
+                 __libc_start_main+0xf0 [libc-2.21.so]
 
 
 Each entry printed is a set of allocations that originate from the same call
@@ -40,8 +40,8 @@ Attaching to malloc and free in pid 5193, Ctrl+C to quit.
         addr = 948d30 size = 16
         addr = 948cf0 size = 16
         64 bytes in 4 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fd460ac2790) 
+                 main+0x6d [allocs]
+                 __libc_start_main+0xf0 [libc-2.21.so]
 
 [11:16:34] Top 2 stacks with outstanding allocations:
         addr = 948d50 size = 16
@@ -55,8 +55,8 @@ Attaching to malloc and free in pid 5193, Ctrl+C to quit.
         addr = 948d70 size = 16
         addr = 948df0 size = 16
         160 bytes in 10 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fd460ac2790) 
+                 main+0x6d [allocs]
+                 __libc_start_main+0xf0 [libc-2.21.so]
 
 
 When using the -p switch, memleak traces the allocations of a particular
@@ -67,35 +67,35 @@ For example:
 Attaching to kmalloc and kfree, Ctrl+C to quit.
 ...
         248 bytes in 4 allocations from stack
-                 bpf_prog_load [kernel] (ffffffff8118c471) 
-                 sys_bpf [kernel] (ffffffff8118c8b5) 
+                 bpf_prog_load [kernel]
+                 sys_bpf [kernel]
 
         328 bytes in 1 allocations from stack
-                 perf_mmap [kernel] (ffffffff811990fd) 
-                 mmap_region [kernel] (ffffffff811df5d4) 
-                 do_mmap [kernel] (ffffffff811dfb83) 
-                 vm_mmap_pgoff [kernel] (ffffffff811c494f) 
-                 sys_mmap_pgoff [kernel] (ffffffff811ddf02) 
-                 sys_mmap [kernel] (ffffffff8101b0ab) 
+                 perf_mmap [kernel]
+                 mmap_region [kernel]
+                 do_mmap [kernel]
+                 vm_mmap_pgoff [kernel]
+                 sys_mmap_pgoff [kernel]
+                 sys_mmap [kernel]
 
         464 bytes in 1 allocations from stack
-                 traceprobe_command [kernel] (ffffffff81187cf2) 
-                 traceprobe_probes_write [kernel] (ffffffff81187d86) 
-                 probes_write [kernel] (ffffffff81181580) 
-                 __vfs_write [kernel] (ffffffff812237b7) 
-                 vfs_write [kernel] (ffffffff81223ec6) 
-                 sys_write [kernel] (ffffffff81224b85) 
-                 entry_SYSCALL_64_fastpath [kernel] (ffffffff8178182e) 
+                 traceprobe_command [kernel]
+                 traceprobe_probes_write [kernel]
+                 probes_write [kernel]
+                 __vfs_write [kernel]
+                 vfs_write [kernel]
+                 sys_write [kernel]
+                 entry_SYSCALL_64_fastpath [kernel]
 
         8192 bytes in 1 allocations from stack
-                 alloc_and_copy_ftrace_hash.constprop.59 [kernel] (ffffffff8115d17e) 
-                 ftrace_set_hash [kernel] (ffffffff8115e767) 
-                 ftrace_set_filter_ip [kernel] (ffffffff8115e9a8) 
-                 arm_kprobe [kernel] (ffffffff81148600) 
-                 enable_kprobe [kernel] (ffffffff811486f6) 
-                 kprobe_register [kernel] (ffffffff81182399) 
-                 perf_trace_init [kernel] (ffffffff8117c4e0) 
-                 perf_tp_event_init [kernel] (ffffffff81192479) 
+                 alloc_and_copy_ftrace_hash.constprop.59 [kernel]
+                 ftrace_set_hash [kernel]
+                 ftrace_set_filter_ip [kernel]
+                 arm_kprobe [kernel]
+                 enable_kprobe [kernel]
+                 kprobe_register [kernel]
+                 perf_trace_init [kernel]
+                 perf_tp_event_init [kernel]
 
 
 Here you can see that arming the kprobe to which our eBPF program is attached
@@ -129,18 +129,18 @@ seconds, 3 times before quitting:
 Attaching to malloc and free in pid 2614, Ctrl+C to quit.
 [11:16:33] Top 2 stacks with outstanding allocations:
         16 bytes in 1 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fdc11ce8790) 
+                 main+0x6d [allocs]
+                 __libc_start_main+0xf0 [libc-2.21.so]
 
 [11:16:38] Top 2 stacks with outstanding allocations:
         16 bytes in 1 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fdc11ce8790) 
+                 main+0x6d [allocs]
+                 __libc_start_main+0xf0 [libc-2.21.so]
 
 [11:16:43] Top 2 stacks with outstanding allocations:
         32 bytes in 2 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fdc11ce8790) 
+                 main+0x6d [allocs]
+                 __libc_start_main+0xf0 [libc-2.21.so]
 
 Note that even though the application leaks 16 bytes of memory every second, 
 the report (printed every 5 seconds) doesn't "see" all the allocations because

tools/offcputime.py

@@ -271,11 +271,11 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
     else:
         # print default multi-line stack output
         for addr in kernel_stack:
-            print("    %016x %s" % (addr, b.ksym(addr)))
+            print("    %s" % b.ksym(addr))
         if need_delimiter:
             print("    --")
         for addr in user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.tgid)))
+            print("    %s" % b.sym(addr, k.tgid))
         print("    %-16s %s (%d)" % ("-", k.name, k.pid))
         print("        %d\n" % v.value)
 

tools/offwaketime.py

@@ -308,22 +308,22 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
         # print wakeup name then stack in reverse order
         print("    %-16s %s %s" % ("waker:", k.waker, k.t_pid))
         for addr in waker_user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.tgid)))
+            print("    %s" % b.sym(addr, k.tgid))
         if args.delimited:
             print("    -")
         for addr in waker_kernel_stack:
-            print("    %016x %s" % (addr, b.ksym(addr)))
+            print("    %s" % b.ksym(addr))
 
         # print waker/wakee delimiter
         print("    %-16s %s" % ("--", "--"))
 
         # print default multi-line stack output
         for addr in target_kernel_stack:
-            print("    %016x %s" % (addr, b.ksym(addr)))
+            print("    %s" % b.ksym(addr))
         if args.delimited:
             print("    -")
         for addr in target_user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.tgid)))
+            print("    %s" % b.sym(addr, k.tgid))
         print("    %-16s %s %s" % ("target:", k.target, k.w_pid))
         print("        %d\n" % v.value)
 

tools/old/memleak.py

@@ -11,36 +11,21 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 # Copyright (C) 2016 Sasha Goldshtein.
 
-from bcc import BPF, ProcessSymbols
+from bcc import BPF
 from time import sleep
 from datetime import datetime
 import argparse
 import subprocess
 import os
 
-class StackDecoder(object):
-        def __init__(self, pid):
-                self.pid = pid
-                if pid != -1:
-                        self.proc_sym = ProcessSymbols(pid)
-
-        def refresh(self):
-                if self.pid != -1:
-                        self.proc_sym.refresh_code_ranges()
-
-        def decode_stack(self, info, is_kernel_trace):
-                stack = ""
-                if info.num_frames <= 0:
-                        return "???"
-                for i in range(0, info.num_frames):
-                        addr = info.callstack[i]
-                        if is_kernel_trace:
-                                stack += " %s [kernel] (%x) ;" % \
-                                        (BPF.ksym(addr), addr)
-                        else:
-                                stack += " %s (%x) ;" % \
-                                        (self.proc_sym.decode_addr(addr), addr)
-                return stack
+def decode_stack(bpf, pid, info):
+        stack = ""
+        if info.num_frames <= 0:
+                return "???"
+        for i in range(0, info.num_frames):
+                addr = info.callstack[i]
+                stack += " %s ;" % bpf.sym(addr, pid, show_offset=True)
+        return stack
 
 def run_command_get_output(command):
         p = subprocess.Popen(command.split(),
@@ -255,8 +240,6 @@ else:
         bpf_program.attach_kretprobe(event="__kmalloc", fn_name="alloc_exit")
         bpf_program.attach_kprobe(event="kfree", fn_name="free_enter")
 
-decoder = StackDecoder(pid)
-
 def print_outstanding():
         stacks = {}
         print("[%s] Top %d stacks with outstanding allocations:" %
@@ -265,7 +248,7 @@ def print_outstanding():
         for address, info in sorted(allocs.items(), key=lambda a: a[1].size):
                 if BPF.monotonic_time() - min_age_ns < info.timestamp_ns:
                         continue
-                stack = decoder.decode_stack(info, kernel_trace)
+                stack = decode_stack(bpf_program, pid, info)
                 if stack in stacks:
                         stacks[stack] = (stacks[stack][0] + 1,
                                          stacks[stack][1] + info.size)
@@ -288,7 +271,6 @@ while True:
                         sleep(interval)
                 except KeyboardInterrupt:
                         exit()
-                decoder.refresh()
                 print_outstanding()
                 count_so_far += 1
                 if num_prints is not None and count_so_far >= num_prints:

tools/old/profile.py

@@ -347,11 +347,11 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
     else:
         # print default multi-line stack output.
         for addr in kernel_stack:
-            print("    %016x %s" % (addr, aksym(addr)))
+            print("    %s" % aksym(addr))
         if do_delimiter:
             print("    --")
         for addr in user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.pid)))
+            print("    %s" % b.sym(addr, k.pid))
         print("    %-16s %s (%d)" % ("-", k.name, k.pid))
         print("        %d\n" % v.value)
 

tools/old/stackcount.py

@@ -145,10 +145,7 @@ def print_frame(addr):
     print("  ", end="")
     if verbose:
         print("%-16x " % addr, end="")
-    if offset:
-        print("%s" % b.ksymaddr(addr))
-    else:
-        print("%s" % b.ksym(addr))
+    print(b.ksym(addr, show_offset=offset))
 
 # output
 exiting = 0 if args.interval else 1

tools/old/stacksnoop.py

@@ -119,10 +119,7 @@ while 1:
     (task, pid, cpu, flags, ts, msg) = b.trace_fields()
     if msg != "":
         (reg, addr) = msg.split(" ")
-        if offset:
-            ip = b.ksymaddr(int(addr, 16))
-        else:
-            ip = b.ksym(int(addr, 16))
+        ip = b.ksym(int(addr, 16), show_offset=offset)
         msg = msg + " " + ip
     if verbose:
         print("%-18.9f %-12.12s %-6d %-3d %s" % (ts, task, pid, cpu, msg))

tools/profile.py

@@ -287,11 +287,11 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
     else:
         # print default multi-line stack output.
         for addr in kernel_stack:
-            print("    %016x %s" % (addr, aksym(addr)))
+            print("    %s" % aksym(addr))
         if do_delimiter:
             print("    --")
         for addr in user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.pid)))
+            print("    %s" % b.sym(addr, k.pid))
         print("    %-16s %s (%d)" % ("-", k.name, k.pid))
         print("        %d\n" % v.value)
 

tools/profile_example.txt

@@ -9,67 +9,66 @@ Example output:
 # ./profile
 Sampling at 49 Hertz of all threads by user + kernel stack... Hit Ctrl-C to end.
 ^C
-    ffffffff81189249 filemap_map_pages
-    ffffffff811bd3f5 handle_mm_fault
-    ffffffff81065990 __do_page_fault
-    ffffffff81065caf do_page_fault
-    ffffffff817ce228 page_fault
-    00007fed989afcc0 [unknown]
+    filemap_map_pages
+    handle_mm_fault
+    __do_page_fault
+    do_page_fault
+    page_fault
+    [unknown]
     -                cp (9036)
         1
 
-    00007f31d76c3251 [unknown]
-    47a2c1e752bf47f7 [unknown]
+    [unknown]
+    [unknown]
     -                sign-file (8877)
         1
 
-    ffffffff813d0af8 __clear_user
-    ffffffff813d5277 iov_iter_zero
-    ffffffff814ec5f2 read_iter_zero
-    ffffffff8120be9d __vfs_read
-    ffffffff8120c385 vfs_read
-    ffffffff8120d786 sys_read
-    ffffffff817cc076 entry_SYSCALL_64_fastpath
-    00007fc5652ad9b0 read
+    __clear_user
+    iov_iter_zero
+    read_iter_zero
+    __vfs_read
+    vfs_read
+    sys_read
+    entry_SYSCALL_64_fastpath
+    read
     -                dd (25036)
         4
 
-    0000000000400542 func_a
-    0000000000400598 main
-    00007f12a133e830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_a
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (13549)
         5
 
 [...]
 
-    ffffffff8105eb66 native_safe_halt
-    ffffffff8103659e default_idle
-    ffffffff81036d1f arch_cpu_idle
-    ffffffff810bba5a default_idle_call
-    ffffffff810bbd07 cpu_startup_entry
-    ffffffff817bf4a7 rest_init
-    ffffffff81d65f58 start_kernel
-    ffffffff81d652db x86_64_start_reservations
-    ffffffff81d65418 x86_64_start_kernel
+    native_safe_halt
+    default_idle
+    arch_cpu_idle
+    default_idle_call
+    cpu_startup_entry
+    rest_init
+    start_kernel
+    x86_64_start_reservations
+    x86_64_start_kernel
     -                swapper/0 (0)
         72
 
-    ffffffff8105eb66 native_safe_halt
-    ffffffff8103659e default_idle
-    ffffffff81036d1f arch_cpu_idle
-    ffffffff810bba5a default_idle_call
-    ffffffff810bbd07 cpu_startup_entry
-    ffffffff8104df55 start_secondary
+    native_safe_halt
+    default_idle
+    arch_cpu_idle
+    default_idle_call
+    cpu_startup_entry
+    start_secondary
     -                swapper/1 (0)
         75
 
 The output was long; I truncated some lines ("[...]").
 
-This default output prints stack traces as two columns (raw addresses, and
-then translated symbol names), followed by a line to describe the process (a
-dash, the process name, and a PID in parenthesis), and then an integer count
-of how many times this stack trace was sampled.
+This default output prints stack traces, followed by a line to describe the
+process (a dash, the process name, and a PID in parenthesis), and then an
+integer count of how many times this stack trace was sampled.
 
 The output above shows the most frequent stack was from the "swapper/1"
 process (PID 0), running the native_safe_halt() function, which was called
@@ -95,57 +94,57 @@ Profiling just that process:
 # ./profile -p 25036
 Sampling at 49 Hertz of PID 25036 by user + kernel stack... Hit Ctrl-C to end.
 ^C
-    0000000000402748 [unknown]
-    00007fc56561422c [unknown]
+    [unknown]
+    [unknown]
     -                dd (25036)
         1
 
-    00007fc5652ada0e __write
+    __write
     -                dd (25036)
         1
 
-    00007fc5652ad9b0 read
+    read
     -                dd (25036)
         1
 
 [...]
 
-    00000000004047b2 [unknown]
-    00007fc56561422c [unknown]
+    [unknown]
+    [unknown]
     -                dd (25036)
         2
 
-    ffffffff817cc060 entry_SYSCALL_64_fastpath
-    00007fc5652ada10 __write
-    00007fc56561422c [unknown]
+    entry_SYSCALL_64_fastpath
+    __write
+    [unknown]
     -                dd (25036)
         3
 
-    ffffffff817cc060 entry_SYSCALL_64_fastpath
-    00007fc5652ad9b0 read
+    entry_SYSCALL_64_fastpath
+    read
     -                dd (25036)
         3
 
-    ffffffff813d0af8 __clear_user
-    ffffffff813d5277 iov_iter_zero
-    ffffffff814ec5f2 read_iter_zero
-    ffffffff8120be9d __vfs_read
-    ffffffff8120c385 vfs_read
-    ffffffff8120d786 sys_read
-    ffffffff817cc076 entry_SYSCALL_64_fastpath
-    00007fc5652ad9b0 read
-    00007fc56561422c [unknown]
+    __clear_user
+    iov_iter_zero
+    read_iter_zero
+    __vfs_read
+    vfs_read
+    sys_read
+    entry_SYSCALL_64_fastpath
+    read
+    [unknown]
     -                dd (25036)
         3
 
-    ffffffff813d0af8 __clear_user
-    ffffffff813d5277 iov_iter_zero
-    ffffffff814ec5f2 read_iter_zero
-    ffffffff8120be9d __vfs_read
-    ffffffff8120c385 vfs_read
-    ffffffff8120d786 sys_read
-    ffffffff817cc076 entry_SYSCALL_64_fastpath
-    00007fc5652ad9b0 read
+    __clear_user
+    iov_iter_zero
+    read_iter_zero
+    __vfs_read
+    vfs_read
+    sys_read
+    entry_SYSCALL_64_fastpath
+    read
     -                dd (25036)
         7
 
@@ -162,41 +161,41 @@ Lets add delimiters between the user and kernel stacks, using -d:
 
 # ./profile -p 25036 -d
 ^C
-    ffffffff8120b385 __vfs_write
-    ffffffff8120d826 sys_write
-    ffffffff817cc076 entry_SYSCALL_64_fastpath
+    __vfs_write
+    sys_write
+    entry_SYSCALL_64_fastpath
     --
-    00007fc5652ada10 __write
+    __write
     -                dd (25036)
         1
 
     --
-    00007fc565255ef3 [unknown]
-    00007fc56561422c [unknown]
+    [unknown]
+    [unknown]
     -                dd (25036)
         1
 
-    ffffffff813d4569 iov_iter_init
-    ffffffff8120be8e __vfs_read
-    ffffffff8120c385 vfs_read
-    ffffffff8120d786 sys_read
-    ffffffff817cc076 entry_SYSCALL_64_fastpath
+    iov_iter_init
+    __vfs_read
+    vfs_read
+    sys_read
+    entry_SYSCALL_64_fastpath
     --
-    00007fc5652ad9b0 read
+    read
     -                dd (25036)
         1
 
 [...]
 
-    ffffffff813d0af8 __clear_user
-    ffffffff813d5277 iov_iter_zero
-    ffffffff814ec5f2 read_iter_zero
-    ffffffff8120be9d __vfs_read
-    ffffffff8120c385 vfs_read
-    ffffffff8120d786 sys_read
-    ffffffff817cc076 entry_SYSCALL_64_fastpath
+    __clear_user
+    iov_iter_zero
+    read_iter_zero
+    __vfs_read
+    vfs_read
+    sys_read
+    entry_SYSCALL_64_fastpath
     --
-    00007fc5652ad9b0 read
+    read
     -                dd (25036)
         9
 
@@ -210,59 +209,59 @@ func_b(). Profiling it for 5 seconds:
 # ./profile -p `pgrep -n func_ab` 5
 Sampling at 49 Hertz of PID 2930 by user + kernel stack for 5 secs.
 
-    000000000040053e func_a
-    0000000000400598 main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_a
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         2
 
-    0000000000400566 func_b
-    00000000004005ac main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_b
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         3
 
-    000000000040053a func_a
-    0000000000400598 main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_a
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         5
 
-    0000000000400562 func_b
-    00000000004005ac main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_b
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         12
 
-    000000000040056a func_b
-    00000000004005ac main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_b
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         19
 
-    0000000000400542 func_a
-    0000000000400598 main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_a
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         22
 
-    0000000000400571 func_b
-    00000000004005ac main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_b
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         64
 
-    0000000000400549 func_a
-    0000000000400598 main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_a
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         72
 
@@ -540,33 +539,33 @@ You can increase or decrease the sample frequency. Eg, sampling at 9 Hertz:
 # ./profile -F 9
 Sampling at 9 Hertz of all threads by user + kernel stack... Hit Ctrl-C to end.
 ^C
-    000000000040056a func_b
-    00000000004005ac main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_b
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         1
 
 [...]
 
-    ffffffff8105eb66 native_safe_halt
-    ffffffff8103659e default_idle
-    ffffffff81036d1f arch_cpu_idle
-    ffffffff810bba5a default_idle_call
-    ffffffff810bbd07 cpu_startup_entry
-    ffffffff8104df55 start_secondary
+    native_safe_halt
+    default_idle
+    arch_cpu_idle
+    default_idle_call
+    cpu_startup_entry
+    start_secondary
     -                swapper/3 (0)
         8
 
-    ffffffff8105eb66 native_safe_halt
-    ffffffff8103659e default_idle
-    ffffffff81036d1f arch_cpu_idle
-    ffffffff810bba5a default_idle_call
-    ffffffff810bbd07 cpu_startup_entry
-    ffffffff817bf497 rest_init
-    ffffffff81d65f58 start_kernel
-    ffffffff81d652db x86_64_start_reservations
-    ffffffff81d65418 x86_64_start_kernel
+    native_safe_halt
+    default_idle
+    arch_cpu_idle
+    default_idle_call
+    cpu_startup_entry
+    rest_init
+    start_kernel
+    x86_64_start_reservations
+    x86_64_start_kernel
     -                swapper/0 (0)
         8
 
@@ -577,99 +576,99 @@ For example, just user stacks:
 # ./profile -U
 Sampling at 49 Hertz of all threads by user stack... Hit Ctrl-C to end.
 ^C
-    0000000000402ccc [unknown]
-    00007f45a624422c [unknown]
+    [unknown]
+    [unknown]
     -                dd (2931)
         1
 
-    0000000000404b80 [unknown]
-    00007f45a624422c [unknown]
+    [unknown]
+    [unknown]
     -                dd (2931)
         1
 
-    0000000000404d77 [unknown]
-    00007f45a624422c [unknown]
+    [unknown]
+    [unknown]
     -                dd (2931)
         1
 
-    00007f45a5e85e5e [unknown]
-    00007f45a624422c [unknown]
+    [unknown]
+    [unknown]
     -                dd (2931)
         1
 
-    0000000000402d12 [unknown]
-    00007f45a624422c [unknown]
+    [unknown]
+    [unknown]
     -                dd (2931)
         1
 
-    0000000000400562 func_b
-    00000000004005ac main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_b
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         1
 
-    0000000000404805 [unknown]
+    [unknown]
     -                dd (2931)
         1
 
-    00000000004047de [unknown]
+    [unknown]
     -                dd (2931)
         1
 
-    0000000000400542 func_a
-    0000000000400598 main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_a
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         3
 
-    00007f45a5edda10 __write
-    00007f45a624422c [unknown]
+    __write
+    [unknown]
     -                dd (2931)
         3
 
-    000000000040053a func_a
-    0000000000400598 main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_a
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         4
 
-    000000000040056a func_b
-    00000000004005ac main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_b
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         7
 
     -                swapper/6 (0)
         10
 
-    0000000000400571 func_b
-    00000000004005ac main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_b
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         10
 
-    00007f45a5edda10 __write
+    __write
     -                dd (2931)
         10
 
-    0000000000400549 func_a
-    0000000000400598 main
-    00007f0458819830 __libc_start_main
-    083e258d4c544155 [unknown]
+    func_a
+    main
+    __libc_start_main
+    [unknown]
     -                func_ab (2930)
         11
 
-    00007f45a5edd9b0 read
+    read
     -                dd (2931)
         12
 
-    00007f45a5edd9b0 read
-    00007f45a624422c [unknown]
+    read
+    [unknown]
     -                dd (2931)
         14
 

tools/stackcount.py

@@ -225,7 +225,7 @@ class Tool(object):
         if self.args.verbose:
             print("%-16x " % addr, end="")
         if self.args.offset:
-            print("%s" % self.probe.bpf.symaddr(addr, pid))
+            print("%s" % self.probe.bpf.sym(addr, pid, show_offset=True))
         else:
             print("%s" % self.probe.bpf.sym(addr, pid))
 

tools/stacksnoop.py

@@ -120,8 +120,8 @@ def print_event(cpu, data, size):
         print("%-18.9f %s" % (ts, function))
 
     for addr in stack_traces.walk(event.stack_id):
-        sym = b.ksymaddr(addr) if offset else b.ksym(addr)
-        print("\t%016x %s" % (addr, sym))
+        sym = b.ksym(addr, show_offset=offset)
+        print("\t%s" % sym)
 
     print()
 

tools/stacksnoop_example.txt

@@ -8,12 +8,12 @@ to see how they were invoked. For example, tracing the submit_bio() call:
 # ./stacksnoop submit_bio
 TIME(s)            SYSCALL
 3592.838736000     submit_bio
-        ffffffff813bd961 submit_bio
-        ffffffff81257c12 submit_bh
-        ffffffff81301948 jbd2_journal_commit_transaction
-        ffffffff8130653a kjournald2
-        ffffffff810a2df8 kthread
-        ffffffff8183a122 ret_from_fork
+        submit_bio
+        submit_bh
+        jbd2_journal_commit_transaction
+        kjournald2
+        kthread
+        ret_from_fork
 
 This shows that submit_bio() was called by submit_bh(), which was called
 by jbd2_journal_commit_transaction(), and so on. 
@@ -28,12 +28,12 @@ The -v option includes more fields, including the on-CPU process (COMM and PID):
 # ./stacksnoop -v submit_bio
 TIME(s)            COMM         PID    CPU SYSCALL
 3734.855027000     jbd2/dm-0-8  313    0   submit_bio
-        ffffffff813bd961 submit_bio
-        ffffffff81257c12 submit_bh
-        ffffffff81301948 jbd2_journal_commit_transaction
-        ffffffff8130653a kjournald2
-        ffffffff810a2df8 kthread
-        ffffffff8183a122 ret_from_fork
+        submit_bio
+        submit_bh
+        jbd2_journal_commit_transaction
+        kjournald2
+        kthread
+        ret_from_fork
 
 This identifies the application issuing the sync syscall: the jbd2 process
 (COMM column).
@@ -45,30 +45,30 @@ process:
 # ./stacksnoop -v second_overflow
 TIME(s)            COMM         PID    CPU SYSCALL
 3837.526433000     <idle>       0      1   second_overflow
-        ffffffff810fac41 second_overflow
-        ffffffff81102320 tick_do_update_jiffies64
-        ffffffff81102bf0 tick_irq_enter
-        ffffffff810882ac irq_enter
-        ffffffff8183c7df smp_apic_timer_interrupt
-        ffffffff8183aae2 apic_timer_interrupt
-        ffffffff81038f9e default_idle
-        ffffffff8103979f arch_cpu_idle
-        ffffffff810c69da default_idle_call
-        ffffffff810c6cd7 cpu_startup_entry
-        ffffffff81051cbe start_secondary
+        second_overflow
+        tick_do_update_jiffies64
+        tick_irq_enter
+        irq_enter
+        smp_apic_timer_interrupt
+        apic_timer_interrupt
+        default_idle
+        arch_cpu_idle
+        default_idle_call
+        cpu_startup_entry
+        start_secondary
 
 3838.526953000     <idle>       0      1   second_overflow
-        ffffffff810fac41 second_overflow
-        ffffffff81102320 tick_do_update_jiffies64
-        ffffffff81102bf0 tick_irq_enter
-        ffffffff810882ac irq_enter
-        ffffffff8183c7df smp_apic_timer_interrupt
-        ffffffff8183aae2 apic_timer_interrupt
-        ffffffff81038f9e default_idle
-        ffffffff8103979f arch_cpu_idle
-        ffffffff810c69da default_idle_call
-        ffffffff810c6cd7 cpu_startup_entry
-        ffffffff81051cbe start_secondary
+        second_overflow
+        tick_do_update_jiffies64
+        tick_irq_enter
+        irq_enter
+        smp_apic_timer_interrupt
+        apic_timer_interrupt
+        default_idle
+        arch_cpu_idle
+        default_idle_call
+        cpu_startup_entry
+        start_secondary
 
 This fires every second (see TIME(s)), and is from tick_do_update_jiffies64().
 

tools/trace.py

@@ -457,7 +457,8 @@ BPF_PERF_OUTPUT(%s);
 
             stack = list(bpf.get_table(self.stacks_name).walk(stack_id))
             for addr in stack:
-                    print("        %016x %s" % (addr, bpf.sym(addr, tgid)))
+                    print("        %s" % (bpf.sym(addr, tgid,
+                                         show_module=True, show_offset=True)))
 
         def _format_message(self, bpf, tgid, values):
                 # Replace each %K with kernel sym and %U with user sym in tgid
@@ -466,9 +467,10 @@ BPF_PERF_OUTPUT(%s);
                 user_placeholders = [i for i, t in enumerate(self.types)
                                      if t == 'U']
                 for kp in kernel_placeholders:
-                        values[kp] = bpf.ksymaddr(values[kp])
+                        values[kp] = bpf.ksym(values[kp], show_offset=True)
                 for up in user_placeholders:
-                        values[up] = bpf.symaddr(values[up], tgid)
+                        values[up] = bpf.sym(values[up], tgid,
+                                           show_module=True, show_offset=True)
                 return self.python_format % tuple(values)
 
         def print_event(self, bpf, cpu, data, size):

tools/uthreads.py

@@ -104,7 +104,7 @@ def print_event(cpu, data, size):
     event = ct.cast(data, ct.POINTER(ThreadEvent)).contents
     name = event.name
     if event.type == "pthread":
-        name = bpf.sym(event.runtime_id, args.pid)
+        name = bpf.sym(event.runtime_id, args.pid, show_module=True)
         tid = event.native_id
     else:
         tid = "R=%s/N=%s" % (event.runtime_id, event.native_id)

tools/uthreads_example.txt

@@ -25,10 +25,10 @@ Next, trace only pthread creation events in some native application:
 # ./uthreads 27450
 Tracing thread events in process 27450 (language: none)... Ctrl-C to quit.
 TIME     ID               TYPE     DESCRIPTION                   
-0.924    27462            pthread  primes_thread                 
-0.927    27463            pthread  primes_thread                 
-0.928    27464            pthread  primes_thread                 
-0.928    27465            pthread  primes_thread                 
+0.924    27462            pthread  primes_thread [primes]
+0.927    27463            pthread  primes_thread [primes]     
+0.928    27464            pthread  primes_thread [primes]        
+0.928    27465            pthread  primes_thread [primes]        
 ^C
 
 The thread name ("primes_thread" in this example) is resolved from debuginfo.