Merge pull request #967 from goldshtn/debuginfo

External debuginfo support and general symbols overhaul

Merge pull request #967 from goldshtn/debuginfo
External debuginfo support and general symbols overhaul
d60861aa · 4ast · GitHub · c8ba4157 · 3abf7f5f · d60861aa
Commit d60861aa authored Feb 21, 2017 by 4ast Committed by GitHub Feb 21, 2017
28 changed files
--- a/docs/reference_guide.md
+++ b/docs/reference_guide.md
@@ -65,10 +65,9 @@ This guide is incomplete. If something feels missing, check the bcc and kernel s
        - [7. print_linear_hist()](#6-print_linear_hist)
    - [Helpers](#helpers)
        - [1. ksym()](#1-ksym)
-        - [2. ksymaddr()](#2-ksymaddr)
+        - [2. ksymname()](#2-ksymname)
-        - [3. ksymname()](#3-ksymname)
+        - [3. sym()](#3-sym)
-        - [4. sym()](#4-sym)
+        - [4. num_open_kprobes()](#4-num_open_kprobes)
-        - [5. num_open_kprobes()](#5-num_open_kprobes)
 - [BPF Errors](#bpf-errors)
    - [1. Invalid mem access](#1-invalid-mem-access)
@@ -1090,27 +1089,11 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=ksym+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=ksym+path%3Atools+language%3Apython&type=Code)
-### 2. ksymaddr()
+### 2. ksymname()
-Syntax: ```BPF.ksymaddr(addr)```
-Translate a kernel memory address into a kernel function name plus the instruction offset as a hexadecimal number, which is returned as a string.
-Example:
-```Python
-print("kernel function+offset: " + b.ksymaddr(addr))
-```
-Examples in situ:
-[search /examples](https://github.com/iovisor/bcc/search?q=ksymaddr+path%3Aexamples+language%3Apython&type=Code),
-[search /tools](https://github.com/iovisor/bcc/search?q=ksymaddr+path%3Atools+language%3Apython&type=Code)
-### 3. ksymname()
 Syntax: ```BPF.ksymname(name)```
-Translate a kernel name into an address. This is the reverse of ksymaddr. Returns -1 when the function name is unknown.
+Translate a kernel name into an address. This is the reverse of ksym. Returns -1 when the function name is unknown.
 Example:
@@ -1122,11 +1105,11 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=ksymname+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=ksymname+path%3Atools+language%3Apython&type=Code)
-### 4. sym()
+### 3. sym()
-Syntax: ```BPF.sym(addr, pid)```
+Syntax: ```BPF.sym(addr, pid, show_module=False, show_offset=False)```
-Translate a memory address into a function name for a pid, which is returned. A pid of less than zero will access the kernel symbol cache.
+Translate a memory address into a function name for a pid, which is returned. A pid of less than zero will access the kernel symbol cache. The `show_module` and `show_offset` parameters control whether the module in which the symbol lies should be displayed, and whether the instruction offset from the beginning of the symbol should be displayed. These extra parameters default to `False`.
 Example:
@@ -1138,9 +1121,9 @@ Examples in situ:
 [search /examples](https://github.com/iovisor/bcc/search?q=sym+path%3Aexamples+language%3Apython&type=Code),
 [search /tools](https://github.com/iovisor/bcc/search?q=sym+path%3Atools+language%3Apython&type=Code)
-### 5. num_open_kprobes()
+### 4. num_open_kprobes()
-Syntax: ```BPF.num_open_probes()```
+Syntax: ```BPF.num_open_kprobes()```
 Returns the number of open k[ret]probes. Can be useful for scenarios where event_re is used while attaching and detaching probes. Excludes perf_events readers.

--- a/examples/tracing/mallocstacks.py
+++ b/examples/tracing/mallocstacks.py
@@ -11,7 +11,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 from __future__ import print_function
-from bcc import BPF, ProcessSymbols
+from bcc import BPF
 from time import sleep
 import sys
@@ -43,8 +43,6 @@ int alloc_enter(struct pt_regs *ctx, size_t size) {
 b.attach_uprobe(name="c", sym="malloc", fn_name="alloc_enter", pid=pid)
 print("Attaching to malloc in pid %d, Ctrl+C to quit." % pid)
-decoder = ProcessSymbols(pid)
 # sleep until Ctrl-C
 try:
    sleep(99999999)
@@ -57,4 +55,4 @@ stack_traces = b.get_table("stack_traces")
 for k, v in reversed(sorted(calls.items(), key=lambda c: c[1].value)):
    print("%d bytes allocated at:" % v.value)
    for addr in stack_traces.walk(k.value):
-        print("\t%s (%x)" % (decoder.decode_addr(addr), addr))
+        print("\t%s" % b.sym(addr, pid, show_offset=True))
--- a/src/cc/bcc_elf.c
+++ b/src/cc/bcc_elf.c
@@ -15,9 +15,14 @@
 */
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <string.h>
+#include <libgen.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
 #include <gelf.h>
 #include "bcc_elf.h"
@@ -196,20 +201,248 @@ static int listsymbols(Elf *e, bcc_elf_symcb callback, void *payload) {
  return 0;
 }
-int bcc_elf_foreach_sym(const char *path, bcc_elf_symcb callback,
+static Elf_Data * get_section_elf_data(Elf *e, const char *section_name) {
-                        void *payload) {
+  Elf_Scn *section = NULL;
+  GElf_Shdr header;
+  char *name;
+  size_t stridx;
+  if (elf_getshdrstrndx(e, &stridx) != 0)
+    return NULL;
+  while ((section = elf_nextscn(e, section)) != 0) {
+    if (!gelf_getshdr(section, &header))
+      continue;
+    name = elf_strptr(e, stridx, header.sh_name);
+    if (name && !strcmp(name, section_name)) {
+      return elf_getdata(section, NULL);
+    }
+  }
+  return NULL;
+}
+static int find_debuglink(Elf *e, char **debug_file, unsigned int *crc) {
+  Elf_Data *data = NULL;
+  *debug_file = NULL;
+  *crc = 0;
+  data = get_section_elf_data(e, ".gnu_debuglink");
+  if (!data || data->d_size <= 5)
+    return 0;
+  *debug_file = (char *)data->d_buf;
+  *crc = *(unsigned int*)((char *)data->d_buf + data->d_size - 4);
+  return *debug_file ? 1 : 0;
+}
+static int find_buildid(Elf *e, char *buildid) {
+  Elf_Data *data = get_section_elf_data(e, ".note.gnu.build-id");
+  if (data->d_size <= 16 || strcmp((char *)data->d_buf + 12, "GNU"))
+    return 0;
+  char *buf = (char *)data->d_buf + 16;
+  size_t length = data->d_size - 16;
+  for (size_t i = 0; i < length; ++i) {
+    sprintf(buildid + (i * 2), "%02hhx", buf[i]);
+  }
+  return 1;
+}
+// The CRC algorithm used by GNU debuglink. Taken from:
+//    https://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html
+static unsigned int gnu_debuglink_crc32(unsigned int crc,
+                                        char *buf, size_t len) {
+  static const unsigned int crc32_table[256] =
+  {
+    0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419,
+    0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4,
+    0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07,
+    0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
+    0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856,
+    0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
+    0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4,
+    0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+    0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3,
+    0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a,
+    0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599,
+    0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+    0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190,
+    0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f,
+    0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e,
+    0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+    0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed,
+    0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
+    0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3,
+    0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
+    0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a,
+    0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5,
+    0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010,
+    0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+    0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17,
+    0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6,
+    0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615,
+    0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
+    0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344,
+    0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
+    0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a,
+    0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+    0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1,
+    0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c,
+    0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
+    0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+    0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe,
+    0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31,
+    0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c,
+    0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+    0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b,
+    0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
+    0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1,
+    0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
+    0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278,
+    0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7,
+    0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66,
+    0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+    0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605,
+    0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8,
+    0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b,
+    0x2d02ef8d
+  };
+  char *end;
+  crc = ~crc & 0xffffffff;
+  for (end = buf + len; buf < end; ++buf)
+    crc = crc32_table[(crc ^ *buf) & 0xff] ^ (crc >> 8);
+  return ~crc & 0xffffffff;
+}
+static int verify_checksum(const char *file, unsigned int crc) {
+  struct stat st;
+  int fd;
+  void *buf;
+  unsigned int actual;
+  fd = open(file, O_RDONLY);
+  if (fd < 0)
+    return 0;
+  if (fstat(fd, &st) < 0)
+    return 0;
+  buf = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+  if (!buf) {
+    close(fd);
+    return 0;
+  }
+  actual = gnu_debuglink_crc32(0, buf, st.st_size);
+  munmap(buf, st.st_size);
+  close(fd);
+  return actual == crc;
+}
+static char *find_debug_via_debuglink(Elf *e, const char *binpath) {
+  char fullpath[PATH_MAX];
+  char *bindir = NULL;
+  char *res = NULL;
+  unsigned int crc;
+  char *name;  // the name of the debuginfo file
+  if (!find_debuglink(e, &name, &crc))
+    return NULL;
+  bindir = strdup(binpath);
+  bindir = dirname(bindir);
+  // Search for the file in 'binpath'
+  sprintf(fullpath, "%s/%s", bindir, name);
+  if (access(fullpath, F_OK) != -1) {
+    res = strdup(fullpath);
+    goto DONE;
+  }
+  // Search for the file in 'binpath'/.debug
+  sprintf(fullpath, "%s/.debug/%s", bindir, name);
+  if (access(fullpath, F_OK) != -1) {
+    res = strdup(fullpath);
+    goto DONE;
+  }
+  // Search for the file in the global debug directory /usr/lib/debug/'binpath'
+  sprintf(fullpath, "/usr/lib/debug%s/%s", bindir, name);
+  if (access(fullpath, F_OK) != -1) {
+    res = strdup(fullpath);
+    goto DONE;
+  }
+DONE:
+  free(bindir);
+  if (verify_checksum(res, crc))
+    return res;
+  return NULL;
+}
+static char *find_debug_via_buildid(Elf *e) {
+  char fullpath[PATH_MAX];
+  char buildid[128];  // currently 40 seems to be default, let's be safe
+  if (!find_buildid(e, buildid))
+    return NULL;
+  // Search for the file in the global debug directory with a sub-path:
+  //    mm/nnnnnn...nnnn.debug
+  // Where mm are the first two characters of the buildid, and nnnn are the
+  // rest of the build id, followed by .debug.
+  sprintf(fullpath, "/usr/lib/debug/.build-id/%c%c/%s.debug",
+          buildid[0], buildid[1], buildid + 2);
+  if (access(fullpath, F_OK) != -1) {
+    return strdup(fullpath);
+  }
+  return NULL;
+}
+static int foreach_sym_core(const char *path, bcc_elf_symcb callback,
+                            void *payload, int is_debug_file) {
  Elf *e;
  int fd, res;
+  char *debug_file;
  if (openelf(path, &e, &fd) < 0)
    return -1;
+  // If there is a separate debuginfo file, try to locate and read it, first
+  // using the build-id section, then using the debuglink section. These are
+  // also the rules that GDB folows.
+  // See: https://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html
+  if (!is_debug_file) {
+    // The is_debug_file argument helps avoid infinitely resolving debuginfo
+    // files for debuginfo files and so on.
+    debug_file = find_debug_via_buildid(e);
+    if (!debug_file)
+      debug_file = find_debug_via_debuglink(e, path);
+    if (debug_file) {
+      foreach_sym_core(debug_file, callback, payload, 1);
+      free(debug_file);
+    }
+  }
  res = listsymbols(e, callback, payload);
  elf_end(e);
  close(fd);
  return res;
 }
+int bcc_elf_foreach_sym(const char *path, bcc_elf_symcb callback,
+                        void *payload) {
+  return foreach_sym_core(path, callback, payload, 0);
+}
 static int loadaddr(Elf *e, uint64_t *addr) {
  size_t phnum, i;
@@ -268,11 +501,11 @@ int bcc_elf_is_shared_obj(const char *path) {
 int main(int argc, char *argv[])
 {
-	uint64_t addr;
+  uint64_t addr;
-	if (bcc_elf_findsym(argv[1], argv[2], -1, STT_FUNC, &addr) < 0)
+  if (bcc_elf_findsym(argv[1], argv[2], -1, STT_FUNC, &addr) < 0)
-		return -1;
+    return -1;
-	printf("%s: %p\n", argv[2], (void *)addr);
+  printf("%s: %p\n", argv[2], (void *)addr);
-	return 0;
+  return 0;
 }
 #endif
--- a/src/cc/bcc_syms.cc
+++ b/src/cc/bcc_syms.cc
@@ -62,7 +62,7 @@ bool KSyms::resolve_addr(uint64_t addr, struct bcc_symbol *sym) {
  auto it = std::upper_bound(syms_.begin(), syms_.end(), Symbol("", addr)) - 1;
  sym->name = (*it).name.c_str();
  sym->demangle_name = sym->name;
-  sym->module = "[kernel]";
+  sym->module = "kernel";
  sym->offset = addr - (*it).addr;
  return true;
 }
@@ -140,6 +140,11 @@ bool ProcSyms::resolve_name(const char *module, const char *name,
  return false;
 }
+ProcSyms::Module::Module(const char *name, uint64_t start, uint64_t end)
+  : name_(name), start_(start), end_(end) {
+  is_so_ = bcc_elf_is_shared_obj(name) == 1;
+}
 int ProcSyms::Module::_add_symbol(const char *symname, uint64_t start,
                                  uint64_t end, int flags, void *p) {
  Module *m = static_cast<Module *>(p);
@@ -148,10 +153,6 @@ int ProcSyms::Module::_add_symbol(const char *symname, uint64_t start,
  return 0;
 }
-bool ProcSyms::Module::is_so() const {
-  return strstr(name_.c_str(), ".so") != nullptr;
-}
 bool ProcSyms::Module::is_perf_map() const {
  return strstr(name_.c_str(), ".map") != nullptr;
 }
@@ -225,10 +226,10 @@ int bcc_symcache_resolve(void *resolver, uint64_t addr,
  return cache->resolve_addr(addr, sym) ? 0 : -1;
 }
-int bcc_symcache_resolve_name(void *resolver, const char *name,
+int bcc_symcache_resolve_name(void *resolver, const char *module,
-                              uint64_t *addr) {
+                              const char *name, uint64_t *addr) {
  SymbolCache *cache = static_cast<SymbolCache *>(resolver);
-  return cache->resolve_name(nullptr, name, addr) ? 0 : -1;
+  return cache->resolve_name(module, name, addr) ? 0 : -1;
 }
 void bcc_symcache_refresh(void *resolver) {

--- a/src/cc/bcc_syms.h
+++ b/src/cc/bcc_syms.h
@@ -35,7 +35,8 @@ void *bcc_symcache_new(int pid);
 void bcc_free_symcache(void *symcache, int pid);
 int bcc_symcache_resolve(void *symcache, uint64_t addr, struct bcc_symbol *sym);
-int bcc_symcache_resolve_name(void *resolver, const char *name, uint64_t *addr);
+int bcc_symcache_resolve_name(void *resolver, const char *module,
+                              const char *name, uint64_t *addr);
 void bcc_symcache_refresh(void *resolver);
 int bcc_resolve_global_addr(int pid, const char *module, const uint64_t address,

--- a/src/cc/syms.h
+++ b/src/cc/syms.h
@@ -79,18 +79,18 @@ class ProcSyms : SymbolCache {
  };
  struct Module {
-    Module(const char *name, uint64_t start, uint64_t end)
+    Module(const char *name, uint64_t start, uint64_t end);
-        : name_(name), start_(start), end_(end) {}
    std::string name_;
    uint64_t start_;
    uint64_t end_;
+    bool is_so_;
    std::unordered_set<std::string> symnames_;
    std::vector<Symbol> syms_;
    void load_sym_table();
    bool find_addr(uint64_t addr, struct bcc_symbol *sym);
    bool find_name(const char *symname, uint64_t *addr);
-    bool is_so() const;
+    bool is_so() const { return is_so_; }
    bool is_perf_map() const;
    static int _add_symbol(const char *symname, uint64_t start, uint64_t end,

--- a/src/python/bcc/__init__.py
+++ b/src/python/bcc/__init__.py
@@ -27,7 +27,6 @@ basestring = (unicode if sys.version_info[0] < 3 else str)
 from .libbcc import lib, _CB_TYPE, bcc_symbol, _SYM_CB_TYPE
 from .table import Table
 from .perf import Perf
-from .usyms import ProcessSymbols
 from .utils import get_online_cpus
 _kprobe_limit = 1000
@@ -50,15 +49,29 @@ class SymbolCache(object):
        self.cache = lib.bcc_symcache_new(pid)
    def resolve(self, addr):
+        """
+        Return a tuple of the symbol (function), its offset from the beginning
+        of the function, and the module in which it lies. For example:
+            ("start_thread", 0x202, "/usr/lib/.../libpthread-2.24.so")
+        If the symbol cannot be found but we know which module it is in,
+        return the module name and the offset from the beginning of the
+        module. If we don't even know the module, return the absolute
+        address as the offset.
+        """
        sym = bcc_symbol()
        psym = ct.pointer(sym)
        if lib.bcc_symcache_resolve(self.cache, addr, psym) < 0:
-            return "[unknown]", 0
+            if sym.module and sym.offset:
-        return sym.demangle_name.decode(), sym.offset
+                return (None, sym.offset,
+                        ct.cast(sym.module, ct.c_char_p).value.decode())
-    def resolve_name(self, name):
+            return (None, addr, None)
+        return (sym.demangle_name.decode(), sym.offset,
+                ct.cast(sym.module, ct.c_char_p).value.decode())
+    def resolve_name(self, module, name):
        addr = ct.c_ulonglong()
-        if lib.bcc_symcache_resolve_name(self.cache, name, ct.pointer(addr)) < 0:
+        if lib.bcc_symcache_resolve_name(
+                    self.cache, module, name, ct.pointer(addr)) < 0:
            return -1
        return addr.value
@@ -968,53 +981,50 @@ class BPF(object):
        return BPF._sym_caches[pid]
    @staticmethod
-    def sym(addr, pid):
+    def sym(addr, pid, show_module=False, show_offset=False):
-        """sym(addr, pid)
+        """sym(addr, pid, show_module=False, show_offset=False)
        Translate a memory address into a function name for a pid, which is
-        returned.
+        returned. When show_module is True, the module name is also included.
+        When show_offset is True, the instruction offset as a hexadecimal
+        number is also included in the string.
        A pid of less than zero will access the kernel symbol cache.
-        """
-        name, _ = BPF._sym_cache(pid).resolve(addr)
-        return name
-    @staticmethod
-    def symaddr(addr, pid):
-        """symaddr(addr, pid)
-        Translate a memory address into a function name plus the instruction
+        Example output when both show_module and show_offset are True:
-        offset as a hexadecimal number, which is returned as a string.
+            "start_thread+0x202 [libpthread-2.24.so]"
-        A pid of less than zero will access the kernel symbol cache.
+        Example output when both show_module and show_offset are False:
+            "start_thread"
        """
-        name, offset = BPF._sym_cache(pid).resolve(addr)
+        name, offset, module = BPF._sym_cache(pid).resolve(addr)
-        return "%s+0x%x" % (name, offset)
+        offset = "+0x%x" % offset if show_offset and name is not None else ""
+        name = name or "[unknown]"
+        name = name + offset
+        module = " [%s]" % os.path.basename(module) if show_module else ""
+        return name + module
    @staticmethod
-    def ksym(addr):
+    def ksym(addr, show_module=False, show_offset=False):
        """ksym(addr)
        Translate a kernel memory address into a kernel function name, which is
-        returned.
+        returned. When show_module is True, the module name ("kernel") is also
-        """
+        included. When show_offset is true, the instruction offset as a 
-        return BPF.sym(addr, -1)
+        hexadecimal number is also included in the string.
-    @staticmethod
-    def ksymaddr(addr):
-        """ksymaddr(addr)
-        Translate a kernel memory address into a kernel function name plus the
+        Example output when both show_module and show_offset are True:
-        instruction offset as a hexidecimal number, which is returned as a
+            "default_idle+0x0 [kernel]"
-        string.
        """
-        return BPF.symaddr(addr, -1)
+        return BPF.sym(addr, -1, show_module, show_offset)
    @staticmethod
    def ksymname(name):
        """ksymname(name)
        Translate a kernel name into an address. This is the reverse of
-        ksymaddr. Returns -1 when the function name is unknown."""
+        ksym. Returns -1 when the function name is unknown."""
-        return BPF._sym_cache(-1).resolve_name(name)
+        return BPF._sym_cache(-1).resolve_name(None, name)
    def num_open_kprobes(self):
        """num_open_kprobes()

--- a/src/python/bcc/libbcc.py
+++ b/src/python/bcc/libbcc.py
@@ -154,7 +154,7 @@ lib.bcc_symcache_resolve.argtypes = [ct.c_void_p, ct.c_ulonglong, ct.POINTER(bcc
 lib.bcc_symcache_resolve_name.restype = ct.c_int
 lib.bcc_symcache_resolve_name.argtypes = [
-    ct.c_void_p, ct.c_char_p, ct.POINTER(ct.c_ulonglong)]
+    ct.c_void_p, ct.c_char_p, ct.c_char_p, ct.POINTER(ct.c_ulonglong)]
 lib.bcc_symcache_refresh.restype = None
 lib.bcc_symcache_refresh.argtypes = [ct.c_void_p]

--- a/src/python/bcc/usyms.py
+++ b/src/python/bcc/usyms.py
-# Copyright 2016 Sasha Goldshtein
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import ctypes as ct
-from .libbcc import lib, bcc_symbol
-class ProcessSymbols(object):
-    def __init__(self, pid):
-        """
-        Initializes the process symbols store for the specified pid.
-        Call refresh_code_ranges() periodically if you anticipate changes
-        in the set of loaded libraries or their addresses.
-        """
-        self.cache = lib.bcc_symcache_new(pid)
-    def refresh_code_ranges(self):
-        lib.bcc_symcache_refresh(self.cache)
-    def decode_addr(self, addr):
-        """
-        Given an address, return the best symbolic representation of it.
-        If it doesn't fall in any module, return its hex string. If it
-        falls within a module but we don't have a symbol for it, return
-        the hex string and the module. If we do have a symbol for it,
-        return the symbol and the module, e.g. "readline+0x10 [bash]".
-        """
-        sym = bcc_symbol()
-        psym = ct.pointer(sym)
-        if lib.bcc_symcache_resolve(self.cache, addr, psym) < 0:
-            if sym.module and sym.offset:
-                return "0x%x [%s]" % (sym.offset, sym.module)
-            return "%x" % addr
-        return "%s+0x%x [%s]" % (sym.name, sym.offset, sym.module)
--- a/tests/python/CMakeLists.txt
+++ b/tests/python/CMakeLists.txt
@@ -36,6 +36,8 @@ add_test(NAME py_test_trace4 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  COMMAND ${TEST_WRAPPER} py_trace4 sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_trace4.py)
 add_test(NAME py_test_probe_count WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  COMMAND ${TEST_WRAPPER} py_probe_count sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_probe_count.py)
+add_test(NAME py_test_debuginfo WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMAND ${TEST_WRAPPER} py_test_debuginfo sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_debuginfo.py)
 add_test(NAME py_test_brb WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  COMMAND ${TEST_WRAPPER} py_brb_c sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_brb.py test_brb.c)
 add_test(NAME py_test_brb2 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
@@ -60,6 +62,5 @@ add_test(NAME py_test_utils WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  COMMAND ${TEST_WRAPPER} py_test_utils sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_utils.py)
 add_test(NAME py_test_percpu WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  COMMAND ${TEST_WRAPPER} py_test_percpu sudo ${CMAKE_CURRENT_SOURCE_DIR}/test_percpu.py)
 add_test(NAME py_test_dump_func WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  COMMAND ${TEST_WRAPPER} py_dump_func simple ${CMAKE_CURRENT_SOURCE_DIR}/test_dump_func.py)
--- a/tests/python/dummy.c
+++ b/tests/python/dummy.c
+#include <unistd.h>
+#include <stdio.h>
+static __attribute__((noinline)) int some_function(int x, int y) {
+	volatile int z = x + y;
+	return z;
+}
+int main() {
+	printf("%p\n", &some_function);
+	fflush(stdout);
+	printf("result = %d\n", some_function(42, 11));
+	sleep(1000);
+	return 0;
+}
--- a/tests/python/test_debuginfo.py
+++ b/tests/python/test_debuginfo.py
+#!/usr/bin/env python
+# Copyright (c) Sasha Goldshtein
+# Licensed under the Apache License, Version 2.0 (the "License")
+import os
+import subprocess
+from bcc import SymbolCache
+from unittest import main, TestCase
+class Harness(TestCase):
+    def setUp(self):
+        self.build_command()
+        subprocess.check_output('objcopy --only-keep-debug dummy dummy.debug'
+                                .split())
+        self.debug_command()
+        subprocess.check_output('strip dummy'.split())
+        self.process = subprocess.Popen('./dummy', stdout=subprocess.PIPE)
+        # The process prints out the address of some symbol, which we then
+        # try to resolve in the test.
+        self.addr = int(self.process.stdout.readline().strip(), 16)
+        self.syms = SymbolCache(self.process.pid)
+    def tearDown(self):
+        self.process.kill()
+    def resolve_addr(self):
+        sym, offset, module = self.syms.resolve(self.addr)
+        self.assertEqual(sym, 'some_function')
+        self.assertEqual(offset, 0)
+        self.assertTrue(module[-5:] == 'dummy')
+    def resolve_name(self):
+        script_dir = os.path.dirname(os.path.realpath(__file__))
+        addr = self.syms.resolve_name(os.path.join(script_dir, 'dummy'),
+                                      'some_function')
+        self.assertEqual(addr, self.addr)
+        pass
+class TestDebuglink(Harness):
+    def build_command(self):
+        subprocess.check_output('gcc -o dummy dummy.c'.split())
+    def debug_command(self):
+        subprocess.check_output('objcopy --add-gnu-debuglink=dummy.debug dummy'
+                                .split())
+    def tearDown(self):
+        subprocess.check_output('rm dummy dummy.debug'.split())
+    def test_resolve_addr(self):
+        self.resolve_addr()
+    def test_resolve_name(self):
+        self.resolve_name()
+class TestBuildid(Harness):
+    def build_command(self):
+        subprocess.check_output(('gcc -o dummy -Xlinker ' + \
+               '--build-id=0x123456789abcdef0123456789abcdef012345678 dummy.c')
+               .split())
+    def debug_command(self):
+        subprocess.check_output('mkdir -p /usr/lib/debug/.build-id/12'.split())
+        subprocess.check_output(('mv dummy.debug /usr/lib/debug/.build-id' + \
+            '/12/3456789abcdef0123456789abcdef012345678.debug').split())
+    def tearDown(self):
+        subprocess.check_output('rm dummy'.split())
+        subprocess.check_output(('rm /usr/lib/debug/.build-id/12' +
+            '/3456789abcdef0123456789abcdef012345678.debug').split())
+    def test_resolve_name(self):
+        self.resolve_addr()
+    def test_resolve_addr(self):
+        self.resolve_name()
+if __name__ == "__main__":
+    main()
--- a/tools/memleak.py
+++ b/tools/memleak.py
@@ -11,31 +11,13 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 # Copyright (C) 2016 Sasha Goldshtein.
-from bcc import BPF, ProcessSymbols
+from bcc import BPF
 from time import sleep
 from datetime import datetime
 import argparse
 import subprocess
 import os
-class KStackDecoder(object):
-        def refresh(self):
-                pass
-        def __call__(self, addr):
-                return "%s [kernel] (%x)" % (BPF.ksym(addr), addr)
-class UStackDecoder(object):
-        def __init__(self, pid):
-                self.pid = pid
-                self.proc_sym = ProcessSymbols(pid)
-        def refresh(self):
-                self.proc_sym.refresh_code_ranges()
-        def __call__(self, addr):
-                return "%s (%x)" % (self.proc_sym.decode_addr(addr), addr)
 class Allocation(object):
    def __init__(self, stack, size):
        self.stack = stack
@@ -240,8 +222,6 @@ else:
        bpf_program.attach_kretprobe(event="__kmalloc", fn_name="alloc_exit")
        bpf_program.attach_kprobe(event="kfree", fn_name="free_enter")
-decoder = KStackDecoder() if kernel_trace else UStackDecoder(pid)
 def print_outstanding():
        print("[%s] Top %d stacks with outstanding allocations:" %
              (datetime.now().strftime("%H:%M:%S"), top_stacks))
@@ -256,8 +236,12 @@ def print_outstanding():
                if info.stack_id in alloc_info:
                        alloc_info[info.stack_id].update(info.size)
                else:
-                        stack = list(stack_traces.walk(info.stack_id, decoder))
+                        stack = list(stack_traces.walk(info.stack_id))
-                        alloc_info[info.stack_id] = Allocation(stack,
+                        combined = []
+                        for addr in stack:
+                                combined.append(bpf_program.sym(addr, pid,
+                                        show_module=True, show_offset=True))
+                        alloc_info[info.stack_id] = Allocation(combined,
                                                               info.size)
                if args.show_allocs:
                        print("\taddr = %x size = %s" %
@@ -277,7 +261,6 @@ while True:
                        sleep(interval)
                except KeyboardInterrupt:
                        exit()
-                decoder.refresh()
                print_outstanding()
                count_so_far += 1
                if num_prints is not None and count_so_far >= num_prints:

--- a/tools/memleak_example.txt
+++ b/tools/memleak_example.txt
@@ -10,13 +10,13 @@ For example:
 Attaching to malloc and free in pid 5193, Ctrl+C to quit.
 [11:16:33] Top 2 stacks with outstanding allocations:
        80 bytes in 5 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
+                 main+0x6d [allocs]
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fd460ac2790) 
+                 __libc_start_main+0xf0 [libc-2.21.so]
 [11:16:34] Top 2 stacks with outstanding allocations:
        160 bytes in 10 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
+                 main+0x6d [allocs]
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fd460ac2790) 
+                 __libc_start_main+0xf0 [libc-2.21.so]
 Each entry printed is a set of allocations that originate from the same call
@@ -40,8 +40,8 @@ Attaching to malloc and free in pid 5193, Ctrl+C to quit.
        addr = 948d30 size = 16
        addr = 948cf0 size = 16
        64 bytes in 4 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
+                 main+0x6d [allocs]
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fd460ac2790) 
+                 __libc_start_main+0xf0 [libc-2.21.so]
 [11:16:34] Top 2 stacks with outstanding allocations:
        addr = 948d50 size = 16
@@ -55,8 +55,8 @@ Attaching to malloc and free in pid 5193, Ctrl+C to quit.
        addr = 948d70 size = 16
        addr = 948df0 size = 16
        160 bytes in 10 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
+                 main+0x6d [allocs]
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fd460ac2790) 
+                 __libc_start_main+0xf0 [libc-2.21.so]
 When using the -p switch, memleak traces the allocations of a particular
@@ -67,35 +67,35 @@ For example:
 Attaching to kmalloc and kfree, Ctrl+C to quit.
 ...
        248 bytes in 4 allocations from stack
-                 bpf_prog_load [kernel] (ffffffff8118c471) 
+                 bpf_prog_load [kernel]
-                 sys_bpf [kernel] (ffffffff8118c8b5) 
+                 sys_bpf [kernel]
        328 bytes in 1 allocations from stack
-                 perf_mmap [kernel] (ffffffff811990fd) 
+                 perf_mmap [kernel]
-                 mmap_region [kernel] (ffffffff811df5d4) 
+                 mmap_region [kernel]
-                 do_mmap [kernel] (ffffffff811dfb83) 
+                 do_mmap [kernel]
-                 vm_mmap_pgoff [kernel] (ffffffff811c494f) 
+                 vm_mmap_pgoff [kernel]
-                 sys_mmap_pgoff [kernel] (ffffffff811ddf02) 
+                 sys_mmap_pgoff [kernel]
-                 sys_mmap [kernel] (ffffffff8101b0ab) 
+                 sys_mmap [kernel]
        464 bytes in 1 allocations from stack
-                 traceprobe_command [kernel] (ffffffff81187cf2) 
+                 traceprobe_command [kernel]
-                 traceprobe_probes_write [kernel] (ffffffff81187d86) 
+                 traceprobe_probes_write [kernel]
-                 probes_write [kernel] (ffffffff81181580) 
+                 probes_write [kernel]
-                 __vfs_write [kernel] (ffffffff812237b7) 
+                 __vfs_write [kernel]
-                 vfs_write [kernel] (ffffffff81223ec6) 
+                 vfs_write [kernel]
-                 sys_write [kernel] (ffffffff81224b85) 
+                 sys_write [kernel]
-                 entry_SYSCALL_64_fastpath [kernel] (ffffffff8178182e) 
+                 entry_SYSCALL_64_fastpath [kernel]
        8192 bytes in 1 allocations from stack
-                 alloc_and_copy_ftrace_hash.constprop.59 [kernel] (ffffffff8115d17e) 
+                 alloc_and_copy_ftrace_hash.constprop.59 [kernel]
-                 ftrace_set_hash [kernel] (ffffffff8115e767) 
+                 ftrace_set_hash [kernel]
-                 ftrace_set_filter_ip [kernel] (ffffffff8115e9a8) 
+                 ftrace_set_filter_ip [kernel]
-                 arm_kprobe [kernel] (ffffffff81148600) 
+                 arm_kprobe [kernel]
-                 enable_kprobe [kernel] (ffffffff811486f6) 
+                 enable_kprobe [kernel]
-                 kprobe_register [kernel] (ffffffff81182399) 
+                 kprobe_register [kernel]
-                 perf_trace_init [kernel] (ffffffff8117c4e0) 
+                 perf_trace_init [kernel]
-                 perf_tp_event_init [kernel] (ffffffff81192479) 
+                 perf_tp_event_init [kernel]
 Here you can see that arming the kprobe to which our eBPF program is attached
@@ -129,18 +129,18 @@ seconds, 3 times before quitting:
 Attaching to malloc and free in pid 2614, Ctrl+C to quit.
 [11:16:33] Top 2 stacks with outstanding allocations:
        16 bytes in 1 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
+                 main+0x6d [allocs]
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fdc11ce8790) 
+                 __libc_start_main+0xf0 [libc-2.21.so]
 [11:16:38] Top 2 stacks with outstanding allocations:
        16 bytes in 1 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
+                 main+0x6d [allocs]
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fdc11ce8790) 
+                 __libc_start_main+0xf0 [libc-2.21.so]
 [11:16:43] Top 2 stacks with outstanding allocations:
        32 bytes in 2 allocations from stack
-                 main+0x6d [/home/vagrant/allocs] (400862) 
+                 main+0x6d [allocs]
-                 __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fdc11ce8790) 
+                 __libc_start_main+0xf0 [libc-2.21.so]
 Note that even though the application leaks 16 bytes of memory every second, 
 the report (printed every 5 seconds) doesn't "see" all the allocations because

--- a/tools/offcputime.py
+++ b/tools/offcputime.py
@@ -271,11 +271,11 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
    else:
        # print default multi-line stack output
        for addr in kernel_stack:
-            print("    %016x %s" % (addr, b.ksym(addr)))
+            print("    %s" % b.ksym(addr))
        if need_delimiter:
            print("    --")
        for addr in user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.tgid)))
+            print("    %s" % b.sym(addr, k.tgid))
        print("    %-16s %s (%d)" % ("-", k.name, k.pid))
        print("        %d\n" % v.value)

--- a/tools/offwaketime.py
+++ b/tools/offwaketime.py
@@ -308,22 +308,22 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
        # print wakeup name then stack in reverse order
        print("    %-16s %s %s" % ("waker:", k.waker, k.t_pid))
        for addr in waker_user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.tgid)))
+            print("    %s" % b.sym(addr, k.tgid))
        if args.delimited:
            print("    -")
        for addr in waker_kernel_stack:
-            print("    %016x %s" % (addr, b.ksym(addr)))
+            print("    %s" % b.ksym(addr))
        # print waker/wakee delimiter
        print("    %-16s %s" % ("--", "--"))
        # print default multi-line stack output
        for addr in target_kernel_stack:
-            print("    %016x %s" % (addr, b.ksym(addr)))
+            print("    %s" % b.ksym(addr))
        if args.delimited:
            print("    -")
        for addr in target_user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.tgid)))
+            print("    %s" % b.sym(addr, k.tgid))
        print("    %-16s %s %s" % ("target:", k.target, k.w_pid))
        print("        %d\n" % v.value)

--- a/tools/old/memleak.py
+++ b/tools/old/memleak.py
@@ -11,36 +11,21 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 # Copyright (C) 2016 Sasha Goldshtein.
-from bcc import BPF, ProcessSymbols
+from bcc import BPF
 from time import sleep
 from datetime import datetime
 import argparse
 import subprocess
 import os
-class StackDecoder(object):
+def decode_stack(bpf, pid, info):
-        def __init__(self, pid):
+        stack = ""
-                self.pid = pid
+        if info.num_frames <= 0:
-                if pid != -1:
+                return "???"
-                        self.proc_sym = ProcessSymbols(pid)
+        for i in range(0, info.num_frames):
+                addr = info.callstack[i]
-        def refresh(self):
+                stack += " %s ;" % bpf.sym(addr, pid, show_offset=True)
-                if self.pid != -1:
+        return stack
-                        self.proc_sym.refresh_code_ranges()
-        def decode_stack(self, info, is_kernel_trace):
-                stack = ""
-                if info.num_frames <= 0:
-                        return "???"
-                for i in range(0, info.num_frames):
-                        addr = info.callstack[i]
-                        if is_kernel_trace:
-                                stack += " %s [kernel] (%x) ;" % \
-                                        (BPF.ksym(addr), addr)
-                        else:
-                                stack += " %s (%x) ;" % \
-                                        (self.proc_sym.decode_addr(addr), addr)
-                return stack
 def run_command_get_output(command):
        p = subprocess.Popen(command.split(),
@@ -255,8 +240,6 @@ else:
        bpf_program.attach_kretprobe(event="__kmalloc", fn_name="alloc_exit")
        bpf_program.attach_kprobe(event="kfree", fn_name="free_enter")
-decoder = StackDecoder(pid)
 def print_outstanding():
        stacks = {}
        print("[%s] Top %d stacks with outstanding allocations:" %
@@ -265,7 +248,7 @@ def print_outstanding():
        for address, info in sorted(allocs.items(), key=lambda a: a[1].size):
                if BPF.monotonic_time() - min_age_ns < info.timestamp_ns:
                        continue
-                stack = decoder.decode_stack(info, kernel_trace)
+                stack = decode_stack(bpf_program, pid, info)
                if stack in stacks:
                        stacks[stack] = (stacks[stack][0] + 1,
                                         stacks[stack][1] + info.size)
@@ -288,7 +271,6 @@ while True:
                        sleep(interval)
                except KeyboardInterrupt:
                        exit()
-                decoder.refresh()
                print_outstanding()
                count_so_far += 1
                if num_prints is not None and count_so_far >= num_prints:

--- a/tools/old/profile.py
+++ b/tools/old/profile.py
@@ -347,11 +347,11 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
    else:
        # print default multi-line stack output.
        for addr in kernel_stack:
-            print("    %016x %s" % (addr, aksym(addr)))
+            print("    %s" % aksym(addr))
        if do_delimiter:
            print("    --")
        for addr in user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.pid)))
+            print("    %s" % b.sym(addr, k.pid))
        print("    %-16s %s (%d)" % ("-", k.name, k.pid))
        print("        %d\n" % v.value)

--- a/tools/old/stackcount.py
+++ b/tools/old/stackcount.py
@@ -145,10 +145,7 @@ def print_frame(addr):
    print("  ", end="")
    if verbose:
        print("%-16x " % addr, end="")
-    if offset:
+    print(b.ksym(addr, show_offset=offset))
-        print("%s" % b.ksymaddr(addr))
-    else:
-        print("%s" % b.ksym(addr))
 # output
 exiting = 0 if args.interval else 1

--- a/tools/old/stacksnoop.py
+++ b/tools/old/stacksnoop.py
@@ -119,10 +119,7 @@ while 1:
    (task, pid, cpu, flags, ts, msg) = b.trace_fields()
    if msg != "":
        (reg, addr) = msg.split(" ")
-        if offset:
+        ip = b.ksym(int(addr, 16), show_offset=offset)
-            ip = b.ksymaddr(int(addr, 16))
-        else:
-            ip = b.ksym(int(addr, 16))
        msg = msg + " " + ip
    if verbose:
        print("%-18.9f %-12.12s %-6d %-3d %s" % (ts, task, pid, cpu, msg))

--- a/tools/profile.py
+++ b/tools/profile.py
@@ -287,11 +287,11 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
    else:
        # print default multi-line stack output.
        for addr in kernel_stack:
-            print("    %016x %s" % (addr, aksym(addr)))
+            print("    %s" % aksym(addr))
        if do_delimiter:
            print("    --")
        for addr in user_stack:
-            print("    %016x %s" % (addr, b.sym(addr, k.pid)))
+            print("    %s" % b.sym(addr, k.pid))
        print("    %-16s %s (%d)" % ("-", k.name, k.pid))
        print("        %d\n" % v.value)

--- a/tools/profile_example.txt
+++ b/tools/profile_example.txt
--- a/tools/stackcount.py
+++ b/tools/stackcount.py
@@ -225,7 +225,7 @@ class Tool(object):
        if self.args.verbose:
            print("%-16x " % addr, end="")
        if self.args.offset:
-            print("%s" % self.probe.bpf.symaddr(addr, pid))
+            print("%s" % self.probe.bpf.sym(addr, pid, show_offset=True))
        else:
            print("%s" % self.probe.bpf.sym(addr, pid))

--- a/tools/stacksnoop.py
+++ b/tools/stacksnoop.py
@@ -120,8 +120,8 @@ def print_event(cpu, data, size):
        print("%-18.9f %s" % (ts, function))
    for addr in stack_traces.walk(event.stack_id):
-        sym = b.ksymaddr(addr) if offset else b.ksym(addr)
+        sym = b.ksym(addr, show_offset=offset)
-        print("\t%016x %s" % (addr, sym))
+        print("\t%s" % sym)
    print()

--- a/tools/stacksnoop_example.txt
+++ b/tools/stacksnoop_example.txt
@@ -8,12 +8,12 @@ to see how they were invoked. For example, tracing the submit_bio() call:
 # ./stacksnoop submit_bio
 TIME(s)            SYSCALL
 3592.838736000     submit_bio
-        ffffffff813bd961 submit_bio
+        submit_bio
-        ffffffff81257c12 submit_bh
+        submit_bh
-        ffffffff81301948 jbd2_journal_commit_transaction
+        jbd2_journal_commit_transaction
-        ffffffff8130653a kjournald2
+        kjournald2
-        ffffffff810a2df8 kthread
+        kthread
-        ffffffff8183a122 ret_from_fork
+        ret_from_fork
 This shows that submit_bio() was called by submit_bh(), which was called
 by jbd2_journal_commit_transaction(), and so on. 
@@ -28,12 +28,12 @@ The -v option includes more fields, including the on-CPU process (COMM and PID):
 # ./stacksnoop -v submit_bio
 TIME(s)            COMM         PID    CPU SYSCALL
 3734.855027000     jbd2/dm-0-8  313    0   submit_bio
-        ffffffff813bd961 submit_bio
+        submit_bio
-        ffffffff81257c12 submit_bh
+        submit_bh
-        ffffffff81301948 jbd2_journal_commit_transaction
+        jbd2_journal_commit_transaction
-        ffffffff8130653a kjournald2
+        kjournald2
-        ffffffff810a2df8 kthread
+        kthread
-        ffffffff8183a122 ret_from_fork
+        ret_from_fork
 This identifies the application issuing the sync syscall: the jbd2 process
 (COMM column).
@@ -45,30 +45,30 @@ process:
 # ./stacksnoop -v second_overflow
 TIME(s)            COMM         PID    CPU SYSCALL
 3837.526433000     <idle>       0      1   second_overflow
-        ffffffff810fac41 second_overflow
+        second_overflow
-        ffffffff81102320 tick_do_update_jiffies64
+        tick_do_update_jiffies64
-        ffffffff81102bf0 tick_irq_enter
+        tick_irq_enter
-        ffffffff810882ac irq_enter
+        irq_enter
-        ffffffff8183c7df smp_apic_timer_interrupt
+        smp_apic_timer_interrupt
-        ffffffff8183aae2 apic_timer_interrupt
+        apic_timer_interrupt
-        ffffffff81038f9e default_idle
+        default_idle
-        ffffffff8103979f arch_cpu_idle
+        arch_cpu_idle
-        ffffffff810c69da default_idle_call
+        default_idle_call
-        ffffffff810c6cd7 cpu_startup_entry
+        cpu_startup_entry
-        ffffffff81051cbe start_secondary
+        start_secondary
 3838.526953000     <idle>       0      1   second_overflow
-        ffffffff810fac41 second_overflow
+        second_overflow
-        ffffffff81102320 tick_do_update_jiffies64
+        tick_do_update_jiffies64
-        ffffffff81102bf0 tick_irq_enter
+        tick_irq_enter
-        ffffffff810882ac irq_enter
+        irq_enter
-        ffffffff8183c7df smp_apic_timer_interrupt
+        smp_apic_timer_interrupt
-        ffffffff8183aae2 apic_timer_interrupt
+        apic_timer_interrupt
-        ffffffff81038f9e default_idle
+        default_idle
-        ffffffff8103979f arch_cpu_idle
+        arch_cpu_idle
-        ffffffff810c69da default_idle_call
+        default_idle_call
-        ffffffff810c6cd7 cpu_startup_entry
+        cpu_startup_entry
-        ffffffff81051cbe start_secondary
+        start_secondary
 This fires every second (see TIME(s)), and is from tick_do_update_jiffies64().

--- a/tools/trace.py
+++ b/tools/trace.py
@@ -457,7 +457,8 @@ BPF_PERF_OUTPUT(%s);
            stack = list(bpf.get_table(self.stacks_name).walk(stack_id))
            for addr in stack:
-                    print("        %016x %s" % (addr, bpf.sym(addr, tgid)))
+                    print("        %s" % (bpf.sym(addr, tgid,
+                                         show_module=True, show_offset=True)))
        def _format_message(self, bpf, tgid, values):
                # Replace each %K with kernel sym and %U with user sym in tgid
@@ -466,9 +467,10 @@ BPF_PERF_OUTPUT(%s);
                user_placeholders = [i for i, t in enumerate(self.types)
                                     if t == 'U']
                for kp in kernel_placeholders:
-                        values[kp] = bpf.ksymaddr(values[kp])
+                        values[kp] = bpf.ksym(values[kp], show_offset=True)
                for up in user_placeholders:
-                        values[up] = bpf.symaddr(values[up], tgid)
+                        values[up] = bpf.sym(values[up], tgid,
+                                           show_module=True, show_offset=True)
                return self.python_format % tuple(values)
        def print_event(self, bpf, cpu, data, size):

--- a/tools/uthreads.py
+++ b/tools/uthreads.py
@@ -104,7 +104,7 @@ def print_event(cpu, data, size):
    event = ct.cast(data, ct.POINTER(ThreadEvent)).contents
    name = event.name
    if event.type == "pthread":
-        name = bpf.sym(event.runtime_id, args.pid)
+        name = bpf.sym(event.runtime_id, args.pid, show_module=True)
        tid = event.native_id
    else:
        tid = "R=%s/N=%s" % (event.runtime_id, event.native_id)

--- a/tools/uthreads_example.txt
+++ b/tools/uthreads_example.txt
@@ -25,10 +25,10 @@ Next, trace only pthread creation events in some native application:
 # ./uthreads 27450
 Tracing thread events in process 27450 (language: none)... Ctrl-C to quit.
 TIME     ID               TYPE     DESCRIPTION                   
-0.924    27462            pthread  primes_thread                 
+0.924    27462            pthread  primes_thread [primes]
-0.927    27463            pthread  primes_thread                 
+0.927    27463            pthread  primes_thread [primes]     
-0.928    27464            pthread  primes_thread                 
+0.928    27464            pthread  primes_thread [primes]        
-0.928    27465            pthread  primes_thread                 
+0.928    27465            pthread  primes_thread [primes]        
 ^C
 The thread name ("primes_thread" in this example) is resolved from debuginfo.