Link against system installed bcc (#327)

Link against system bcc

Previously, bpftrace was clone bcc from github at build time, building
a version of bcc, then linking against the built bcc. This is a no-go
for many packaging systems including Fedora, openSUSE, and the internal
Facebook one.

This patch requires bcc and bcc development headers be installed to
build bpftrace and removes the build time downloading behavior.
Also simplifies some of the header search logic.

Test plan:
```
$ ./build/tests/bpftrace_test
...
$ sudo make -C build runtime-tests
...

$ cat ~/scratch/bpftrace/path.bt

kprobe:vfs_open
{
    printf("open path: %s\n", str(((path *)arg0)->dentry->d_name.name));
}

$ sudo ./build/src/bpftrace ~/scratch/bpftrace/path.bt
[sudo] password for dlxu:
Attaching 1 probe...
open path: stat
open path: user-745673103.journal
open path: user-1001.journal
open path: user-745673103.journal
open path: user-1001.journal
open path: user-745673103.journal
open path: user-1001.journal
open path: iio:device0
open path: stat
...

CMakeLists.txt

@@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 2.8.12)
 project(bpftrace)
 
 set(STATIC_LINKING OFF CACHE BOOL "Build bpftrace as a statically linked executable")
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 
 add_compile_options("-std=c++14")
 add_compile_options("-Wno-format-security")
@@ -22,25 +23,6 @@ add_compile_options("-Wno-format-security")
 
 enable_testing()
 
-if (OFFLINE_BUILDS)
-  include(ExternalProject)
-  ExternalProject_Add(bcc
-    GIT_REPOSITORY https://github.com/iovisor/bcc
-    STEP_TARGETS build update
-    EXCLUDE_FROM_ALL 1
-    UPDATE_DISCONNECTED 1
-    BUILD_COMMAND ${CMAKE_COMMAND} --build . --target bcc-static
-    )
-else()
-  include(ExternalProject)
-  ExternalProject_Add(bcc
-    GIT_REPOSITORY https://github.com/iovisor/bcc
-    STEP_TARGETS build update
-    EXCLUDE_FROM_ALL 1
-    BUILD_COMMAND ${CMAKE_COMMAND} --build . --target bcc-static
-    )
-endif()
-
 if (STATIC_LINKING)
   set(CMAKE_EXE_LINKER_FLAGS "-static")
   set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
@@ -48,7 +30,9 @@ if (STATIC_LINKING)
   set(CMAKE_LINK_SEARCH_END_STATIC TRUE)
 endif()
 
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+find_package(LibBcc REQUIRED)
+include_directories(${LIBBCC_INCLUDE_DIRS})
+
 find_package(LibElf REQUIRED)
 include_directories(${LIBELF_INCLUDE_DIRS})
 

INSTALL.md

@@ -56,7 +56,7 @@ deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main
 deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main
 EOF
 sudo apt-get update
-sudo apt-get install -y bison cmake flex g++ git libelf-dev zlib1g-dev libfl-dev
+sudo apt-get install -y bison cmake flex g++ git libelf-dev zlib1g-dev libfl-dev bcc-dev
 sudo apt-get install clang-5.0 libclang-5.0-dev libclang-common-5.0-dev libclang1-5.0 libllvm5.0 llvm-5.0 llvm-5.0-dev llvm-5.0-runtime
 git clone https://github.com/iovisor/bpftrace
 cd bpftrace
@@ -72,7 +72,7 @@ The bpftrace binary will be in installed in /usr/local/bin/bpftrace, and tools i
 You'll want the newest kernel possible (see kernel requirements), eg, by using Fedora 28 or newer.
 
 ```
-sudo dnf install -y bison flex cmake make git gcc-c++ elfutils-libelf-devel zlib-devel llvm-devel clang-devel
+sudo dnf install -y bison flex cmake make git gcc-c++ elfutils-libelf-devel zlib-devel llvm-devel clang-devel bcc-devel
 git clone https://github.com/iovisor/bpftrace
 cd bpftrace
 mkdir build; cd build; cmake -DCMAKE_BUILD_TYPE=DEBUG ..
@@ -158,6 +158,7 @@ Use specific OS build sections listed earlier if available (Ubuntu, Docker).
 - Flex
 - Bison
 - LLVM & Clang 5.0 (or 6.0) development packages
+- BCC development package
 - LibElf
 - Kernel requirements described earlier
 

cmake/FindLibBcc.cmake

+# - Try to find libbcc
+# Once done this will define
+#
+#  LIBBCC_FOUND - system has libbcc
+#  LIBBCC_INCLUDE_DIRS - the libbcc include directory
+#  LIBBCC_LIBRARIES - Link these to use libbcc
+#  LIBBCC_DEFINITIONS - Compiler switches required for using libbcc
+
+if (LIBBCC_LIBRARIES AND LIBBCC_INCLUDE_DIRS)
+  set (LibBcc_FIND_QUIETLY TRUE)
+endif (LIBBCC_LIBRARIES AND LIBBCC_INCLUDE_DIRS)
+
+find_path (LIBBCC_INCLUDE_DIRS
+  NAMES
+    libbpf.h
+  PATHS
+    /usr/include
+    /usr/include/bcc
+    /usr/local/include
+    /usr/local/include/libbcc
+    /opt/local/include
+    /opt/local/include/libbcc
+    /sw/include
+    /sw/include/libbcc
+    ENV CPATH)
+
+find_library (LIBBCC_LIBRARIES
+  NAMES
+    bcc
+  PATHS
+    /usr/lib
+    /usr/local/lib
+    /opt/local/lib
+    /sw/lib
+    ENV LIBRARY_PATH
+    ENV LD_LIBRARY_PATH)
+
+include (FindPackageHandleStandardArgs)
+
+
+# handle the QUIETLY and REQUIRED arguments and set LIBBCC_FOUND to TRUE if all listed variables are TRUE
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibBcc "Please install the libbcc development package"
+  LIBBCC_LIBRARIES
+  LIBBCC_INCLUDE_DIRS)

cmake/FindLibElf.cmake

@@ -47,7 +47,7 @@ include (FindPackageHandleStandardArgs)
 
 
 # handle the QUIETLY and REQUIRED arguments and set LIBELF_FOUND to TRUE if all listed variables are TRUE
-FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibElf DEFAULT_MSG
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibElf "Please install the libelf development package"
   LIBELF_LIBRARIES
   LIBELF_INCLUDE_DIRS)
 

docker/Dockerfile.alpine

@@ -11,7 +11,8 @@ RUN apk add --update \
   linux-headers \
   llvm5-dev \
   llvm5-static \
-  zlib-dev
+  zlib-dev \
+  bcc-dev
 
 # Put LLVM directories where CMake expects them to be
 RUN ln -s /usr/lib/cmake/llvm5 /usr/lib/cmake/llvm

docker/Dockerfile.fedora27

@@ -9,7 +9,8 @@ RUN dnf install -y \
     git \
     llvm-devel \
     make \
-    zlib-devel
+    zlib-devel \
+    bcc-devel
 
 COPY build.sh /build.sh
 ENTRYPOINT ["/bin/sh", "/build.sh"]

docker/Dockerfile.fedora28

@@ -9,7 +9,8 @@ RUN dnf install -y \
     git \
     llvm-devel \
     make \
-    zlib-devel
+    zlib-devel \
+    bcc-devel
 
 COPY build.sh /build.sh
 ENTRYPOINT ["/bin/sh", "/build.sh"]

docker/Dockerfile.fedora29

@@ -9,7 +9,8 @@ RUN dnf install -y \
     git \
     llvm-devel \
     make \
-    zlib-devel
+    zlib-devel \
+    bcc-devel
 
 COPY build.sh /build.sh
 ENTRYPOINT ["/bin/sh", "/build.sh"]

docker/Dockerfile.ubuntu

@@ -8,7 +8,8 @@ RUN apt-get update && apt-get install -y \
     libclang-5.0-dev \
     libelf-dev \
     llvm-5.0-dev \
-    zlib1g-dev
+    zlib1g-dev \
+    bcc-dev
 
 COPY build.sh /build.sh
 ENTRYPOINT ["bash", "/build.sh"]

src/CMakeLists.txt

@@ -11,6 +11,7 @@ add_executable(bpftrace
   resolve_cgroupid.cpp
   tracepoint_format_parser.cpp
   types.cpp
+  utils.cpp
   list.cpp
 )
 
@@ -19,12 +20,7 @@ if(HAVE_NAME_TO_HANDLE_AT)
 endif(HAVE_NAME_TO_HANDLE_AT)
 target_link_libraries(bpftrace arch ast parser resources)
 
-ExternalProject_Get_Property(bcc source_dir binary_dir)
-target_include_directories(bpftrace PUBLIC ${source_dir}/src/cc)
-target_link_libraries(bpftrace ${binary_dir}/src/cc/libbpf.a)
-target_link_libraries(bpftrace ${binary_dir}/src/cc/libbcc-loader-static.a)
-target_link_libraries(bpftrace ${binary_dir}/src/cc/libbcc.a)
-target_link_libraries(bpftrace ${binary_dir}/src/cc/frontends/clang/libclang_frontend.a)
+target_link_libraries(bpftrace ${LIBBCC_LIBRARIES})
 target_link_libraries(bpftrace ${LIBELF_LIBRARIES})
 
 install(TARGETS bpftrace DESTINATION bin)

src/ast/CMakeLists.txt

@@ -11,9 +11,7 @@ target_include_directories(ast PUBLIC ${CMAKE_SOURCE_DIR}/src/ast)
 target_include_directories(ast PUBLIC ${CMAKE_BINARY_DIR})
 target_link_libraries(ast arch)
 
-add_dependencies(ast bcc-build parser)
-ExternalProject_Get_Property(bcc source_dir)
-target_include_directories(ast PUBLIC ${source_dir}/src/cc)
+add_dependencies(ast parser)
 
 if (STATIC_LINKING)
   set(clang_libs

src/attached_probe.cpp

@@ -12,9 +12,9 @@
 
 #include "attached_probe.h"
 #include "bpftrace.h"
+#include "utils.h"
 #include "bcc_syms.h"
 #include "bcc_usdt.h"
-#include "common.h"
 #include "libbpf.h"
 #include "utils.h"
 #include <linux/perf_event.h>
@@ -474,7 +474,7 @@ void AttachedProbe::attach_profile()
     abort();
   }
 
-  std::vector<int> cpus = ebpf::get_online_cpus();
+  std::vector<int> cpus = get_online_cpus();
   for (int cpu : cpus)
   {
     int perf_event_fd = bpf_attach_perf_event(progfd_, PERF_TYPE_SOFTWARE,
@@ -584,7 +584,7 @@ void AttachedProbe::attach_software()
   if (period == 0)
     period = defaultp;
 
-  std::vector<int> cpus = ebpf::get_online_cpus();
+  std::vector<int> cpus = get_online_cpus();
   for (int cpu : cpus)
   {
     int perf_event_fd = bpf_attach_perf_event(progfd_, PERF_TYPE_SOFTWARE,
@@ -654,7 +654,7 @@ void AttachedProbe::attach_hardware()
   if (period == 0)
     period = defaultp;
 
-  std::vector<int> cpus = ebpf::get_online_cpus();
+  std::vector<int> cpus = get_online_cpus();
   for (int cpu : cpus)
   {
     int perf_event_fd = bpf_attach_perf_event(progfd_, PERF_TYPE_HARDWARE,

src/bpftrace.cpp

@@ -42,6 +42,15 @@ BPFtrace::~BPFtrace()
     int status;
     waitpid(pid, &status, 0);
   }
+
+  for (const auto& pair : pid_sym_)
+  {
+    if (pair.second)
+      bcc_free_symcache(pair.second, pair.first);
+  }
+
+  if (ksyms_)
+    bcc_free_symcache(ksyms_, -1);
 }
 
 int BPFtrace::add_probe(ast::Probe &p)
@@ -567,7 +576,7 @@ int BPFtrace::setup_perf_events()
     return -1;
   }
 
-  std::vector<int> cpus = ebpf::get_online_cpus();
+  std::vector<int> cpus = get_online_cpus();
   online_cpus_ = cpus.size();
   for (int cpu : cpus)
   {
@@ -1481,7 +1490,10 @@ std::string BPFtrace::resolve_sym(uintptr_t addr, bool show_offset)
   struct bcc_symbol sym;
   std::ostringstream symbol;
 
-  if (ksyms_.resolve_addr(addr, &sym))
+  if (!ksyms_)
+    ksyms_ = bcc_symcache_new(-1, nullptr);
+
+  if (bcc_symcache_resolve(ksyms_, addr, &sym))
   {
     symbol << sym.name;
     if (show_offset)
@@ -1614,7 +1626,7 @@ std::string BPFtrace::resolve_usym(uintptr_t addr, int pid, bool show_offset)
     psyms = pid_sym_[pid];
   }
 
-  if (((ProcSyms *)psyms)->resolve_addr(addr, &sym))
+  if (bcc_symcache_resolve(psyms, addr, &sym))
   {
     symbol << sym.name;
     if (show_offset)
@@ -1625,8 +1637,6 @@ std::string BPFtrace::resolve_usym(uintptr_t addr, int pid, bool show_offset)
     symbol << (void*)addr;
   }
 
-  // TODO: deal with process exit and clearing its psyms entry
-
   return symbol.str();
 }
 

src/bpftrace.h

@@ -5,14 +5,12 @@
 #include <set>
 #include <vector>
 
-#include "common.h"
-#include "syms.h"
-
 #include "ast.h"
 #include "attached_probe.h"
 #include "imap.h"
 #include "printf.h"
 #include "struct.h"
+#include "utils.h"
 #include "types.h"
 
 namespace bpftrace {
@@ -50,7 +48,7 @@ inline DebugLevel operator++(DebugLevel& level, int)
 class BPFtrace
 {
 public:
-  BPFtrace() : ncpus_(ebpf::get_possible_cpus().size()) { }
+  BPFtrace() : ncpus_(get_possible_cpus().size()) { }
   virtual ~BPFtrace();
   virtual int add_probe(ast::Probe &p);
   int num_probes() const;
@@ -102,7 +100,7 @@ protected:
 private:
   std::vector<std::unique_ptr<AttachedProbe>> attached_probes_;
   std::vector<std::unique_ptr<AttachedProbe>> special_attached_probes_;
-  KSyms ksyms_;
+  void* ksyms_{nullptr};
   std::map<int, void *> pid_sym_;
   int ncpus_;
   int online_cpus_;

src/clang_parser.cpp

@@ -2,13 +2,13 @@
 #include <iostream>
 #include <string.h>
 #include <sys/utsname.h>
-
-#include "frontends/clang/kbuild_helper.h"
+#include <sys/stat.h>
 
 #include "ast.h"
 #include "bpftrace.h"
 #include "clang_parser.h"
 #include "types.h"
+#include "utils.h"
 
 extern "C" const char __stddef_max_align_t_h[];
 extern "C" const unsigned __stddef_max_align_t_h_len;
@@ -134,11 +134,21 @@ static bool is_dir(const std::string& path)
   return S_ISDIR(buf.st_mode);
 }
 
-static std::pair<bool, std::string> get_kernel_path_info(const std::string &kdir)
+static std::string get_kernel_root_dir(const struct utsname& utsname)
 {
+#ifdef KERNEL_HEADERS_DIR
+  return KERNEL_HEADERS_DIR;
+#endif
+
+  const char *kpath_env = ::getenv("BPFTRACE_KERNEL_SOURCE");
+  if (kpath_env)
+    return kpath_env;
+
+  std::string kdir = std::string("/lib/modules/") + utsname.release;
   if (is_dir(kdir + "/build") && is_dir(kdir + "/source"))
-    return std::make_pair(true, "source");
-  return std::make_pair(false, "build");
+    return kdir + "/source";
+  else
+    return kdir + "/build";
 }
 
 void ClangParser::parse(ast::Program *program, StructMap &structs)
@@ -186,33 +196,12 @@ void ClangParser::parse(ast::Program *program, StructMap &structs)
     },
   };
 
+  std::vector<std::string> kflags;
   struct utsname utsname;
   uname(&utsname);
-  const char *kpath_env = ::getenv("BPFTRACE_KERNEL_SOURCE");
-  const char *kpath_fixed =
-  #ifdef KERNEL_HEADERS_DIR
-    kpath_env ? kpath_env : KERNEL_HEADERS_DIR;
-  #else
-    kpath_env;
-  #endif
-  std::string kdir = kpath_fixed ?
-    std::string(kpath_fixed) :
-    std::string("/lib/modules/") + utsname.release;
-
-  auto kpath_info = get_kernel_path_info(kdir);
-  auto kpath = kpath_fixed ?
-    kdir :
-    kdir + "/" + kpath_info.second;
-  bool has_kpath_source = kpath_fixed ? false : kpath_info.first;
-
-  std::vector<std::string> kflags;
-
-  ebpf::DirStack dstack(kpath);
-  if (dstack.ok())
-  {
-    ebpf::KBuildHelper kbuild_helper(kdir, has_kpath_source);
-    kbuild_helper.get_flags(utsname.machine, &kflags);
-  }
+  auto kpath = get_kernel_root_dir(utsname);
+  if (is_dir(kpath))
+    kflags = get_kernel_cflags(utsname.machine, kpath);
 
   std::vector<const char *> args =
   {

src/map.cpp

@@ -2,8 +2,8 @@
 #include <unistd.h>
 #include <linux/version.h>
 
-#include "common.h"
 #include "libbpf.h"
+#include "utils.h"
 
 #include "map.h"
 
@@ -68,7 +68,7 @@ Map::Map(enum bpf_map_type map_type)
   }
   else if (map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY)
   {
-    std::vector<int> cpus = ebpf::get_online_cpus();
+    std::vector<int> cpus = get_online_cpus();
     name = "printf";
     key_size = 4;
     value_size = 4;

src/utils.cpp

+#include "utils.h"
+
+#include <string.h>
+
+#include <fstream>
+
+namespace {
+
+std::vector<int> read_cpu_range(std::string path)
+{
+  std::ifstream cpus_range_stream { path };
+  std::vector<int> cpus;
+  std::string cpu_range;
+
+  while (std::getline(cpus_range_stream, cpu_range, ',')) {
+    std::size_t rangeop = cpu_range.find('-');
+    if (rangeop == std::string::npos) {
+      cpus.push_back(std::stoi(cpu_range));
+    }
+    else {
+      int start = std::stoi(cpu_range.substr(0, rangeop));
+      int end = std::stoi(cpu_range.substr(rangeop + 1));
+      for (int i = start; i <= end; i++)
+        cpus.push_back(i);
+    }
+  }
+  return cpus;
+}
+
+} // namespace
+
+namespace bpftrace {
+
+std::vector<int> get_online_cpus()
+{
+  return read_cpu_range("/sys/devices/system/cpu/online");
+}
+
+std::vector<int> get_possible_cpus()
+{
+  return read_cpu_range("/sys/devices/system/cpu/possible");
+}
+
+std::vector<std::string> get_kernel_cflags(
+    const char* uname_machine,
+    const std::string& kdir)
+{
+  std::vector<std::string> cflags;
+  std::string arch = uname_machine;
+  const char *archenv;
+
+  if (!strncmp(uname_machine, "x86_64", 6)) {
+    arch = "x86";
+  } else if (uname_machine[0] == 'i' && !strncmp(&uname_machine[2], "86", 2)) {
+    arch = "x86";
+  } else if (!strncmp(uname_machine, "arm", 3)) {
+    arch = "arm";
+  } else if (!strncmp(uname_machine, "sa110", 5)) {
+    arch = "arm";
+  } else if (!strncmp(uname_machine, "s390x", 5)) {
+    arch = "s390";
+  } else if (!strncmp(uname_machine, "parisc64", 8)) {
+    arch = "parisc";
+  } else if (!strncmp(uname_machine, "ppc", 3)) {
+    arch = "powerpc";
+  } else if (!strncmp(uname_machine, "mips", 4)) {
+    arch = "mips";
+  } else if (!strncmp(uname_machine, "sh", 2)) {
+    arch = "sh";
+  } else if (!strncmp(uname_machine, "aarch64", 7)) {
+    arch = "arm64";
+  }
+
+  // If ARCH env is defined, use it over uname
+  archenv = getenv("ARCH");
+  if (archenv)
+    arch = std::string(archenv);
+
+  cflags.push_back("-nostdinc");
+  cflags.push_back("-isystem");
+  cflags.push_back("/virtual/lib/clang/include");
+
+  cflags.push_back("-I" + kdir + "/arch/"+arch+"/include");
+  cflags.push_back("-I" + kdir + "/arch/"+arch+"/include/generated/uapi");
+  cflags.push_back("-I" + kdir + "/arch/"+arch+"/include/generated");
+  cflags.push_back("-I" + kdir + "/include");
+  cflags.push_back("-I" + kdir + "/./arch/"+arch+"/include/uapi");
+  cflags.push_back("-I" + kdir + "/arch/"+arch+"/include/generated/uapi");
+  cflags.push_back("-I" + kdir + "/include/uapi");
+  cflags.push_back("-I" + kdir + "/include/generated");
+  cflags.push_back("-I" + kdir + "/include/generated/uapi");
+
+  cflags.push_back("-include");
+  cflags.push_back(kdir + "/include/linux/kconfig.h");
+  cflags.push_back("-D__KERNEL__");
+  cflags.push_back("-D__HAVE_BUILTIN_BSWAP16__");
+  cflags.push_back("-D__HAVE_BUILTIN_BSWAP32__");
+  cflags.push_back("-D__HAVE_BUILTIN_BSWAP64__");
+
+  // If ARCH env variable is set, pass this along.
+  if (archenv)
+	cflags.push_back("-D__TARGET_ARCH_" + arch);
+
+  return cflags;
+}
+
+} // namespace bpftrace

src/utils.h

 #pragma once
 
 #include <string>
+#include <vector>
 
 namespace bpftrace {
 
 inline std::string GetProviderFromPath(std::string path);
+std::vector<int> get_online_cpus();
+std::vector<int> get_possible_cpus();
+std::vector<std::string> get_kernel_cflags(
+    const char* uname_machine,
+    const std::string& kdir);
 
 } // namespace bpftrace
 

tests/CMakeLists.txt

@@ -25,6 +25,7 @@ add_executable(bpftrace_test
   ${CMAKE_SOURCE_DIR}/src/resolve_cgroupid.cpp
   ${CMAKE_SOURCE_DIR}/src/tracepoint_format_parser.cpp
   ${CMAKE_SOURCE_DIR}/src/types.cpp
+  ${CMAKE_SOURCE_DIR}/src/utils.cpp
 )
 
 if(HAVE_NAME_TO_HANDLE_AT)
@@ -32,12 +33,7 @@ if(HAVE_NAME_TO_HANDLE_AT)
 endif(HAVE_NAME_TO_HANDLE_AT)
 target_link_libraries(bpftrace_test arch ast parser resources)
 
-ExternalProject_Get_Property(bcc source_dir binary_dir)
-target_include_directories(bpftrace_test PUBLIC ${source_dir}/src/cc)
-target_link_libraries(bpftrace_test ${binary_dir}/src/cc/libbpf.a)
-target_link_libraries(bpftrace_test ${binary_dir}/src/cc/libbcc-loader-static.a)
-target_link_libraries(bpftrace_test ${binary_dir}/src/cc/libbcc.a)
-target_link_libraries(bpftrace_test ${binary_dir}/src/cc/frontends/clang/libclang_frontend.a)
+target_link_libraries(bpftrace_test ${LIBBCC_LIBRARIES})
 target_link_libraries(bpftrace_test ${LIBELF_LIBRARIES})
 
 find_package(Threads REQUIRED)