Embed runtime header files in libbcc.so

To avoid installing header files needed by clang to disk
(/usr/share/bcc), embed the files as strings inside the library and feed
them to clang as memory buffers. The mechanism that we use to do this
retains all of the existing features, as in one can still do `#include
<bcc/helpers.h>`, even though it is redundant, and clang will pick up
the embedded file.

Fixes: #333
Signed-off-by: Brenden Blanco <bblanco@plumgrid.com>

SPECS/bcc.el6.spec

@@ -107,8 +107,6 @@ Python bindings for BPF Compiler Collection (BCC)
 
 %files -n libbcc
 /usr/lib64/*
-/usr/share/bcc/include/*
-/usr/share/bcc/lib/*
 /usr/include/bcc/*
 
 %files -n libbcc-examples

SPECS/bcc.el6.spec.in

@@ -107,8 +107,6 @@ Python bindings for BPF Compiler Collection (BCC)
 
 %files -n libbcc
 /usr/lib64/*
-/usr/share/bcc/include/*
-/usr/share/bcc/lib/*
 /usr/include/bcc/*
 
 %files -n libbcc-examples

SPECS/bcc.el7.spec

@@ -76,8 +76,6 @@ Python bindings for BPF Compiler Collection (BCC)
 
 %files -n libbcc
 /usr/lib64/*
-/usr/share/bcc/include/*
-/usr/share/bcc/lib/*
 /usr/include/bcc/*
 
 %files -n libbcc-examples

SPECS/bcc.el7.spec.in

@@ -76,8 +76,6 @@ Python bindings for BPF Compiler Collection (BCC)
 
 %files -n libbcc
 /usr/lib64/*
-/usr/share/bcc/include/*
-/usr/share/bcc/lib/*
 /usr/include/bcc/*
 
 %files -n libbcc-examples

SPECS/bcc.f22.spec

@@ -76,8 +76,6 @@ Python bindings for BPF Compiler Collection (BCC)
 
 %files -n libbcc
 /usr/lib64/*
-/usr/share/bcc/include/*
-/usr/share/bcc/lib/*
 /usr/include/bcc/*
 
 %files -n libbcc-examples

SPECS/bcc.f22.spec.in

@@ -76,8 +76,6 @@ Python bindings for BPF Compiler Collection (BCC)
 
 %files -n libbcc
 /usr/lib64/*
-/usr/share/bcc/include/*
-/usr/share/bcc/lib/*
 /usr/include/bcc/*
 
 %files -n libbcc-examples

SPECS/bcc.spec

@@ -71,8 +71,6 @@ Command line tools for BPF Compiler Collection (BCC)
 
 %files -n libbcc
 /usr/lib64/*
-/usr/share/bcc/include/*
-/usr/share/bcc/lib/*
 /usr/include/bcc/*
 
 %files -n libbcc-examples

debian/libbcc.install

 usr/include/bcc/*
 usr/lib/x86_64-linux-gnu/libbcc*
-usr/share/bcc/include/*
-usr/share/bcc/lib/*

src/cc/CMakeLists.txt

@@ -32,10 +32,7 @@ if (CMAKE_COMPILER_IS_GNUCC)
   endif()
 endif()
 
-# tell the shared library where it is being installed so it can find shared header files
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBCC_INSTALL_PREFIX='\"${CMAKE_INSTALL_PREFIX}\"'")
-
-add_library(bcc SHARED bpf_common.cc bpf_module.cc libbpf.c perf_reader.c shared_table.cc)
+add_library(bcc SHARED bpf_common.cc bpf_module.cc libbpf.c perf_reader.c shared_table.cc exported_files.cc)
 set_target_properties(bcc PROPERTIES VERSION ${REVISION_LAST} SOVERSION 0)
 
 # BPF is still experimental otherwise it should be available
@@ -52,9 +49,6 @@ target_link_libraries(bcc b_frontend clang_frontend ${clang_libs} ${llvm_libs} L
 
 install(TARGETS bcc LIBRARY COMPONENT libbcc
   DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(DIRECTORY export/ COMPONENT libbcc
-  DESTINATION share/bcc/include/bcc
-  FILES_MATCHING PATTERN "*.h")
 install(FILES bpf_common.h bpf_module.h ../libbpf.h COMPONENT libbcc
   DESTINATION include/bcc)
 install(DIRECTORY compat/linux/ COMPONENT libbcc
@@ -62,8 +56,5 @@ install(DIRECTORY compat/linux/ COMPONENT libbcc
   FILES_MATCHING PATTERN "*.h")
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libbcc.pc COMPONENT libbcc
   DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
-install(DIRECTORY clang COMPONENT libbcc
-  DESTINATION share/bcc/lib
-  FILES_MATCHING PATTERN "*.h")
 
 add_subdirectory(frontends)

src/cc/bpf_module.cc

@@ -48,6 +48,7 @@
 #include "frontends/clang/loader.h"
 #include "frontends/clang/b_frontend_action.h"
 #include "bpf_module.h"
+#include "exported_files.h"
 #include "kbuild_helper.h"
 #include "shared_table.h"
 #include "libbpf.h"
@@ -331,9 +332,9 @@ int BPFModule::load_cfile(const string &file, bool in_memory, const char *cflags
 
 // Load in a pre-built list of functions into the initial Module object, then
 // build an ExecutionEngine.
-int BPFModule::load_includes(const string &tmpfile) {
+int BPFModule::load_includes(const string &text) {
   clang_loader_ = make_unique<ClangLoader>(&*ctx_, flags_);
-  if (clang_loader_->parse(&mod_, &tables_, tmpfile, false, nullptr, 0))
+  if (clang_loader_->parse(&mod_, &tables_, text, true, nullptr, 0))
     return -1;
   return 0;
 }
@@ -683,7 +684,12 @@ int BPFModule::load_b(const string &filename, const string &proto_filename) {
 
   // Helpers are inlined in the following file (C). Load the definitions and
   // pass the partially compiled module to the B frontend to continue with.
-  if (int rc = load_includes(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h"))
+  auto helpers_h = ExportedFiles::headers().find("/virtual/include/bcc/helpers.h");
+  if (helpers_h == ExportedFiles::headers().end()) {
+    fprintf(stderr, "Internal error: missing bcc/helpers.h");
+    return -1;
+  }
+  if (int rc = load_includes(helpers_h->second))
     return rc;
 
   b_loader_.reset(new BLoader(flags_));

src/cc/bpf_module.h

@@ -47,7 +47,7 @@ class BPFModule {
   llvm::Function * make_writer(llvm::Module *mod, llvm::Type *type);
   void dump_ir(llvm::Module &mod);
   int load_file_module(std::unique_ptr<llvm::Module> *mod, const std::string &file, bool in_memory);
-  int load_includes(const std::string &tmpfile);
+  int load_includes(const std::string &text);
   int load_cfile(const std::string &file, bool in_memory, const char *cflags[], int ncflags);
   int kbuild_flags(const char *uname_release, std::vector<std::string> *cflags);
   int run_pass_manager(llvm::Module &mod);

src/cc/clang/include/stdarg.h

+R"********(
 /*===---- stdarg.h - Variable argument handling ----------------------------===
  *
  * Copyright (c) 2008 Eli Friedman
@@ -50,3 +51,4 @@ typedef __builtin_va_list va_list;
 typedef __builtin_va_list __gnuc_va_list;
 
 #endif /* __STDARG_H */
+)********"

src/cc/export/helpers.h

+R"********(
 /*
  * Copyright (c) 2015 PLUMgrid, Inc.
  *
@@ -392,3 +393,4 @@ int bpf_num_cpus() asm("llvm.bpf.extra");
 #define lock_xadd(ptr, val) ((void)__sync_fetch_and_add(ptr, val))
 
 #endif
+)********"

src/cc/export/proto.h

+R"********(
 /*
  * Copyright (c) 2015 PLUMgrid, Inc.
  *
@@ -95,3 +96,4 @@ struct vxlan_t {
   unsigned int key:24;
   unsigned int rsv4:8;
 } BPF_PACKET_HEADER;
+)********"

src/cc/exported_files.cc

+/*
+ * Copyright (c) 2016 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "exported_files.h"
+
+using std::map;
+using std::string;
+
+namespace ebpf {
+
+// c++11 feature for including raw string literals
+// see http://www.stroustrup.com/C++11FAQ.html#raw-strings
+
+map<string, const char *> ExportedFiles::headers_ = {
+  {
+    "/virtual/include/bcc/proto.h",
+    #include "export/proto.h"
+  },
+  {
+    "/virtual/include/bcc/helpers.h",
+    #include "export/helpers.h"
+  },
+  {
+    "/virtual/lib/clang/include/stdarg.h",
+    #include "clang/include/stdarg.h"
+  },
+};
+
+}

src/cc/exported_files.h

+/*
+ * Copyright (c) 2016 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <string>
+
+namespace ebpf {
+
+class ExportedFiles {
+  static std::map<std::string, const char *> headers_;
+ public:
+  static const std::map<std::string, const char *> & headers() { return headers_; }
+};
+
+}

src/cc/frontends/clang/kbuild_helper.cc

@@ -57,7 +57,7 @@ int KBuildHelper::get_flags(const char *uname_machine, vector<string> *cflags) {
 
   cflags->push_back("-nostdinc");
   cflags->push_back("-isystem");
-  cflags->push_back(BCC_INSTALL_PREFIX "/share/bcc/lib/clang/include");
+  cflags->push_back("/virtual/lib/clang/include");
   cflags->push_back("-I./arch/"+arch+"/include");
   cflags->push_back("-Iarch/"+arch+"/include/generated/uapi");
   cflags->push_back("-Iarch/"+arch+"/include/generated");

src/cc/frontends/clang/loader.cc

@@ -47,6 +47,7 @@
 
 #include "common.h"
 #include "exception.h"
+#include "exported_files.h"
 #include "kbuild_helper.h"
 #include "b_frontend_action.h"
 #include "loader.h"
@@ -58,9 +59,16 @@ using std::vector;
 
 namespace ebpf {
 
+map<string, unique_ptr<llvm::MemoryBuffer>> ClangLoader::remapped_files_;
+
 ClangLoader::ClangLoader(llvm::LLVMContext *ctx, unsigned flags)
     : ctx_(ctx), flags_(flags)
-{}
+{
+  if (remapped_files_.empty()) {
+    for (auto f : ExportedFiles::headers())
+      remapped_files_[f.first] = llvm::MemoryBuffer::getMemBuffer(f.second);
+  }
+}
 
 ClangLoader::~ClangLoader() {}
 
@@ -68,6 +76,10 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
                        const string &file, bool in_memory, const char *cflags[], int ncflags) {
   using namespace clang;
 
+  string main_path = "/virtual/main.c";
+  string proto_path = "/virtual/include/bcc/proto.h";
+  string helpers_path = "/virtual/include/bcc/helpers.h";
+  unique_ptr<llvm::MemoryBuffer> main_buf;
   struct utsname un;
   uname(&un);
   char kdir[256];
@@ -80,7 +92,8 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
 
   string abs_file;
   if (in_memory) {
-    abs_file = "<bcc-memory-buffer>";
+    abs_file = main_path;
+    main_buf = llvm::MemoryBuffer::getMemBuffer(file);
   } else {
     if (file.substr(0, 1) == "/")
       abs_file = file;
@@ -98,9 +111,9 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
   if (kbuild_helper.get_flags(un.machine, &kflags))
     return -1;
   kflags.push_back("-include");
-  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h");
-  kflags.push_back("-I");
-  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include");
+  kflags.push_back("/virtual/include/bcc/helpers.h");
+  kflags.push_back("-isystem");
+  kflags.push_back("/virtual/include");
   for (auto it = kflags.begin(); it != kflags.end(); ++it)
     flags_cstr.push_back(it->c_str());
   if (cflags) {
@@ -156,11 +169,17 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
                                           const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
     return -1;
 
+  // This option instructs clang whether or not to free the file buffers that we
+  // give to it. Since the embedded header files should be copied fewer times
+  // and reused if possible, set this flag to true.
+  invocation1->getPreprocessorOpts().RetainRemappedFileBuffers = true;
+  for (const auto &f : remapped_files_)
+    invocation1->getPreprocessorOpts().addRemappedFile(f.first, &*f.second);
+
   if (in_memory) {
-    invocation1->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
-                                                       llvm::MemoryBuffer::getMemBuffer(file).release());
+    invocation1->getPreprocessorOpts().addRemappedFile(main_path, &*main_buf);
     invocation1->getFrontendOpts().Inputs.clear();
-    invocation1->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
+    invocation1->getFrontendOpts().Inputs.push_back(FrontendInputFile(main_path, IK_C));
   }
   invocation1->getFrontendOpts().DisableFree = false;
 
@@ -174,6 +193,7 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
   BFrontendAction bact(os, flags_);
   if (!compiler1.ExecuteAction(bact))
     return -1;
+  unique_ptr<llvm::MemoryBuffer> out_buf = llvm::MemoryBuffer::getMemBuffer(out_str);
   // this contains the open FDs
   *tables = bact.take_tables();
 
@@ -183,10 +203,12 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
                                           const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
     return -1;
   CompilerInstance compiler2;
-  invocation2->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
-                                                     llvm::MemoryBuffer::getMemBuffer(out_str).release());
+  invocation2->getPreprocessorOpts().RetainRemappedFileBuffers = true;
+  for (const auto &f : remapped_files_)
+    invocation2->getPreprocessorOpts().addRemappedFile(f.first, &*f.second);
+  invocation2->getPreprocessorOpts().addRemappedFile(main_path, &*out_buf);
   invocation2->getFrontendOpts().Inputs.clear();
-  invocation2->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
+  invocation2->getFrontendOpts().Inputs.push_back(FrontendInputFile(main_path, IK_C));
   invocation2->getFrontendOpts().DisableFree = false;
   // suppress warnings in the 2nd pass, but bail out on errors (our fault)
   invocation2->getDiagnosticOpts().IgnoreWarnings = true;

src/cc/frontends/clang/loader.h

@@ -23,6 +23,7 @@
 namespace llvm {
 class Module;
 class LLVMContext;
+class MemoryBuffer;
 }
 
 namespace ebpf {
@@ -41,6 +42,7 @@ class ClangLoader {
   int parse(std::unique_ptr<llvm::Module> *mod, std::unique_ptr<std::vector<TableDesc>> *tables,
             const std::string &file, bool in_memory, const char *cflags[], int ncflags);
  private:
+  static std::map<std::string, std::unique_ptr<llvm::MemoryBuffer>> remapped_files_;
   llvm::LLVMContext *ctx_;
   unsigned flags_;
 };