attempt to compile with system bpf.h if default compile failed

Currently, bcc uses its own version of bpf.h which tries to
sync with upstream header regularly. If the host bpf.h version
is lower, bcc can still compile as some bcc codes may requires
a higher version of bpf.h.

Such an approach does have a drawback. Suppose service A,
statically linked with bcc, runs on kernel version X.
Now, the kernel upgrades to version Y. After kernel upgrade/reboot,
service A may not be able to compile since old bcc bpf.h
may not align with the new kernel headers.
For such cases, new version of service A needs rollout.

This patch addresses this issue by attempting a second
compilation using system bpf.h instead. The feature is not on
by default. To enable it, pass -DBCC_BACKUP_COMPILE=1
in cmake setup stage.
Signed-off-by: Yonghong Song <yhs@fb.com>

src/cc/frontends/clang/CMakeLists.txt

@@ -5,5 +5,8 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DKERNEL_MODULES_DIR='\"${BCC_KERNEL_MOD
 if(DEFINED BCC_CUR_CPU_IDENTIFIER)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCUR_CPU_IDENTIFIER='\"${BCC_CUR_CPU_IDENTIFIER}\"'")
 endif()
+if(DEFINED BCC_BACKUP_COMPILE)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBCC_BACKUP_COMPILE='${BCC_BACKUP_COMPILE}'")
+endif()
 
 add_library(clang_frontend STATIC loader.cc b_frontend_action.cc tp_frontend_action.cc kbuild_helper.cc ../../common.cc)

src/cc/frontends/clang/loader.cc

@@ -108,8 +108,6 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, TableStorage &ts,
                        const string &file, bool in_memory, const char *cflags[],
                        int ncflags, const std::string &id, FuncSource &func_src,
                        std::string &mod_src) {
-  using namespace clang;
-
   string main_path = "/virtual/main.c";
   unique_ptr<llvm::MemoryBuffer> main_buf;
   struct utsname un;
@@ -155,22 +153,60 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, TableStorage &ts,
     return -1;
   if (flags_ & DEBUG_SOURCE)
     flags_cstr.push_back("-g");
-  kflags.push_back("-include");
-  kflags.push_back("/virtual/include/bcc/bpf.h");
-  kflags.push_back("-include");
-  kflags.push_back("/virtual/include/bcc/helpers.h");
-  kflags.push_back("-isystem");
-  kflags.push_back("/virtual/include");
   for (auto it = kflags.begin(); it != kflags.end(); ++it)
     flags_cstr.push_back(it->c_str());
+
+  vector<const char *> flags_cstr_rem;
+  flags_cstr_rem.push_back("-include");
+  flags_cstr_rem.push_back("/virtual/include/bcc/helpers.h");
+  flags_cstr_rem.push_back("-isystem");
+  flags_cstr_rem.push_back("/virtual/include");
   if (cflags) {
     for (auto i = 0; i < ncflags; ++i)
-      flags_cstr.push_back(cflags[i]);
+      flags_cstr_rem.push_back(cflags[i]);
   }
 #ifdef CUR_CPU_IDENTIFIER
   string cur_cpu_flag = string("-DCUR_CPU_IDENTIFIER=") + CUR_CPU_IDENTIFIER;
-  flags_cstr.push_back(cur_cpu_flag.c_str());
+  flags_cstr_rem.push_back(cur_cpu_flag.c_str());
+#endif
+
+  if (do_compile(mod, ts, in_memory, flags_cstr, flags_cstr_rem, main_path,
+                 main_buf, id, func_src, mod_src, true)) {
+#if BCC_BACKUP_COMPILE != 1
+    return -1;
+#else
+    // try one more time to compile with system bpf.h
+    llvm::errs() << "WARNING: compilation failure, trying with system bpf.h\n";
+
+    ts.DeletePrefix(Path({id}));
+    func_src.clear();
+    mod_src.clear();
+    if (do_compile(mod, ts, in_memory, flags_cstr, flags_cstr_rem, main_path,
+                   main_buf, id, func_src, mod_src, false))
+      return -1;
 #endif
+  }
+
+  return 0;
+}
+
+int ClangLoader::do_compile(unique_ptr<llvm::Module> *mod, TableStorage &ts,
+                            bool in_memory,
+                            const vector<const char *> &flags_cstr_in,
+                            const vector<const char *> &flags_cstr_rem,
+                            const std::string &main_path,
+                            const unique_ptr<llvm::MemoryBuffer> &main_buf,
+                            const std::string &id, FuncSource &func_src,
+                            std::string &mod_src, bool use_internal_bpfh) {
+  using namespace clang;
+
+  vector<const char *> flags_cstr = flags_cstr_in;
+  if (use_internal_bpfh) {
+    flags_cstr.push_back("-include");
+    flags_cstr.push_back("/virtual/include/bcc/bpf.h");
+  }
+  flags_cstr.insert(flags_cstr.end(), flags_cstr_rem.begin(),
+                    flags_cstr_rem.end());
 
   // set up the error reporting class
   IntrusiveRefCntPtr<DiagnosticOptions> diag_opts(new DiagnosticOptions());

src/cc/frontends/clang/loader.h

@@ -40,6 +40,7 @@ class FuncSource {
   std::map<std::string, SourceCode> funcs_;
  public:
   FuncSource() {}
+  void clear() { funcs_.clear(); }
   const char * src(const std::string& name);
   const char * src_rewritten(const std::string& name);
   void set_src(const std::string& name, const std::string& src);
@@ -55,6 +56,15 @@ class ClangLoader {
             int ncflags, const std::string &id, FuncSource &func_src,
             std::string &mod_src);
 
+ private:
+  int do_compile(std::unique_ptr<llvm::Module> *mod, TableStorage &ts,
+                 bool in_memory, const std::vector<const char *> &flags_cstr_in,
+                 const std::vector<const char *> &flags_cstr_rem,
+                 const std::string &main_path,
+                 const std::unique_ptr<llvm::MemoryBuffer> &main_buf,
+                 const std::string &id, FuncSource &func_src,
+                 std::string &mod_src, bool use_internal_bpfh);
+
  private:
   static std::map<std::string, std::unique_ptr<llvm::MemoryBuffer>> remapped_files_;
   llvm::LLVMContext *ctx_;