Determine kernel dirs at runtime (fix #743)

As some module build directories split headers between `source/` and
`build/` and this varies between distributions, determine the location
at runtime.

CMakeLists.txt

@@ -49,24 +49,6 @@ endif()
 
 find_package(LibElf REQUIRED)
 
-# Set to non-zero if system installs kernel headers with split source and build
-# directories in /lib/modules/`uname -r`/. This is the case for debian and
-# suse, to the best of my knowledge.
-
-if(EXISTS "${ROOT}/lib/modules/${CMAKE_SYSTEM_VERSION}/build" AND EXISTS "${ROOT}/lib/modules/${CMAKE_SYSTEM_VERSION}/source")
-  message("Building for a split build/source kernel headers package")
-  set(BCC_KERNEL_HAS_SOURCE_DIR 1)
-  set(BCC_KERNEL_MODULES_SUFFIX "source")
-else()
-  set(BCC_KERNEL_HAS_SOURCE_DIR 0)
-endif()
-
-# Similar to above, set to custom value if kernel headers in
-# /lib/modules/`uname -r` sit in a different location than build/.
-if(NOT DEFINED BCC_KERNEL_MODULES_SUFFIX)
-  set(BCC_KERNEL_MODULES_SUFFIX "build")
-endif()
-
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
 
 # As reported in issue #735, GCC 6 has some behavioral problems when

src/cc/frontends/clang/CMakeLists.txt

@@ -2,6 +2,4 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DKERNEL_MODULES_DIR='\"${BCC_KERNEL_MODULES_DIR}\"'")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DKERNEL_MODULES_SUFFIX='\"${BCC_KERNEL_MODULES_SUFFIX}\"'")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DKERNEL_HAS_SOURCE_DIR=${BCC_KERNEL_HAS_SOURCE_DIR}")
 add_library(clang_frontend loader.cc b_frontend_action.cc tp_frontend_action.cc kbuild_helper.cc)

src/cc/frontends/clang/kbuild_helper.cc

@@ -22,7 +22,8 @@ namespace ebpf {
 using std::string;
 using std::vector;
 
-KBuildHelper::KBuildHelper(const std::string &kdir) : kdir_(kdir) {
+KBuildHelper::KBuildHelper(const std::string &kdir, bool has_source_dir) : kdir_(kdir),
+                                                                           has_source_dir_(has_source_dir) {
 }
 
 // read the flags from cache or learn
@@ -60,7 +61,7 @@ int KBuildHelper::get_flags(const char *uname_machine, vector<string> *cflags) {
   cflags->push_back("/virtual/lib/clang/include");
 
   // some module build directories split headers between source/ and build/
-  if (KERNEL_HAS_SOURCE_DIR) {
+  if (has_source_dir_) {
     cflags->push_back("-I" + kdir_ + "/build/arch/"+arch+"/include");
     cflags->push_back("-I" + kdir_ + "/build/arch/"+arch+"/include/generated/uapi");
     cflags->push_back("-I" + kdir_ + "/build/arch/"+arch+"/include/generated");

src/cc/frontends/clang/kbuild_helper.h

@@ -92,10 +92,11 @@ class TmpDir {
 //  case we eventually support non-root user programs, cache in $HOME.
 class KBuildHelper {
  public:
-  explicit KBuildHelper(const std::string &kdir);
+  explicit KBuildHelper(const std::string &kdir, bool has_source_dir);
   int get_flags(const char *uname_machine, std::vector<std::string> *cflags);
  private:
   std::string kdir_;
+  bool has_source_dir_;
 };
 
 }  // namespace ebpf

src/cc/frontends/clang/loader.cc

@@ -20,11 +20,14 @@
 #include <fcntl.h>
 #include <ftw.h>
 #include <map>
+#include <stdlib.h>
 #include <stdio.h>
 #include <string>
 #include <sys/stat.h>
+#include <sys/types.h>
 #include <sys/utsname.h>
 #include <unistd.h>
+#include <utility>
 #include <vector>
 #include <linux/bpf.h>
 
@@ -74,6 +77,33 @@ ClangLoader::ClangLoader(llvm::LLVMContext *ctx, unsigned flags)
 
 ClangLoader::~ClangLoader() {}
 
+namespace
+{
+
+bool is_dir(const string& path)
+{
+  struct stat buf;
+
+  if (::stat (path.c_str (), &buf) < 0)
+    return false;
+
+  return S_ISDIR(buf.st_mode);
+}
+
+std::pair<bool, string> get_kernel_path_info(const string kdir)
+{
+  if (is_dir(kdir + "/build") && is_dir(kdir + "/source"))
+    return std::make_pair (true, "source");
+
+  const char* suffix_from_env = ::getenv("BCC_KERNEL_MODULES_SUFFIX");
+  if (suffix_from_env)
+    return std::make_pair(false, string(suffix_from_env));
+
+  return std::make_pair(false, "build");
+}
+
+}
+
 int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDesc>> *tables,
                        const string &file, bool in_memory, const char *cflags[], int ncflags) {
   using namespace clang;
@@ -83,9 +113,10 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
   struct utsname un;
   uname(&un);
   string kdir = string(KERNEL_MODULES_DIR) + "/" + un.release;
+  auto kernel_path_info = get_kernel_path_info (kdir);
 
   // clang needs to run inside the kernel dir
-  DirStack dstack(kdir + "/" + KERNEL_MODULES_SUFFIX);
+  DirStack dstack(kdir + "/" + kernel_path_info.second);
   if (!dstack.ok())
     return -1;
 
@@ -108,7 +139,7 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
                                    "-fno-color-diagnostics",
                                    "-x", "c", "-c", abs_file.c_str()});
 
-  KBuildHelper kbuild_helper(kdir);
+  KBuildHelper kbuild_helper(kdir, kernel_path_info.first);
   vector<string> kflags;
   if (kbuild_helper.get_flags(un.machine, &kflags))
     return -1;