Add option to change kernel build search paths

Some linux distributions structure the /lib/modules directories
differently, causing complexities. Add cmake overrides to be able to
compile different behavior.

If your distro sets up `/lib/modules/$(uname -r)/{source,build}` with
header files split between the two (debian does this), then add
-DBCC_KERNEL_HAS_SOURCE_DIR=1 to the cmake command line.

If your distro just has something other than build/, but things are
still in one subdirectory, then add -DBCC_KERNEL_MODULES_SUFFIX=foo to
the cmake command line.

Also, fix one implicit declaration warning introduced by the new
bpf_get_stackid() helper.

Fixes: #397
Signed-off-by: Brenden Blanco <bblanco@plumgrid.com>

CMakeLists.txt

@@ -39,6 +39,22 @@ FOREACH(DIR ${LLVM_INCLUDE_DIRS})
   include_directories("${DIR}/../tools/clang/include")
 ENDFOREACH()
 
+# Set to non-zero if system installs kernel headers with split source and build
+# directories in /lib/modules/`uname -r`/. This is the case for debian and
+# suse, to the best of my knowledge.
+if(BCC_KERNEL_HAS_SOURCE_DIR)
+  set(BCC_KERNEL_HAS_SOURCE_DIR 1)
+  set(BCC_KERNEL_MODULES_SUFFIX "source")
+else()
+  set(BCC_KERNEL_HAS_SOURCE_DIR 0)
+endif()
+
+# Similar to above, set to custom value if kernel headers in
+# /lib/modules/`uname -r` sit in a different location than build/.
+if(NOT DEFINED BCC_KERNEL_MODULES_SUFFIX)
+  set(BCC_KERNEL_MODULES_SUFFIX "build")
+endif()
+
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall")
 endif()

src/cc/export/helpers.h

@@ -186,6 +186,10 @@ static int (*bpf_skb_load_bytes)(void *ctx, int offset, void *to, u32 len) =
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
 static int (*bpf_get_stackid_)(void *ctx, void *map, u64 flags) =
   (void *) BPF_FUNC_get_stackid;
+static inline __attribute__((always_inline))
+int bpf_get_stackid(uintptr_t map, void *ctx, u64 flags) {
+  return bpf_get_stackid_(ctx, (void *)map, flags);
+}
 static int (*bpf_csum_diff)(void *from, u64 from_size, void *to, u64 to_size, u64 seed) =
   (void *) BPF_FUNC_csum_diff;
 #endif
@@ -378,11 +382,6 @@ int bpf_map_delete_elem_(uintptr_t map, void *key) {
   return bpf_map_delete_elem((void *)map, key);
 }
 
-static inline __attribute__((always_inline))
-int bpf_get_stackid(uintptr_t map, void *ctx, u64 flags) {
-  return bpf_get_stackid_(ctx, (void *)map, flags);
-}
-
 static inline __attribute__((always_inline))
 SEC("helpers")
 int bpf_l3_csum_replace_(void *ctx, u64 off, u64 from, u64 to, u64 flags) {

src/cc/frontends/clang/CMakeLists.txt

 # Copyright (c) PLUMgrid, Inc.
 # Licensed under the Apache License, Version 2.0 (the "License")
 
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DKERNEL_MODULES_SUFFIX='\"${BCC_KERNEL_MODULES_SUFFIX}\"'")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DKERNEL_HAS_SOURCE_DIR=${BCC_KERNEL_HAS_SOURCE_DIR}")
 add_library(clang_frontend loader.cc b_frontend_action.cc kbuild_helper.cc)

src/cc/frontends/clang/loader.cc

@@ -83,7 +83,7 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
   struct utsname un;
   uname(&un);
   char kdir[256];
-  snprintf(kdir, sizeof(kdir), "%s/%s/build", KERNEL_MODULES_DIR, un.release);
+  snprintf(kdir, sizeof(kdir), "%s/%s/%s", KERNEL_MODULES_DIR, un.release, KERNEL_MODULES_SUFFIX);
 
   // clang needs to run inside the kernel dir
   DirStack dstack(kdir);
@@ -110,6 +110,13 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, unique_ptr<vector<TableDes
   vector<string> kflags;
   if (kbuild_helper.get_flags(un.machine, &kflags))
     return -1;
+  // some module build directories split headers between source/ and build/
+  if (KERNEL_HAS_SOURCE_DIR) {
+    kflags.push_back("-isystem");
+    kflags.push_back(string(KERNEL_MODULES_DIR "/") + un.release + "/build/include");
+    kflags.push_back("-isystem");
+    kflags.push_back(string(KERNEL_MODULES_DIR "/") + un.release + "/build/include/generated/uapi");
+  }
   kflags.push_back("-include");
   kflags.push_back("/virtual/include/bcc/helpers.h");
   kflags.push_back("-isystem");