Merge pull request #107 from iovisor/bblanco_dev

Reorganize bpf_module into multiple frontend implementations

src/cc/CMakeLists.txt

@@ -2,18 +2,21 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+# to be removed
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/frontends/b)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/frontends/b)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/frontends/clang)
 include_directories(${LLVM_INCLUDE_DIRS})
 # todo: if check for kernel version
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/compat)
 add_definitions(${LLVM_DEFINITIONS})
 configure_file(libbpfprog.pc.in ${CMAKE_CURRENT_BINARY_DIR}/libbpfprog.pc @ONLY)
 
-BISON_TARGET(Parser parser.yy ${CMAKE_CURRENT_BINARY_DIR}/parser.yy.cc COMPILE_FLAGS "-o parser.yy.cc -v --debug")
-FLEX_TARGET(Lexer lexer.ll ${CMAKE_CURRENT_BINARY_DIR}/lexer.ll.cc COMPILE_FLAGS "--c++ --o lexer.ll.cc")
-ADD_FLEX_BISON_DEPENDENCY(Lexer Parser)
-
 # prune unused llvm static library stuff when linking into the new .so
 set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--exclude-libs=ALL")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
 
 # if gcc 4.9 or higher is used, static libstdc++ is a good option
 if (CMAKE_COMPILER_IS_GNUCC)
@@ -26,10 +29,7 @@ endif()
 # tell the shared library where it is being installed so it can find shared header files
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBCC_INSTALL_PREFIX='\"${CMAKE_INSTALL_PREFIX}\"'")
 
-add_library(bpfprog SHARED bpf_common.cc bpf_module.cc codegen_llvm.cc
-  node.cc parser.cc printer.cc type_check.cc libbpf.c b_frontend_action.cc
-  kbuild_helper.cc
-  ${BISON_Parser_OUTPUTS} ${FLEX_Lexer_OUTPUTS})
+add_library(bpfprog SHARED bpf_common.cc bpf_module.cc libbpf.c)
 
 # BPF is still experimental otherwise it should be available
 #llvm_map_components_to_libnames(llvm_libs bpf mcjit irreader passes)
@@ -40,7 +40,7 @@ set(clang_libs ${libclangFrontend} ${libclangSerialization} ${libclangDriver} ${
   ${libclangAST} ${libclangLex} ${libclangBasic})
 
 # Link against LLVM libraries
-target_link_libraries(bpfprog ${clang_libs} ${llvm_libs} LLVMBPFCodeGen)
+target_link_libraries(bpfprog b_frontend clang_frontend ${clang_libs} ${llvm_libs} LLVMBPFCodeGen)
 
 install(TARGETS bpfprog LIBRARY DESTINATION lib${LIBSUFFIX})
 install(DIRECTORY export/ DESTINATION share/bcc/include/bcc
@@ -49,3 +49,5 @@ install(FILES bpf_common.h ../libbpf.h DESTINATION include/bcc)
 install(DIRECTORY compat/linux/ DESTINATION include/bcc/compat/linux
   FILES_MATCHING PATTERN "*.h")
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libbpfprog.pc DESTINATION lib${LIBSUFFIX}/pkgconfig)
+
+add_subdirectory(frontends)

src/cc/bpf_common.cc

@@ -17,16 +17,25 @@
 #include "cc/bpf_common.h"
 
 extern "C" {
-void * bpf_module_create(const char *filename, const char *proto_filename, unsigned flags) {
+void * bpf_module_create_b(const char *filename, const char *proto_filename, unsigned flags) {
   auto mod = new ebpf::BPFModule(flags);
-  if (mod->load(filename, proto_filename) != 0) {
+  if (mod->load_b(filename, proto_filename) != 0) {
     delete mod;
     return nullptr;
   }
   return mod;
 }
 
-void * bpf_module_create_from_string(const char *text, unsigned flags) {
+void * bpf_module_create_c(const char *filename, unsigned flags) {
+  auto mod = new ebpf::BPFModule(flags);
+  if (mod->load_c(filename) != 0) {
+    delete mod;
+    return nullptr;
+  }
+  return mod;
+}
+
+void * bpf_module_create_c_from_string(const char *text, unsigned flags) {
   auto mod = new ebpf::BPFModule(flags);
   if (mod->load_string(text) != 0) {
     delete mod;

src/cc/bpf_common.h

@@ -24,8 +24,9 @@
 extern "C" {
 #endif
 
-void * bpf_module_create(const char *filename, const char *proto_filename, unsigned flags);
-void * bpf_module_create_from_string(const char *text, unsigned flags);
+void * bpf_module_create_b(const char *filename, const char *proto_filename, unsigned flags);
+void * bpf_module_create_c(const char *filename, unsigned flags);
+void * bpf_module_create_c_from_string(const char *text, unsigned flags);
 void bpf_module_destroy(void *program);
 char * bpf_module_license(void *program);
 unsigned bpf_module_kern_version(void *program);

src/cc/bpf_module.cc

@@ -25,21 +25,6 @@
 #include <vector>
 #include <linux/bpf.h>
 
-#include <clang/Basic/FileManager.h>
-#include <clang/Basic/TargetInfo.h>
-#include <clang/CodeGen/BackendUtil.h>
-#include <clang/CodeGen/CodeGenAction.h>
-#include <clang/Driver/Compilation.h>
-#include <clang/Driver/Driver.h>
-#include <clang/Driver/Job.h>
-#include <clang/Driver/Tool.h>
-#include <clang/Frontend/CompilerInstance.h>
-#include <clang/Frontend/CompilerInvocation.h>
-#include <clang/Frontend/FrontendActions.h>
-#include <clang/Frontend/FrontendDiagnostic.h>
-#include <clang/Frontend/TextDiagnosticPrinter.h>
-#include <clang/FrontendTool/Utils.h>
-
 #include <llvm/ADT/STLExtras.h>
 #include <llvm/ExecutionEngine/MCJIT.h>
 #include <llvm/ExecutionEngine/SectionMemoryManager.h>
@@ -57,10 +42,9 @@
 #include <llvm/Transforms/IPO/PassManagerBuilder.h>
 
 #include "exception.h"
-#include "parser.h"
-#include "type_check.h"
-#include "codegen_llvm.h"
-#include "b_frontend_action.h"
+#include "frontends/b/loader.h"
+#include "frontends/clang/loader.h"
+#include "frontends/clang/b_frontend_action.h"
 #include "bpf_module.h"
 #include "kbuild_helper.h"
 #include "libbpf.h"
@@ -123,134 +107,11 @@ BPFModule::~BPFModule() {
   ctx_.reset();
 }
 
-int BPFModule::load_file_module(unique_ptr<llvm::Module> *mod, const string &file, bool in_memory) {
-  using namespace clang;
-
-  struct utsname un;
-  uname(&un);
-  char kdir[256];
-  snprintf(kdir, sizeof(kdir), "%s/%s/build", KERNEL_MODULES_DIR, un.release);
-
-  // clang needs to run inside the kernel dir
-  DirStack dstack(kdir);
-  if (!dstack.ok())
-    return -1;
-
-  string abs_file;
-  if (in_memory) {
-    abs_file = "<bcc-memory-buffer>";
-  } else {
-    if (file.substr(0, 1) == "/")
-      abs_file = file;
-    else
-      abs_file = string(dstack.cwd()) + "/" + file;
-  }
-
-  vector<const char *> flags_cstr({"-O0", "-emit-llvm", "-I", dstack.cwd(),
-                                   "-Wno-deprecated-declarations",
-                                   "-x", "c", "-c", abs_file.c_str()});
-
-  KBuildHelper kbuild_helper;
-  vector<string> kflags;
-  if (kbuild_helper.get_flags(un.release, &kflags))
-    return -1;
-  kflags.push_back("-include");
-  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h");
-  kflags.push_back("-I");
-  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include");
-  for (auto it = kflags.begin(); it != kflags.end(); ++it)
-    flags_cstr.push_back(it->c_str());
-
-  // set up the error reporting class
-  IntrusiveRefCntPtr<DiagnosticOptions> diag_opts(new DiagnosticOptions());
-  auto diag_client = new TextDiagnosticPrinter(llvm::errs(), &*diag_opts);
-
-  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  DiagnosticsEngine diags(DiagID, &*diag_opts, diag_client);
-
-  // set up the command line argument wrapper
-  driver::Driver drv("", "x86_64-unknown-linux-gnu", diags);
-  drv.setTitle("bcc-clang-driver");
-  drv.setCheckInputsExist(false);
-
-  unique_ptr<driver::Compilation> compilation(drv.BuildCompilation(flags_cstr));
-  if (!compilation)
-    return -1;
-
-  // expect exactly 1 job, otherwise error
-  const driver::JobList &jobs = compilation->getJobs();
-  if (jobs.size() != 1 || !isa<driver::Command>(*jobs.begin())) {
-    SmallString<256> msg;
-    llvm::raw_svector_ostream os(msg);
-    jobs.Print(os, "; ", true);
-    diags.Report(diag::err_fe_expected_compiler_job) << os.str();
-    return -1;
-  }
-
-  const driver::Command &cmd = cast<driver::Command>(*jobs.begin());
-  if (llvm::StringRef(cmd.getCreator().getName()) != "clang") {
-    diags.Report(diag::err_fe_expected_clang_command);
-    return -1;
-  }
-
-  // Initialize a compiler invocation object from the clang (-cc1) arguments.
-  const driver::ArgStringList &ccargs = cmd.getArguments();
-
-  // first pass
-  auto invocation1 = make_unique<CompilerInvocation>();
-  if (!CompilerInvocation::CreateFromArgs(*invocation1, const_cast<const char **>(ccargs.data()),
-                                          const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
-    return -1;
-
-  if (in_memory) {
-    invocation1->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
-                                                       llvm::MemoryBuffer::getMemBuffer(file).release());
-    invocation1->getFrontendOpts().Inputs.clear();
-    invocation1->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
-  }
-  invocation1->getFrontendOpts().DisableFree = false;
-
-  CompilerInstance compiler1;
-  compiler1.setInvocation(invocation1.release());
-  compiler1.createDiagnostics();
-
-  // capture the rewritten c file
-  string out_str;
-  llvm::raw_string_ostream os(out_str);
-  BFrontendAction bact(os);
-  if (!compiler1.ExecuteAction(bact))
-    return -1;
-  // this contains the open FDs
-  tables_ = bact.take_tables();
-
-  // second pass, clear input and take rewrite buffer
-  auto invocation2 = make_unique<CompilerInvocation>();
-  if (!CompilerInvocation::CreateFromArgs(*invocation2, const_cast<const char **>(ccargs.data()),
-                                          const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
-    return -1;
-  CompilerInstance compiler2;
-  invocation2->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
-                                                     llvm::MemoryBuffer::getMemBuffer(out_str).release());
-  invocation2->getFrontendOpts().Inputs.clear();
-  invocation2->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
-  invocation2->getFrontendOpts().DisableFree = false;
-  // suppress warnings in the 2nd pass, but bail out on errors (our fault)
-  invocation2->getDiagnosticOpts().IgnoreWarnings = true;
-  compiler2.setInvocation(invocation2.release());
-  compiler2.createDiagnostics();
-
-  EmitLLVMOnlyAction ir_act(&*ctx_);
-  if (!compiler2.ExecuteAction(ir_act))
-    return -1;
-  *mod = ir_act.takeModule();
-
-  return 0;
-}
-
 // load an entire c file as a module
 int BPFModule::load_cfile(const string &file, bool in_memory) {
+  clang_loader_ = make_unique<ClangLoader>(&*ctx_);
   unique_ptr<Module> mod;
-  if (load_file_module(&mod, file, in_memory))
+  if (clang_loader_->parse(&mod, &tables_, file, in_memory))
     return -1;
   mod_ = &*mod;
 
@@ -280,8 +141,9 @@ int BPFModule::load_cfile(const string &file, bool in_memory) {
 // Load in a pre-built list of functions into the initial Module object, then
 // build an ExecutionEngine.
 int BPFModule::load_includes(const string &tmpfile) {
+  clang_loader_ = make_unique<ClangLoader>(&*ctx_);
   unique_ptr<Module> mod;
-  if (load_file_module(&mod, tmpfile, false))
+  if (clang_loader_->parse(&mod, &tables_, tmpfile, false))
     return -1;
   mod_ = &*mod;
 
@@ -311,46 +173,6 @@ void BPFModule::dump_ir() {
   PM.run(*mod_);
 }
 
-int BPFModule::parse() {
-  int rc;
-
-  proto_parser_ = make_unique<ebpf::cc::Parser>(proto_filename_);
-  rc = proto_parser_->parse();
-  if (rc) {
-    fprintf(stderr, "In file: %s\n", filename_.c_str());
-    return rc;
-  }
-
-  parser_ = make_unique<ebpf::cc::Parser>(filename_);
-  rc = parser_->parse();
-  if (rc) {
-    fprintf(stderr, "In file: %s\n", filename_.c_str());
-    return rc;
-  }
-
-  //ebpf::cc::Printer printer(stderr);
-  //printer.visit(parser_->root_node_);
-
-  ebpf::cc::TypeCheck type_check(parser_->scopes_.get(), proto_parser_->scopes_.get(), parser_->pragmas_);
-  auto ret = type_check.visit(parser_->root_node_);
-  if (get<0>(ret) != 0 || get<1>(ret).size()) {
-    fprintf(stderr, "Type error @line=%d: %s\n", get<0>(ret), get<1>(ret).c_str());
-    return -1;
-  }
-
-  if (load_includes(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h") < 0)
-    return -1;
-
-  codegen_ = ebpf::make_unique<ebpf::cc::CodegenLLVM>(mod_, parser_->scopes_.get(), proto_parser_->scopes_.get());
-  ret = codegen_->visit(parser_->root_node_);
-  if (get<0>(ret) != 0 || get<1>(ret).size()) {
-    fprintf(stderr, "Codegen error @line=%d: %s\n", get<0>(ret), get<1>(ret).c_str());
-    return get<0>(ret);
-  }
-
-  return 0;
-}
-
 int BPFModule::finalize() {
   if (verifyModule(*mod_, &errs())) {
     if (flags_ & 1)
@@ -446,7 +268,7 @@ size_t BPFModule::num_tables() const {
 }
 
 int BPFModule::table_fd(const string &name) const {
-  int fd = codegen_ ? codegen_->get_table_fd(name) : -1;
+  int fd = b_loader_ ? b_loader_->get_table_fd(name) : -1;
   if (fd >= 0) return fd;
   auto table_it = tables_->find(name);
   if (table_it == tables_->end()) return -1;
@@ -469,7 +291,7 @@ const char * BPFModule::table_key_desc(size_t id) const {
 }
 
 const char * BPFModule::table_key_desc(const string &name) const {
-  if (codegen_) return nullptr;
+  if (b_loader_) return nullptr;
   auto table_it = tables_->find(name);
   if (table_it == tables_->end()) return nullptr;
   return table_it->second.key_desc.c_str();
@@ -481,39 +303,59 @@ const char * BPFModule::table_leaf_desc(size_t id) const {
 }
 
 const char * BPFModule::table_leaf_desc(const string &name) const {
-  if (codegen_) return nullptr;
+  if (b_loader_) return nullptr;
   auto table_it = tables_->find(name);
   if (table_it == tables_->end()) return nullptr;
   return table_it->second.leaf_desc.c_str();
 }
 
-int BPFModule::load(const string &filename, const string &proto_filename) {
+// load a B file, which comes in two parts
+int BPFModule::load_b(const string &filename, const string &proto_filename) {
   if (!sections_.empty()) {
     fprintf(stderr, "Program already initialized\n");
     return -1;
   }
-  filename_ = filename;
-  proto_filename_ = proto_filename;
-  if (proto_filename_.empty()) {
-    // direct load of .b file
-    if (int rc = load_cfile(filename_, false))
-      return rc;
-  } else {
-    // old lex .b file
-    if (int rc = parse())
-      return rc;
+  if (filename.empty() || proto_filename.empty()) {
+    fprintf(stderr, "Invalid filenames\n");
+    return -1;
   }
+
+  // Helpers are inlined in the following file (C). Load the definitions and
+  // pass the partially compiled module to the B frontend to continue with.
+  if (int rc = load_includes(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h"))
+    return rc;
+
+  b_loader_.reset(new BLoader);
+  if (int rc = b_loader_->parse(mod_, filename, proto_filename))
+    return rc;
+  if (int rc = finalize())
+    return rc;
+  return 0;
+}
+
+// load a C file
+int BPFModule::load_c(const string &filename) {
+  if (!sections_.empty()) {
+    fprintf(stderr, "Program already initialized\n");
+    return -1;
+  }
+  if (filename.empty()) {
+    fprintf(stderr, "Invalid filename\n");
+    return -1;
+  }
+  if (int rc = load_cfile(filename, false))
+    return rc;
   if (int rc = finalize())
     return rc;
   return 0;
 }
 
+// load a C text string
 int BPFModule::load_string(const string &text) {
   if (!sections_.empty()) {
     fprintf(stderr, "Program already initialized\n");
     return -1;
   }
-  filename_ = "<memory>";
   if (int rc = load_cfile(text, true))
     return rc;
 

src/cc/bpf_module.h

@@ -30,17 +30,14 @@ class Module;
 
 namespace ebpf {
 class BPFTable;
-
-namespace cc {
-class CodegenLLVM;
-class Parser;
-}
+class BLoader;
+class ClangLoader;
 
 class BPFModule {
  private:
   static const std::string FN_PREFIX;
   int init_engine();
-  int parse();
+  int parse(llvm::Module *mod);
   int finalize();
   void dump_ir();
   int load_file_module(std::unique_ptr<llvm::Module> *mod, const std::string &file, bool in_memory);
@@ -50,7 +47,8 @@ class BPFModule {
  public:
   BPFModule(unsigned flags);
   ~BPFModule();
-  int load(const std::string &filename, const std::string &proto_filename);
+  int load_b(const std::string &filename, const std::string &proto_filename);
+  int load_c(const std::string &filename);
   int load_string(const std::string &text);
   size_t num_functions() const;
   uint8_t * function_start(size_t id) const;
@@ -75,9 +73,8 @@ class BPFModule {
   std::unique_ptr<llvm::LLVMContext> ctx_;
   std::unique_ptr<llvm::ExecutionEngine> engine_;
   llvm::Module *mod_;
-  std::unique_ptr<ebpf::cc::Parser> parser_;
-  std::unique_ptr<ebpf::cc::Parser> proto_parser_;
-  std::unique_ptr<ebpf::cc::CodegenLLVM> codegen_;
+  std::unique_ptr<BLoader> b_loader_;
+  std::unique_ptr<ClangLoader> clang_loader_;
   std::map<std::string, std::tuple<uint8_t *, uintptr_t>> sections_;
   std::unique_ptr<std::map<std::string, BPFTable>> tables_;
   std::vector<std::string> table_names_;

src/cc/common.h

+/*
+ * Copyright (c) 2015 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <tuple>
+
+namespace ebpf {
+
+template <class T, class... Args>
+typename std::enable_if<!std::is_array<T>::value, std::unique_ptr<T>>::type
+make_unique(Args &&... args) {
+  return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
+}
+
+typedef std::tuple<int, std::string> StatusTuple;
+
+}  // namespace ebpf

src/cc/export/helpers.h

@@ -74,7 +74,7 @@ static u64 (*bpf_get_current_uid_gid)(void) =
 	(void *) BPF_FUNC_get_current_uid_gid;
 static int (*bpf_get_current_comm)(void *buf, int buf_size) =
 	(void *) BPF_FUNC_get_current_comm;
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,3,0)
+//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,3,0)
 static u64 (*bpf_get_cgroup_classid)(void *ctx) =
         (void *) BPF_FUNC_get_cgroup_classid;
 static u64 (*bpf_skb_vlan_push)(void *ctx, u16 proto, u16 vlan_tci) =
@@ -88,7 +88,7 @@ static int (*bpf_skb_get_tunnel_key)(void *ctx, void *to, u32 size, u64 flags) =
   (void *) BPF_FUNC_skb_get_tunnel_key;
 static int (*bpf_skb_set_tunnel_key)(void *ctx, void *from, u32 size, u64 flags) =
   (void *) BPF_FUNC_skb_set_tunnel_key;
-#endif
+//#endif
 #endif
 
 /* llvm builtin functions that eBPF C program may use to

src/cc/frontends/CMakeLists.txt

+# Copyright (c) PLUMgrid, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+add_subdirectory(b)
+add_subdirectory(clang)

src/cc/frontends/b/CMakeLists.txt

+# Copyright (c) PLUMgrid, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+BISON_TARGET(Parser parser.yy ${CMAKE_CURRENT_BINARY_DIR}/parser.yy.cc COMPILE_FLAGS "-o parser.yy.cc -v --debug")
+FLEX_TARGET(Lexer lexer.ll ${CMAKE_CURRENT_BINARY_DIR}/lexer.ll.cc COMPILE_FLAGS "--c++ --o lexer.ll.cc")
+ADD_FLEX_BISON_DEPENDENCY(Lexer Parser)
+
+add_library(b_frontend loader.cc codegen_llvm.cc node.cc parser.cc printer.cc
+  type_check.cc ${BISON_Parser_OUTPUTS} ${FLEX_Lexer_OUTPUTS})

src/cc/codegen_llvm.cc → src/cc/frontends/b/codegen_llvm.cc

@@ -34,9 +34,9 @@
 #include <llvm/IR/Module.h>
 
 #include "exception.h"
-#include "cc/codegen_llvm.h"
-#include "cc/lexer.h"
-#include "cc/type_helper.h"
+#include "codegen_llvm.h"
+#include "lexer.h"
+#include "type_helper.h"
 #include "linux/bpf.h"
 #include "libbpf.h"
 

src/cc/codegen_llvm.h → src/cc/frontends/b/codegen_llvm.h

@@ -21,8 +21,8 @@
 #include <string>
 #include <set>
 
-#include "cc/node.h"
-#include "cc/scope.h"
+#include "node.h"
+#include "scope.h"
 
 namespace llvm {
 class AllocaInst;
@@ -35,6 +35,8 @@ class LLVMContext;
 class Module;
 class StructType;
 class SwitchInst;
+class Value;
+class GlobalVariable;
 }
 
 namespace ebpf {

src/cc/lexer.h → src/cc/frontends/b/lexer.h

@@ -27,7 +27,7 @@
 
 #include <iostream> // NOLINT
 #include <list>
-#include "cc/parser.yy.hh"
+#include "parser.yy.hh"
 
 namespace ebpf {
 namespace cc {

src/cc/lexer.ll → src/cc/frontends/b/lexer.ll

@@ -15,14 +15,14 @@
  */
 
 %{
-#include "cc/lexer.h"
+#include "lexer.h"
 %}
 
 %option yylineno nodefault yyclass="Lexer" noyywrap c++ prefix="ebpfcc"
 %option never-interactive
 %{
 #include <string>
-#include "cc/parser.yy.hh"
+#include "parser.yy.hh"
 std::string tmp_str_cc;
 %}
 

src/cc/frontends/b/loader.cc

+/*
+ * Copyright (c) 2015 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "parser.h"
+#include "type_check.h"
+#include "codegen_llvm.h"
+#include "loader.h"
+
+using std::get;
+using std::string;
+
+namespace ebpf {
+
+BLoader::BLoader() {
+}
+
+BLoader::~BLoader() {
+}
+
+int BLoader::parse(llvm::Module *mod, const string &filename, const string &proto_filename) {
+  int rc;
+
+  proto_parser_ = make_unique<ebpf::cc::Parser>(proto_filename);
+  rc = proto_parser_->parse();
+  if (rc) {
+    fprintf(stderr, "In file: %s\n", filename.c_str());
+    return rc;
+  }
+
+  parser_ = make_unique<ebpf::cc::Parser>(filename);
+  rc = parser_->parse();
+  if (rc) {
+    fprintf(stderr, "In file: %s\n", filename.c_str());
+    return rc;
+  }
+
+  //ebpf::cc::Printer printer(stderr);
+  //printer.visit(parser_->root_node_);
+
+  ebpf::cc::TypeCheck type_check(parser_->scopes_.get(), proto_parser_->scopes_.get(), parser_->pragmas_);
+  auto ret = type_check.visit(parser_->root_node_);
+  if (get<0>(ret) != 0 || get<1>(ret).size()) {
+    fprintf(stderr, "Type error @line=%d: %s\n", get<0>(ret), get<1>(ret).c_str());
+    return -1;
+  }
+
+  codegen_ = ebpf::make_unique<ebpf::cc::CodegenLLVM>(mod, parser_->scopes_.get(), proto_parser_->scopes_.get());
+  ret = codegen_->visit(parser_->root_node_);
+  if (get<0>(ret) != 0 || get<1>(ret).size()) {
+    fprintf(stderr, "Codegen error @line=%d: %s\n", get<0>(ret), get<1>(ret).c_str());
+    return get<0>(ret);
+  }
+
+  return 0;
+}
+
+int BLoader::get_table_fd(const string &name) const {
+  if (!codegen_) return -1;
+  return codegen_->get_table_fd(name);
+}
+
+}  // namespace ebpf

src/cc/frontends/b/loader.h

+/*
+ * Copyright (c) 2015 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+namespace llvm {
+class Module;
+}
+
+namespace ebpf {
+
+namespace cc {
+class Parser;
+class CodegenLLVM;
+}
+
+class BLoader {
+ public:
+  BLoader();
+  ~BLoader();
+  int parse(llvm::Module *mod, const std::string &filename, const std::string &proto_filename);
+  int get_table_fd(const std::string &name) const;
+ private:
+  std::unique_ptr<cc::Parser> parser_;
+  std::unique_ptr<cc::Parser> proto_parser_;
+  std::unique_ptr<cc::CodegenLLVM> codegen_;
+};
+
+}  // namespace ebpf

src/cc/node.cc → src/cc/frontends/b/node.cc

@@ -18,7 +18,7 @@
 #include <vector>
 #include <string>
 
-#include "cc/node.h"
+#include "node.h"
 
 namespace ebpf {
 namespace cc {

src/cc/node.h → src/cc/frontends/b/node.h

@@ -22,7 +22,9 @@
 #include <memory>
 #include <algorithm>
 #include <stdint.h>
-#include "cc/scope.h"
+
+#include "common.h"
+#include "scope.h"
 
 #define REVISION_MASK 0xfff
 #define MAJOR_VER_POS 22
@@ -41,14 +43,6 @@
 
 namespace ebpf {
 
-template <class T, class... Args>
-typename std::enable_if<!std::is_array<T>::value, std::unique_ptr<T>>::type
-make_unique(Args &&... args) {
-  return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
-}
-
-typedef std::tuple<int, std::string> StatusTuple;
-
 namespace cc {
 
 using std::unique_ptr;

src/cc/parser.cc → src/cc/frontends/b/parser.cc

@@ -17,8 +17,8 @@
 #include <algorithm>
 #include <assert.h>
 #include "exception.h"
-#include "cc/parser.h"
-#include "cc/type_helper.h"
+#include "parser.h"
+#include "type_helper.h"
 
 namespace ebpf {
 namespace cc {

src/cc/parser.h → src/cc/frontends/b/parser.h

@@ -17,9 +17,9 @@
 #pragma once
 
 #include <fstream> // NOLINT
-#include "cc/node.h"
-#include "cc/lexer.h"
-#include "cc/scope.h"
+#include "node.h"
+#include "lexer.h"
+#include "scope.h"
 
 namespace ebpf {
 namespace cc {

src/cc/parser.yy → src/cc/frontends/b/parser.yy

@@ -27,7 +27,7 @@
     #include <memory>
     #include <vector>
     #include <string>
-    #include "cc/node.h"
+    #include "node.h"
     // forward declaration
     namespace ebpf { namespace cc {
         class Lexer;
@@ -42,8 +42,8 @@
 }
 
 %{
-    #include "cc/node.h"
-    #include "cc/parser.h"
+    #include "node.h"
+    #include "parser.h"
     using std::unique_ptr;
     using std::vector;
     using std::string;
@@ -620,7 +620,7 @@ void ebpf::cc::BisonParser::error(const ebpf::cc::BisonParser::location_type &lo
     std::cerr << "Error: " << loc << " " << msg << std::endl;
 }
 
-#include "cc/lexer.h"
+#include "lexer.h"
 static int yylex(ebpf::cc::BisonParser::semantic_type *yylval,
                  ebpf::cc::BisonParser::location_type *yylloc,
                  ebpf::cc::Lexer &lexer) {

src/cc/printer.cc → src/cc/frontends/b/printer.cc

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#include "cc/printer.h"
-#include "cc/lexer.h"
+#include "printer.h"
+#include "lexer.h"
 #include "exception.h"
 
 namespace ebpf {

src/cc/printer.h → src/cc/frontends/b/printer.h

@@ -18,7 +18,7 @@
 
 #include <stdio.h>
 
-#include "cc/node.h"
+#include "node.h"
 
 namespace ebpf {
 namespace cc {

src/cc/type_check.cc → src/cc/frontends/b/type_check.cc

@@ -17,8 +17,8 @@
 #include <set>
 #include <algorithm>
 #include "exception.h"
-#include "cc/type_check.h"
-#include "cc/lexer.h"
+#include "type_check.h"
+#include "lexer.h"
 
 namespace ebpf {
 namespace cc {

src/cc/type_check.h → src/cc/frontends/b/type_check.h

@@ -18,8 +18,8 @@
 
 #include <vector>
 #include <string>
-#include "cc/node.h"
-#include "cc/scope.h"
+#include "node.h"
+#include "scope.h"
 
 namespace ebpf {
 namespace cc {

src/cc/frontends/clang/CMakeLists.txt

+# Copyright (c) PLUMgrid, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+add_library(clang_frontend loader.cc b_frontend_action.cc kbuild_helper.cc)

src/cc/frontends/clang/loader.cc

+/*
+ * Copyright (c) 2015 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <map>
+#include <string>
+#include <algorithm>
+#include <fcntl.h>
+#include <ftw.h>
+#include <map>
+#include <stdio.h>
+#include <string>
+#include <sys/stat.h>
+#include <sys/utsname.h>
+#include <unistd.h>
+#include <vector>
+#include <linux/bpf.h>
+
+#include <clang/Basic/FileManager.h>
+#include <clang/Basic/TargetInfo.h>
+#include <clang/CodeGen/BackendUtil.h>
+#include <clang/CodeGen/CodeGenAction.h>
+#include <clang/Driver/Compilation.h>
+#include <clang/Driver/Driver.h>
+#include <clang/Driver/Job.h>
+#include <clang/Driver/Tool.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendActions.h>
+#include <clang/Frontend/FrontendDiagnostic.h>
+#include <clang/Frontend/TextDiagnosticPrinter.h>
+#include <clang/FrontendTool/Utils.h>
+
+#include <llvm/IR/Module.h>
+
+#include "common.h"
+#include "exception.h"
+#include "kbuild_helper.h"
+#include "b_frontend_action.h"
+#include "loader.h"
+
+using std::map;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+namespace ebpf {
+
+ClangLoader::ClangLoader(llvm::LLVMContext *ctx)
+    : ctx_(ctx)
+{}
+
+ClangLoader::~ClangLoader() {}
+
+int ClangLoader::parse(unique_ptr<llvm::Module> *mod,
+                       unique_ptr<map<string, BPFTable>> *tables,
+                       const string &file, bool in_memory) {
+  using namespace clang;
+
+  struct utsname un;
+  uname(&un);
+  char kdir[256];
+  snprintf(kdir, sizeof(kdir), "%s/%s/build", KERNEL_MODULES_DIR, un.release);
+
+  // clang needs to run inside the kernel dir
+  DirStack dstack(kdir);
+  if (!dstack.ok())
+    return -1;
+
+  string abs_file;
+  if (in_memory) {
+    abs_file = "<bcc-memory-buffer>";
+  } else {
+    if (file.substr(0, 1) == "/")
+      abs_file = file;
+    else
+      abs_file = string(dstack.cwd()) + "/" + file;
+  }
+
+  vector<const char *> flags_cstr({"-O0", "-emit-llvm", "-I", dstack.cwd(),
+                                   "-Wno-deprecated-declarations",
+                                   "-x", "c", "-c", abs_file.c_str()});
+
+  KBuildHelper kbuild_helper;
+  vector<string> kflags;
+  if (kbuild_helper.get_flags(un.release, &kflags))
+    return -1;
+  kflags.push_back("-include");
+  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h");
+  kflags.push_back("-I");
+  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include");
+  for (auto it = kflags.begin(); it != kflags.end(); ++it)
+    flags_cstr.push_back(it->c_str());
+
+  // set up the error reporting class
+  IntrusiveRefCntPtr<DiagnosticOptions> diag_opts(new DiagnosticOptions());
+  auto diag_client = new TextDiagnosticPrinter(llvm::errs(), &*diag_opts);
+
+  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+  DiagnosticsEngine diags(DiagID, &*diag_opts, diag_client);
+
+  // set up the command line argument wrapper
+  driver::Driver drv("", "x86_64-unknown-linux-gnu", diags);
+  drv.setTitle("bcc-clang-driver");
+  drv.setCheckInputsExist(false);
+
+  unique_ptr<driver::Compilation> compilation(drv.BuildCompilation(flags_cstr));
+  if (!compilation)
+    return -1;
+
+  // expect exactly 1 job, otherwise error
+  const driver::JobList &jobs = compilation->getJobs();
+  if (jobs.size() != 1 || !isa<driver::Command>(*jobs.begin())) {
+    SmallString<256> msg;
+    llvm::raw_svector_ostream os(msg);
+    jobs.Print(os, "; ", true);
+    diags.Report(diag::err_fe_expected_compiler_job) << os.str();
+    return -1;
+  }
+
+  const driver::Command &cmd = cast<driver::Command>(*jobs.begin());
+  if (llvm::StringRef(cmd.getCreator().getName()) != "clang") {
+    diags.Report(diag::err_fe_expected_clang_command);
+    return -1;
+  }
+
+  // Initialize a compiler invocation object from the clang (-cc1) arguments.
+  const driver::ArgStringList &ccargs = cmd.getArguments();
+
+  // first pass
+  auto invocation1 = make_unique<CompilerInvocation>();
+  if (!CompilerInvocation::CreateFromArgs(*invocation1, const_cast<const char **>(ccargs.data()),
+                                          const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
+    return -1;
+
+  if (in_memory) {
+    invocation1->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
+                                                       llvm::MemoryBuffer::getMemBuffer(file).release());
+    invocation1->getFrontendOpts().Inputs.clear();
+    invocation1->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
+  }
+  invocation1->getFrontendOpts().DisableFree = false;
+
+  CompilerInstance compiler1;
+  compiler1.setInvocation(invocation1.release());
+  compiler1.createDiagnostics();
+
+  // capture the rewritten c file
+  string out_str;
+  llvm::raw_string_ostream os(out_str);
+  BFrontendAction bact(os);
+  if (!compiler1.ExecuteAction(bact))
+    return -1;
+  // this contains the open FDs
+  *tables = bact.take_tables();
+
+  // second pass, clear input and take rewrite buffer
+  auto invocation2 = make_unique<CompilerInvocation>();
+  if (!CompilerInvocation::CreateFromArgs(*invocation2, const_cast<const char **>(ccargs.data()),
+                                          const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
+    return -1;
+  CompilerInstance compiler2;
+  invocation2->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
+                                                     llvm::MemoryBuffer::getMemBuffer(out_str).release());
+  invocation2->getFrontendOpts().Inputs.clear();
+  invocation2->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
+  invocation2->getFrontendOpts().DisableFree = false;
+  // suppress warnings in the 2nd pass, but bail out on errors (our fault)
+  invocation2->getDiagnosticOpts().IgnoreWarnings = true;
+  compiler2.setInvocation(invocation2.release());
+  compiler2.createDiagnostics();
+
+  EmitLLVMOnlyAction ir_act(&*ctx_);
+  if (!compiler2.ExecuteAction(ir_act))
+    return -1;
+  *mod = ir_act.takeModule();
+
+  return 0;
+}
+
+
+}  // namespace ebpf

src/cc/frontends/clang/loader.h

+/*
+ * Copyright (c) 2015 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+namespace llvm {
+class Module;
+class LLVMContext;
+}
+
+namespace ebpf {
+
+class BPFTable;
+
+namespace cc {
+class Parser;
+class CodegenLLVM;
+}
+
+class ClangLoader {
+ public:
+  explicit ClangLoader(llvm::LLVMContext *ctx);
+  ~ClangLoader();
+  int parse(std::unique_ptr<llvm::Module> *mod,
+            std::unique_ptr<std::map<std::string, BPFTable>> *tables,
+            const std::string &file, bool in_memory);
+ private:
+  llvm::LLVMContext *ctx_;
+};
+
+}  // namespace ebpf

src/python/bpf/__init__.py

@@ -23,10 +23,12 @@ basestring = (unicode if sys.version_info[0] < 3 else str)
 lib = ct.CDLL("libbpfprog.so")
 
 # keep in sync with bpf_common.h
-lib.bpf_module_create.restype = ct.c_void_p
-lib.bpf_module_create.argtypes = [ct.c_char_p, ct.c_char_p, ct.c_uint]
-lib.bpf_module_create_from_string.restype = ct.c_void_p
-lib.bpf_module_create_from_string.argtypes = [ct.c_char_p, ct.c_uint]
+lib.bpf_module_create_b.restype = ct.c_void_p
+lib.bpf_module_create_b.argtypes = [ct.c_char_p, ct.c_char_p, ct.c_uint]
+lib.bpf_module_create_c.restype = ct.c_void_p
+lib.bpf_module_create_c.argtypes = [ct.c_char_p, ct.c_uint]
+lib.bpf_module_create_c_from_string.restype = ct.c_void_p
+lib.bpf_module_create_c_from_string.argtypes = [ct.c_char_p, ct.c_uint]
 lib.bpf_module_destroy.restype = None
 lib.bpf_module_destroy.argtypes = [ct.c_void_p]
 lib.bpf_module_license.restype = ct.c_char_p
@@ -171,12 +173,16 @@ class BPF(object):
         self.debug = debug
         self.funcs = {}
         if text:
-            self.module = lib.bpf_module_create_from_string(text.encode("ascii"), self.debug)
+            self.module = lib.bpf_module_create_c_from_string(text.encode("ascii"), self.debug)
         else:
             src_file = BPF._find_file(src_file)
             hdr_file = BPF._find_file(hdr_file)
-            self.module = lib.bpf_module_create(src_file.encode("ascii"),
-                    hdr_file.encode("ascii"), self.debug)
+            if src_file.endswith(".b"):
+                self.module = lib.bpf_module_create_b(src_file.encode("ascii"),
+                        hdr_file.encode("ascii"), self.debug)
+            else:
+                self.module = lib.bpf_module_create_c(src_file.encode("ascii"),
+                        self.debug)
 
         if self.module == None:
             raise Exception("Failed to compile BPF module %s" % src_file)