Move (some) frontend specific logic to the respective directories

Signed-off-by: Brenden Blanco <bblanco@plumgrid.com>

src/cc/bpf_common.cc

@@ -17,16 +17,25 @@
 #include "cc/bpf_common.h"
 
 extern "C" {
-void * bpf_module_create(const char *filename, const char *proto_filename, unsigned flags) {
+void * bpf_module_create_b(const char *filename, const char *proto_filename, unsigned flags) {
   auto mod = new ebpf::BPFModule(flags);
-  if (mod->load(filename, proto_filename) != 0) {
+  if (mod->load_b(filename, proto_filename) != 0) {
     delete mod;
     return nullptr;
   }
   return mod;
 }
 
-void * bpf_module_create_from_string(const char *text, unsigned flags) {
+void * bpf_module_create_c(const char *filename, unsigned flags) {
+  auto mod = new ebpf::BPFModule(flags);
+  if (mod->load_c(filename) != 0) {
+    delete mod;
+    return nullptr;
+  }
+  return mod;
+}
+
+void * bpf_module_create_c_from_string(const char *text, unsigned flags) {
   auto mod = new ebpf::BPFModule(flags);
   if (mod->load_string(text) != 0) {
     delete mod;

src/cc/bpf_common.h

@@ -24,8 +24,9 @@
 extern "C" {
 #endif
 
-void * bpf_module_create(const char *filename, const char *proto_filename, unsigned flags);
-void * bpf_module_create_from_string(const char *text, unsigned flags);
+void * bpf_module_create_b(const char *filename, const char *proto_filename, unsigned flags);
+void * bpf_module_create_c(const char *filename, unsigned flags);
+void * bpf_module_create_c_from_string(const char *text, unsigned flags);
 void bpf_module_destroy(void *program);
 char * bpf_module_license(void *program);
 unsigned bpf_module_kern_version(void *program);

src/cc/bpf_module.cc

@@ -57,9 +57,7 @@
 #include <llvm/Transforms/IPO/PassManagerBuilder.h>
 
 #include "exception.h"
-#include "frontends/b/parser.h"
-#include "frontends/b/type_check.h"
-#include "frontends/b/codegen_llvm.h"
+#include "frontends/b/loader.h"
 #include "frontends/clang/b_frontend_action.h"
 #include "bpf_module.h"
 #include "kbuild_helper.h"
@@ -311,46 +309,6 @@ void BPFModule::dump_ir() {
   PM.run(*mod_);
 }
 
-int BPFModule::parse() {
-  int rc;
-
-  proto_parser_ = make_unique<ebpf::cc::Parser>(proto_filename_);
-  rc = proto_parser_->parse();
-  if (rc) {
-    fprintf(stderr, "In file: %s\n", filename_.c_str());
-    return rc;
-  }
-
-  parser_ = make_unique<ebpf::cc::Parser>(filename_);
-  rc = parser_->parse();
-  if (rc) {
-    fprintf(stderr, "In file: %s\n", filename_.c_str());
-    return rc;
-  }
-
-  //ebpf::cc::Printer printer(stderr);
-  //printer.visit(parser_->root_node_);
-
-  ebpf::cc::TypeCheck type_check(parser_->scopes_.get(), proto_parser_->scopes_.get(), parser_->pragmas_);
-  auto ret = type_check.visit(parser_->root_node_);
-  if (get<0>(ret) != 0 || get<1>(ret).size()) {
-    fprintf(stderr, "Type error @line=%d: %s\n", get<0>(ret), get<1>(ret).c_str());
-    return -1;
-  }
-
-  if (load_includes(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h") < 0)
-    return -1;
-
-  codegen_ = ebpf::make_unique<ebpf::cc::CodegenLLVM>(mod_, parser_->scopes_.get(), proto_parser_->scopes_.get());
-  ret = codegen_->visit(parser_->root_node_);
-  if (get<0>(ret) != 0 || get<1>(ret).size()) {
-    fprintf(stderr, "Codegen error @line=%d: %s\n", get<0>(ret), get<1>(ret).c_str());
-    return get<0>(ret);
-  }
-
-  return 0;
-}
-
 int BPFModule::finalize() {
   if (verifyModule(*mod_, &errs())) {
     if (flags_ & 1)
@@ -446,7 +404,7 @@ size_t BPFModule::num_tables() const {
 }
 
 int BPFModule::table_fd(const string &name) const {
-  int fd = codegen_ ? codegen_->get_table_fd(name) : -1;
+  int fd = b_loader_ ? b_loader_->get_table_fd(name) : -1;
   if (fd >= 0) return fd;
   auto table_it = tables_->find(name);
   if (table_it == tables_->end()) return -1;
@@ -469,7 +427,7 @@ const char * BPFModule::table_key_desc(size_t id) const {
 }
 
 const char * BPFModule::table_key_desc(const string &name) const {
-  if (codegen_) return nullptr;
+  if (b_loader_) return nullptr;
   auto table_it = tables_->find(name);
   if (table_it == tables_->end()) return nullptr;
   return table_it->second.key_desc.c_str();
@@ -481,39 +439,59 @@ const char * BPFModule::table_leaf_desc(size_t id) const {
 }
 
 const char * BPFModule::table_leaf_desc(const string &name) const {
-  if (codegen_) return nullptr;
+  if (b_loader_) return nullptr;
   auto table_it = tables_->find(name);
   if (table_it == tables_->end()) return nullptr;
   return table_it->second.leaf_desc.c_str();
 }
 
-int BPFModule::load(const string &filename, const string &proto_filename) {
+// load a B file, which comes in two parts
+int BPFModule::load_b(const string &filename, const string &proto_filename) {
   if (!sections_.empty()) {
     fprintf(stderr, "Program already initialized\n");
     return -1;
   }
-  filename_ = filename;
-  proto_filename_ = proto_filename;
-  if (proto_filename_.empty()) {
-    // direct load of .b file
-    if (int rc = load_cfile(filename_, false))
-      return rc;
-  } else {
-    // old lex .b file
-    if (int rc = parse())
-      return rc;
+  if (filename.empty() || proto_filename.empty()) {
+    fprintf(stderr, "Invalid filenames\n");
+    return -1;
+  }
+
+  // Helpers are inlined in the following file (C). Load the definitions and
+  // pass the partially compiled module to the B frontend to continue with.
+  if (int rc = load_includes(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h"))
+    return rc;
+
+  b_loader_.reset(new BLoader);
+  if (int rc = b_loader_->parse(mod_, filename, proto_filename))
+    return rc;
+  if (int rc = finalize())
+    return rc;
+  return 0;
+}
+
+// load a C file
+int BPFModule::load_c(const string &filename) {
+  if (!sections_.empty()) {
+    fprintf(stderr, "Program already initialized\n");
+    return -1;
+  }
+  if (filename.empty()) {
+    fprintf(stderr, "Invalid filename\n");
+    return -1;
   }
+  if (int rc = load_cfile(filename, false))
+    return rc;
   if (int rc = finalize())
     return rc;
   return 0;
 }
 
+// load a C text string
 int BPFModule::load_string(const string &text) {
   if (!sections_.empty()) {
     fprintf(stderr, "Program already initialized\n");
     return -1;
   }
-  filename_ = "<memory>";
   if (int rc = load_cfile(text, true))
     return rc;
 

src/cc/bpf_module.h

@@ -30,17 +30,13 @@ class Module;
 
 namespace ebpf {
 class BPFTable;
-
-namespace cc {
-class CodegenLLVM;
-class Parser;
-}
+class BLoader;
 
 class BPFModule {
  private:
   static const std::string FN_PREFIX;
   int init_engine();
-  int parse();
+  int parse(llvm::Module *mod);
   int finalize();
   void dump_ir();
   int load_file_module(std::unique_ptr<llvm::Module> *mod, const std::string &file, bool in_memory);
@@ -50,7 +46,8 @@ class BPFModule {
  public:
   BPFModule(unsigned flags);
   ~BPFModule();
-  int load(const std::string &filename, const std::string &proto_filename);
+  int load_b(const std::string &filename, const std::string &proto_filename);
+  int load_c(const std::string &filename);
   int load_string(const std::string &text);
   size_t num_functions() const;
   uint8_t * function_start(size_t id) const;
@@ -75,9 +72,7 @@ class BPFModule {
   std::unique_ptr<llvm::LLVMContext> ctx_;
   std::unique_ptr<llvm::ExecutionEngine> engine_;
   llvm::Module *mod_;
-  std::unique_ptr<ebpf::cc::Parser> parser_;
-  std::unique_ptr<ebpf::cc::Parser> proto_parser_;
-  std::unique_ptr<ebpf::cc::CodegenLLVM> codegen_;
+  std::unique_ptr<BLoader> b_loader_;
   std::map<std::string, std::tuple<uint8_t *, uintptr_t>> sections_;
   std::unique_ptr<std::map<std::string, BPFTable>> tables_;
   std::vector<std::string> table_names_;

src/cc/export/helpers.h

@@ -74,7 +74,7 @@ static u64 (*bpf_get_current_uid_gid)(void) =
 	(void *) BPF_FUNC_get_current_uid_gid;
 static int (*bpf_get_current_comm)(void *buf, int buf_size) =
 	(void *) BPF_FUNC_get_current_comm;
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,3,0)
+//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,3,0)
 static u64 (*bpf_get_cgroup_classid)(void *ctx) =
         (void *) BPF_FUNC_get_cgroup_classid;
 static u64 (*bpf_skb_vlan_push)(void *ctx, u16 proto, u16 vlan_tci) =
@@ -88,7 +88,7 @@ static int (*bpf_skb_get_tunnel_key)(void *ctx, void *to, u32 size, u64 flags) =
   (void *) BPF_FUNC_skb_get_tunnel_key;
 static int (*bpf_skb_set_tunnel_key)(void *ctx, void *from, u32 size, u64 flags) =
   (void *) BPF_FUNC_skb_set_tunnel_key;
-#endif
+//#endif
 #endif
 
 /* llvm builtin functions that eBPF C program may use to

src/cc/frontends/b/CMakeLists.txt

@@ -8,5 +8,5 @@ BISON_TARGET(Parser parser.yy ${CMAKE_CURRENT_BINARY_DIR}/parser.yy.cc COMPILE_F
 FLEX_TARGET(Lexer lexer.ll ${CMAKE_CURRENT_BINARY_DIR}/lexer.ll.cc COMPILE_FLAGS "--c++ --o lexer.ll.cc")
 ADD_FLEX_BISON_DEPENDENCY(Lexer Parser)
 
-add_library(b_frontend codegen_llvm.cc node.cc parser.cc printer.cc
+add_library(b_frontend loader.cc codegen_llvm.cc node.cc parser.cc printer.cc
   type_check.cc ${BISON_Parser_OUTPUTS} ${FLEX_Lexer_OUTPUTS})

src/cc/frontends/b/codegen_llvm.h

@@ -35,6 +35,8 @@ class LLVMContext;
 class Module;
 class StructType;
 class SwitchInst;
+class Value;
+class GlobalVariable;
 }
 
 namespace ebpf {

src/cc/frontends/b/loader.cc

+/*
+ * Copyright (c) 2015 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "parser.h"
+#include "type_check.h"
+#include "codegen_llvm.h"
+#include "loader.h"
+
+using std::get;
+using std::string;
+
+namespace ebpf {
+
+BLoader::BLoader() {
+}
+
+BLoader::~BLoader() {
+}
+
+int BLoader::parse(llvm::Module *mod, const string &filename, const string &proto_filename) {
+  int rc;
+
+  proto_parser_ = make_unique<ebpf::cc::Parser>(proto_filename);
+  rc = proto_parser_->parse();
+  if (rc) {
+    fprintf(stderr, "In file: %s\n", filename.c_str());
+    return rc;
+  }
+
+  parser_ = make_unique<ebpf::cc::Parser>(filename);
+  rc = parser_->parse();
+  if (rc) {
+    fprintf(stderr, "In file: %s\n", filename.c_str());
+    return rc;
+  }
+
+  //ebpf::cc::Printer printer(stderr);
+  //printer.visit(parser_->root_node_);
+
+  ebpf::cc::TypeCheck type_check(parser_->scopes_.get(), proto_parser_->scopes_.get(), parser_->pragmas_);
+  auto ret = type_check.visit(parser_->root_node_);
+  if (get<0>(ret) != 0 || get<1>(ret).size()) {
+    fprintf(stderr, "Type error @line=%d: %s\n", get<0>(ret), get<1>(ret).c_str());
+    return -1;
+  }
+
+  codegen_ = ebpf::make_unique<ebpf::cc::CodegenLLVM>(mod, parser_->scopes_.get(), proto_parser_->scopes_.get());
+  ret = codegen_->visit(parser_->root_node_);
+  if (get<0>(ret) != 0 || get<1>(ret).size()) {
+    fprintf(stderr, "Codegen error @line=%d: %s\n", get<0>(ret), get<1>(ret).c_str());
+    return get<0>(ret);
+  }
+
+  return 0;
+}
+
+int BLoader::get_table_fd(const string &name) const {
+  if (!codegen_) return -1;
+  return codegen_->get_table_fd(name);
+}
+
+}  // namespace ebpf

src/cc/frontends/b/loader.h

+/*
+ * Copyright (c) 2015 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+
+namespace llvm {
+class Module;
+}
+
+namespace ebpf {
+
+namespace cc {
+class Parser;
+class CodegenLLVM;
+}
+
+class BLoader {
+ public:
+  BLoader();
+  ~BLoader();
+  int parse(llvm::Module *mod, const std::string &filename, const std::string &proto_filename);
+  int get_table_fd(const std::string &name) const;
+ private:
+  std::unique_ptr<cc::Parser> parser_;
+  std::unique_ptr<cc::Parser> proto_parser_;
+  std::unique_ptr<cc::CodegenLLVM> codegen_;
+};
+
+}  // namespace ebpf

src/cc/frontends/clang/loader.cc

+/*
+ * Copyright (c) 2015 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace ebpf {
+
+int Loader::load_file_module(unique_ptr<llvm::Module> *mod, const string &file, bool in_memory) {
+  using namespace clang;
+
+  struct utsname un;
+  uname(&un);
+  char kdir[256];
+  snprintf(kdir, sizeof(kdir), "%s/%s/build", KERNEL_MODULES_DIR, un.release);
+
+  // clang needs to run inside the kernel dir
+  DirStack dstack(kdir);
+  if (!dstack.ok())
+    return -1;
+
+  string abs_file;
+  if (in_memory) {
+    abs_file = "<bcc-memory-buffer>";
+  } else {
+    if (file.substr(0, 1) == "/")
+      abs_file = file;
+    else
+      abs_file = string(dstack.cwd()) + "/" + file;
+  }
+
+  vector<const char *> flags_cstr({"-O0", "-emit-llvm", "-I", dstack.cwd(),
+                                   "-Wno-deprecated-declarations",
+                                   "-x", "c", "-c", abs_file.c_str()});
+
+  KBuildHelper kbuild_helper;
+  vector<string> kflags;
+  if (kbuild_helper.get_flags(un.release, &kflags))
+    return -1;
+  kflags.push_back("-include");
+  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h");
+  kflags.push_back("-I");
+  kflags.push_back(BCC_INSTALL_PREFIX "/share/bcc/include");
+  for (auto it = kflags.begin(); it != kflags.end(); ++it)
+    flags_cstr.push_back(it->c_str());
+
+  // set up the error reporting class
+  IntrusiveRefCntPtr<DiagnosticOptions> diag_opts(new DiagnosticOptions());
+  auto diag_client = new TextDiagnosticPrinter(llvm::errs(), &*diag_opts);
+
+  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+  DiagnosticsEngine diags(DiagID, &*diag_opts, diag_client);
+
+  // set up the command line argument wrapper
+  driver::Driver drv("", "x86_64-unknown-linux-gnu", diags);
+  drv.setTitle("bcc-clang-driver");
+  drv.setCheckInputsExist(false);
+
+  unique_ptr<driver::Compilation> compilation(drv.BuildCompilation(flags_cstr));
+  if (!compilation)
+    return -1;
+
+  // expect exactly 1 job, otherwise error
+  const driver::JobList &jobs = compilation->getJobs();
+  if (jobs.size() != 1 || !isa<driver::Command>(*jobs.begin())) {
+    SmallString<256> msg;
+    llvm::raw_svector_ostream os(msg);
+    jobs.Print(os, "; ", true);
+    diags.Report(diag::err_fe_expected_compiler_job) << os.str();
+    return -1;
+  }
+
+  const driver::Command &cmd = cast<driver::Command>(*jobs.begin());
+  if (llvm::StringRef(cmd.getCreator().getName()) != "clang") {
+    diags.Report(diag::err_fe_expected_clang_command);
+    return -1;
+  }
+
+  // Initialize a compiler invocation object from the clang (-cc1) arguments.
+  const driver::ArgStringList &ccargs = cmd.getArguments();
+
+  // first pass
+  auto invocation1 = make_unique<CompilerInvocation>();
+  if (!CompilerInvocation::CreateFromArgs(*invocation1, const_cast<const char **>(ccargs.data()),
+                                          const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
+    return -1;
+
+  if (in_memory) {
+    invocation1->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
+                                                       llvm::MemoryBuffer::getMemBuffer(file).release());
+    invocation1->getFrontendOpts().Inputs.clear();
+    invocation1->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
+  }
+  invocation1->getFrontendOpts().DisableFree = false;
+
+  CompilerInstance compiler1;
+  compiler1.setInvocation(invocation1.release());
+  compiler1.createDiagnostics();
+
+  // capture the rewritten c file
+  string out_str;
+  llvm::raw_string_ostream os(out_str);
+  BFrontendAction bact(os);
+  if (!compiler1.ExecuteAction(bact))
+    return -1;
+  // this contains the open FDs
+  tables_ = bact.take_tables();
+
+  // second pass, clear input and take rewrite buffer
+  auto invocation2 = make_unique<CompilerInvocation>();
+  if (!CompilerInvocation::CreateFromArgs(*invocation2, const_cast<const char **>(ccargs.data()),
+                                          const_cast<const char **>(ccargs.data()) + ccargs.size(), diags))
+    return -1;
+  CompilerInstance compiler2;
+  invocation2->getPreprocessorOpts().addRemappedFile("<bcc-memory-buffer>",
+                                                     llvm::MemoryBuffer::getMemBuffer(out_str).release());
+  invocation2->getFrontendOpts().Inputs.clear();
+  invocation2->getFrontendOpts().Inputs.push_back(FrontendInputFile("<bcc-memory-buffer>", IK_C));
+  invocation2->getFrontendOpts().DisableFree = false;
+  // suppress warnings in the 2nd pass, but bail out on errors (our fault)
+  invocation2->getDiagnosticOpts().IgnoreWarnings = true;
+  compiler2.setInvocation(invocation2.release());
+  compiler2.createDiagnostics();
+
+  EmitLLVMOnlyAction ir_act(&*ctx_);
+  if (!compiler2.ExecuteAction(ir_act))
+    return -1;
+  *mod = ir_act.takeModule();
+
+  return 0;
+}
+
+
+}  // namespace ebpf

src/cc/frontends/clang/loader.h

+/*
+ * Copyright (c) 2015 PLUMgrid, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+
+namespace llvm {
+class Module;
+}
+
+namespace ebpf {
+
+namespace cc {
+class Parser;
+class CodegenLLVM;
+}
+
+class ClangLoader {
+ public:
+  ClangLoader();
+  ~ClangLoader();
+  int parse(llvm::Module *mod, const std::string &filename, const std::string &proto_filename);
+  int get_table_fd(const std::string &name) const;
+ private:
+};
+
+}  // namespace ebpf

src/python/bpf/__init__.py

@@ -23,10 +23,12 @@ basestring = (unicode if sys.version_info[0] < 3 else str)
 lib = ct.CDLL("libbpfprog.so")
 
 # keep in sync with bpf_common.h
-lib.bpf_module_create.restype = ct.c_void_p
-lib.bpf_module_create.argtypes = [ct.c_char_p, ct.c_char_p, ct.c_uint]
-lib.bpf_module_create_from_string.restype = ct.c_void_p
-lib.bpf_module_create_from_string.argtypes = [ct.c_char_p, ct.c_uint]
+lib.bpf_module_create_b.restype = ct.c_void_p
+lib.bpf_module_create_b.argtypes = [ct.c_char_p, ct.c_char_p, ct.c_uint]
+lib.bpf_module_create_c.restype = ct.c_void_p
+lib.bpf_module_create_c.argtypes = [ct.c_char_p, ct.c_uint]
+lib.bpf_module_create_c_from_string.restype = ct.c_void_p
+lib.bpf_module_create_c_from_string.argtypes = [ct.c_char_p, ct.c_uint]
 lib.bpf_module_destroy.restype = None
 lib.bpf_module_destroy.argtypes = [ct.c_void_p]
 lib.bpf_module_license.restype = ct.c_char_p
@@ -171,12 +173,16 @@ class BPF(object):
         self.debug = debug
         self.funcs = {}
         if text:
-            self.module = lib.bpf_module_create_from_string(text.encode("ascii"), self.debug)
+            self.module = lib.bpf_module_create_c_from_string(text.encode("ascii"), self.debug)
         else:
             src_file = BPF._find_file(src_file)
             hdr_file = BPF._find_file(hdr_file)
-            self.module = lib.bpf_module_create(src_file.encode("ascii"),
-                    hdr_file.encode("ascii"), self.debug)
+            if src_file.endswith(".b"):
+                self.module = lib.bpf_module_create_b(src_file.encode("ascii"),
+                        hdr_file.encode("ascii"), self.debug)
+            else:
+                self.module = lib.bpf_module_create_c(src_file.encode("ascii"),
+                        self.debug)
 
         if self.module == None:
             raise Exception("Failed to compile BPF module %s" % src_file)