Add framework to support map string reader (fuse feature)

Signed-off-by: Brenden Blanco <bblanco@plumgrid.com>

src/cc/CMakeLists.txt

@@ -16,6 +16,7 @@ configure_file(libbpfprog.pc.in ${CMAKE_CURRENT_BINARY_DIR}/libbpfprog.pc @ONLY)
 # prune unused llvm static library stuff when linking into the new .so
 set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--exclude-libs=ALL")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-aliasing")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
 
 # if gcc 4.9 or higher is used, static libstdc++ is a good option
@@ -33,7 +34,9 @@ add_library(bpfprog SHARED bpf_common.cc bpf_module.cc libbpf.c)
 
 # BPF is still experimental otherwise it should be available
 #llvm_map_components_to_libnames(llvm_libs bpf mcjit irreader passes)
-llvm_map_components_to_libnames(llvm_libs mcjit irreader passes linker instrumentation objcarcopts bitwriter option)
+llvm_map_components_to_libnames(llvm_libs mcjit irreader passes linker
+  instrumentation objcarcopts bitwriter option x86codegen)
+message(STATUS "llvm_libs=${llvm_libs}")
 # order is important
 set(clang_libs ${libclangFrontend} ${libclangSerialization} ${libclangDriver} ${libclangParse}
   ${libclangSema} ${libclangCodeGen} ${libclangAnalysis} ${libclangRewrite} ${libclangEdit}

src/cc/bpf_common.cc

@@ -146,4 +146,28 @@ const char * bpf_table_leaf_desc_id(void *program, size_t id) {
   return mod->table_leaf_desc(id);
 }
 
+size_t bpf_table_key_size(void *program, const char *table_name) {
+  auto mod = static_cast<ebpf::BPFModule *>(program);
+  if (!mod) return 0;
+  return mod->table_key_size(table_name);
+}
+
+size_t bpf_table_key_size_id(void *program, size_t id) {
+  auto mod = static_cast<ebpf::BPFModule *>(program);
+  if (!mod) return 0;
+  return mod->table_key_size(id);
+}
+
+size_t bpf_table_leaf_size(void *program, const char *table_name) {
+  auto mod = static_cast<ebpf::BPFModule *>(program);
+  if (!mod) return 0;
+  return mod->table_leaf_size(table_name);
+}
+
+size_t bpf_table_leaf_size_id(void *program, size_t id) {
+  auto mod = static_cast<ebpf::BPFModule *>(program);
+  if (!mod) return 0;
+  return mod->table_leaf_size(id);
+}
+
 }

src/cc/bpf_common.h

@@ -44,6 +44,10 @@ const char * bpf_table_key_desc(void *program, const char *table_name);
 const char * bpf_table_key_desc_id(void *program, size_t id);
 const char * bpf_table_leaf_desc(void *program, const char *table_name);
 const char * bpf_table_leaf_desc_id(void *program, size_t id);
+size_t bpf_table_key_size(void *program, const char *table_name);
+size_t bpf_table_key_size_id(void *program, size_t id);
+size_t bpf_table_leaf_size(void *program, const char *table_name);
+size_t bpf_table_leaf_size_id(void *program, size_t id);
 
 #ifdef __cplusplus
 }

src/cc/bpf_module.cc

@@ -38,6 +38,7 @@
 #include <llvm/Support/FormattedStream.h>
 #include <llvm/Support/Host.h>
 #include <llvm/Support/SourceMgr.h>
+#include <llvm/Support/TargetSelect.h>
 #include <llvm/Transforms/IPO.h>
 #include <llvm/Transforms/IPO/PassManagerBuilder.h>
 
@@ -95,6 +96,8 @@ class MyMemoryManager : public SectionMemoryManager {
 
 BPFModule::BPFModule(unsigned flags)
     : flags_(flags), ctx_(new LLVMContext) {
+  InitializeNativeTarget();
+  InitializeNativeTargetAsmPrinter();
   LLVMInitializeBPFTarget();
   LLVMInitializeBPFTargetMC();
   LLVMInitializeBPFTargetInfo();
@@ -107,31 +110,34 @@ BPFModule::~BPFModule() {
   ctx_.reset();
 }
 
-// load an entire c file as a module
-int BPFModule::load_cfile(const string &file, bool in_memory) {
-  clang_loader_ = make_unique<ClangLoader>(&*ctx_);
-  unique_ptr<Module> mod;
-  if (clang_loader_->parse(&mod, &tables_, file, in_memory))
-    return -1;
-  mod_ = &*mod;
-
-  mod_->setDataLayout("e-m:e-p:64:64-i64:64-n32:64-S128");
-  mod_->setTargetTriple("bpf-pc-linux");
+unique_ptr<ExecutionEngine> BPFModule::make_reader(LLVMContext &ctx) {
+  auto m = make_unique<Module>("scanf_reader", ctx);
+  Module *mod = &*m;
+  auto structs = mod->getIdentifiedStructTypes();
+  for (auto s : structs) {
+    fprintf(stderr, "struct %s\n", s->getName().str().c_str());
+  }
 
-  for (auto fn = mod_->getFunctionList().begin(); fn != mod_->getFunctionList().end(); ++fn)
-    fn->addFnAttr(Attribute::AlwaysInline);
+  dump_ir(*mod);
+  run_pass_manager(*mod);
 
   string err;
-  engine_ = unique_ptr<ExecutionEngine>(EngineBuilder(move(mod))
-      .setErrorStr(&err)
-      .setMCJITMemoryManager(make_unique<MyMemoryManager>(&sections_))
-      .setMArch("bpf")
-      .create());
-  if (!engine_) {
+  map<string, tuple<uint8_t *, uintptr_t>> sections;
+  EngineBuilder builder(move(m));
+  builder.setErrorStr(&err);
+  builder.setMCJITMemoryManager(make_unique<MyMemoryManager>(&sections));
+  builder.setUseOrcMCJITReplacement(true);
+  auto engine = unique_ptr<ExecutionEngine>(builder.create());
+  if (!engine)
     fprintf(stderr, "Could not create ExecutionEngine: %s\n", err.c_str());
-    return -1;
-  }
+  return engine;
+}
 
+// load an entire c file as a module
+int BPFModule::load_cfile(const string &file, bool in_memory) {
+  clang_loader_ = make_unique<ClangLoader>(&*ctx_);
+  if (clang_loader_->parse(&mod_, &tables_, file, in_memory))
+    return -1;
   return 0;
 }
 
@@ -142,41 +148,44 @@ int BPFModule::load_cfile(const string &file, bool in_memory) {
 // build an ExecutionEngine.
 int BPFModule::load_includes(const string &tmpfile) {
   clang_loader_ = make_unique<ClangLoader>(&*ctx_);
-  unique_ptr<Module> mod;
-  if (clang_loader_->parse(&mod, &tables_, tmpfile, false))
+  if (clang_loader_->parse(&mod_, &tables_, tmpfile, false))
     return -1;
-  mod_ = &*mod;
-
-  mod_->setDataLayout("e-m:e-p:64:64-i64:64-n32:64-S128");
-  mod_->setTargetTriple("bpf-pc-linux");
+  return 0;
+}
 
+int BPFModule::annotate() {
   for (auto fn = mod_->getFunctionList().begin(); fn != mod_->getFunctionList().end(); ++fn)
     fn->addFnAttr(Attribute::AlwaysInline);
 
-  string err;
-  engine_ = unique_ptr<ExecutionEngine>(EngineBuilder(move(mod))
-      .setErrorStr(&err)
-      .setMCJITMemoryManager(make_unique<MyMemoryManager>(&sections_))
-      .setMArch("bpf")
-      .create());
-  if (!engine_) {
-    fprintf(stderr, "Could not create ExecutionEngine: %s\n", err.c_str());
-    return -1;
+  //for (auto s : mod_->getIdentifiedStructTypes()) {
+  //  llvm::errs() << "struct " << s->getName() << "\n";
+  //  for (auto e : s->elements()) {
+  //    llvm::errs() << " ";
+  //    e->print(llvm::errs());
+  //    llvm::errs() << "\n";
+  //  }
+  //}
+
+  if (1) {
+    auto engine = make_reader(*ctx_);
+    if (engine)
+      engine->finalizeObject();
   }
 
+
   return 0;
 }
 
-void BPFModule::dump_ir() {
+void BPFModule::dump_ir(Module &mod) {
   legacy::PassManager PM;
   PM.add(createPrintModulePass(outs()));
-  PM.run(*mod_);
+  PM.run(mod);
 }
 
-int BPFModule::finalize() {
-  if (verifyModule(*mod_, &errs())) {
+int BPFModule::run_pass_manager(Module &mod) {
+  if (verifyModule(mod, &errs())) {
     if (flags_ & 1)
-      dump_ir();
+      dump_ir(mod);
     return -1;
   }
 
@@ -188,7 +197,30 @@ int BPFModule::finalize() {
   PMB.populateModulePassManager(PM);
   if (flags_ & 1)
     PM.add(createPrintModulePass(outs()));
-  PM.run(*mod_);
+  PM.run(mod);
+  return 0;
+}
+
+int BPFModule::finalize() {
+  Module *mod = &*mod_;
+
+  mod->setDataLayout("e-m:e-p:64:64-i64:64-n32:64-S128");
+  mod->setTargetTriple("bpf-pc-linux");
+
+  string err;
+  EngineBuilder builder(move(mod_));
+  builder.setErrorStr(&err);
+  builder.setMCJITMemoryManager(make_unique<MyMemoryManager>(&sections_));
+  builder.setMArch("bpf");
+  builder.setUseOrcMCJITReplacement(true);
+  engine_ = unique_ptr<ExecutionEngine>(builder.create());
+  if (!engine_) {
+    fprintf(stderr, "Could not create ExecutionEngine: %s\n", err.c_str());
+    return -1;
+  }
+
+  if (int rc = run_pass_manager(*mod))
+    return rc;
 
   engine_->finalizeObject();
 
@@ -308,6 +340,27 @@ const char * BPFModule::table_leaf_desc(const string &name) const {
   if (table_it == tables_->end()) return nullptr;
   return table_it->second.leaf_desc.c_str();
 }
+size_t BPFModule::table_key_size(size_t id) const {
+  if (id >= table_names_.size()) return 0;
+  return table_key_size(table_names_[id]);
+}
+size_t BPFModule::table_key_size(const string &name) const {
+  if (b_loader_) return 0;
+  auto table_it = tables_->find(name);
+  if (table_it == tables_->end()) return 0;
+  return table_it->second.key_size;
+}
+
+size_t BPFModule::table_leaf_size(size_t id) const {
+  if (id >= table_names_.size()) return 0;
+  return table_leaf_size(table_names_[id]);
+}
+size_t BPFModule::table_leaf_size(const string &name) const {
+  if (b_loader_) return 0;
+  auto table_it = tables_->find(name);
+  if (table_it == tables_->end()) return 0;
+  return table_it->second.leaf_size;
+}
 
 // load a B file, which comes in two parts
 int BPFModule::load_b(const string &filename, const string &proto_filename) {
@@ -324,9 +377,11 @@ int BPFModule::load_b(const string &filename, const string &proto_filename) {
   // pass the partially compiled module to the B frontend to continue with.
   if (int rc = load_includes(BCC_INSTALL_PREFIX "/share/bcc/include/bcc/helpers.h"))
     return rc;
+  if (int rc = annotate())
+    return rc;
 
   b_loader_.reset(new BLoader);
-  if (int rc = b_loader_->parse(mod_, filename, proto_filename))
+  if (int rc = b_loader_->parse(&*mod_, filename, proto_filename))
     return rc;
   if (int rc = finalize())
     return rc;
@@ -345,6 +400,8 @@ int BPFModule::load_c(const string &filename) {
   }
   if (int rc = load_cfile(filename, false))
     return rc;
+  if (int rc = annotate())
+    return rc;
   if (int rc = finalize())
     return rc;
   return 0;
@@ -358,6 +415,8 @@ int BPFModule::load_string(const string &text) {
   }
   if (int rc = load_cfile(text, true))
     return rc;
+  if (int rc = annotate())
+    return rc;
 
   if (int rc = finalize())
     return rc;

src/cc/bpf_module.h

@@ -39,11 +39,14 @@ class BPFModule {
   int init_engine();
   int parse(llvm::Module *mod);
   int finalize();
-  void dump_ir();
+  int annotate();
+  std::unique_ptr<llvm::ExecutionEngine> make_reader(llvm::LLVMContext &ctx);
+  void dump_ir(llvm::Module &mod);
   int load_file_module(std::unique_ptr<llvm::Module> *mod, const std::string &file, bool in_memory);
   int load_includes(const std::string &tmpfile);
   int load_cfile(const std::string &file, bool in_memory);
   int kbuild_flags(const char *uname_release, std::vector<std::string> *cflags);
+  int run_pass_manager(llvm::Module &mod);
  public:
   BPFModule(unsigned flags);
   ~BPFModule();
@@ -62,8 +65,12 @@ class BPFModule {
   const char * table_name(size_t id) const;
   const char * table_key_desc(size_t id) const;
   const char * table_key_desc(const std::string &name) const;
+  size_t table_key_size(size_t id) const;
+  size_t table_key_size(const std::string &name) const;
   const char * table_leaf_desc(size_t id) const;
   const char * table_leaf_desc(const std::string &name) const;
+  size_t table_leaf_size(size_t id) const;
+  size_t table_leaf_size(const std::string &name) const;
   char * license() const;
   unsigned kern_version() const;
  private:
@@ -72,7 +79,7 @@ class BPFModule {
   std::string proto_filename_;
   std::unique_ptr<llvm::LLVMContext> ctx_;
   std::unique_ptr<llvm::ExecutionEngine> engine_;
-  llvm::Module *mod_;
+  std::unique_ptr<llvm::Module> mod_;
   std::unique_ptr<BLoader> b_loader_;
   std::unique_ptr<ClangLoader> clang_loader_;
   std::map<std::string, std::tuple<uint8_t *, uintptr_t>> sections_;

src/cc/frontends/clang/b_frontend_action.cc

@@ -51,6 +51,13 @@ bool BMapDeclVisitor::VisitFieldDecl(FieldDecl *D) {
   result_ += "\",";
   return true;
 }
+
+bool BMapDeclVisitor::TraverseRecordDecl(RecordDecl *D) {
+  // skip children, handled in Visit...
+  if (!WalkUpFromRecordDecl(D))
+    return false;
+  return true;
+}
 bool BMapDeclVisitor::VisitRecordDecl(RecordDecl *D) {
   result_ += "[\"";
   result_ += D->getName();
@@ -65,7 +72,7 @@ bool BMapDeclVisitor::VisitRecordDecl(RecordDecl *D) {
   if (!D->getDefinition()->field_empty())
     result_.erase(result_.end() - 2);
   result_ += "]]";
-  return false;
+  return true;
 }
 bool BMapDeclVisitor::VisitTagType(const TagType *T) {
   return TraverseDecl(T->getDecl()->getDefinition());
@@ -80,6 +87,64 @@ bool BMapDeclVisitor::VisitBuiltinType(const BuiltinType *T) {
   return true;
 }
 
+BScanfVisitor::BScanfVisitor(ASTContext &C)
+    : C(C), n_args_(0) {}
+
+bool BScanfVisitor::VisitFieldDecl(FieldDecl *D) {
+  args_ += "&val->" + D->getName().str();
+  ++n_args_;
+  return true;
+}
+
+bool BScanfVisitor::TraverseRecordDecl(RecordDecl *D) {
+  // skip children, handled in Visit...
+  if (!WalkUpFromRecordDecl(D))
+    return false;
+  return true;
+}
+
+bool BScanfVisitor::VisitRecordDecl(RecordDecl *D) {
+  if (type_.empty())
+    type_ = "struct " + D->getDefinition()->getName().str();
+  fmt_ += "{ ";
+  for (auto F : D->getDefinition()->fields()) {
+    TraverseDecl(F);
+    if (F->isBitField())
+      fmt_ += ", " + to_string(F->getBitWidthValue(C));
+    fmt_ += ", ";
+    args_ += ", ";
+  }
+  if (!D->getDefinition()->field_empty()) {
+    fmt_.erase(fmt_.end() - 2);
+    args_.erase(args_.end() - 2);
+  }
+  fmt_ += "}";
+  return true;
+}
+bool BScanfVisitor::VisitTagType(const TagType *T) {
+  return TraverseDecl(T->getDecl()->getDefinition());
+}
+bool BScanfVisitor::VisitTypedefType(const TypedefType *T) {
+  return TraverseDecl(T->getDecl());
+}
+bool BScanfVisitor::VisitBuiltinType(const BuiltinType *T) {
+  if (type_.empty()) {
+    type_ = T->getName(C.getPrintingPolicy());
+    args_ += "val";
+    ++n_args_;
+  }
+  fmt_ += "%i";
+  return true;
+}
+void BScanfVisitor::finalize(string &result) {
+  result  = "int read_entry(const char *str, void *buf) {\n";
+  result += "  " + type_ + " *val = buf;\n";
+  result += "  int n = sscanf(str, \"" + fmt_ + "\", " + args_ + ");\n";
+  result += "  if (n < " + std::to_string(n_args_) + ") return -1;\n";
+  result += "  return 0;\n";
+  result += "}\n";
+}
+
 BTypeVisitor::BTypeVisitor(ASTContext &C, Rewriter &rewriter, map<string, BPFTable> &tables)
     : C(C), rewriter_(rewriter), out_(llvm::errs()), tables_(tables) {
 }
@@ -361,11 +426,21 @@ bool BTypeVisitor::VisitVarDecl(VarDecl *Decl) {
       if (F->getName() == "key") {
         table.key_size = sz;
         BMapDeclVisitor visitor(C, table.key_desc);
-        visitor.TraverseType(F->getType());
+        if (!visitor.TraverseType(F->getType()))
+          return false;
+        BScanfVisitor scanf_visitor(C);
+        if (!scanf_visitor.TraverseType(F->getType()))
+          return false;
+        scanf_visitor.finalize(table.key_reader);
       } else if (F->getName() == "leaf") {
         table.leaf_size = sz;
         BMapDeclVisitor visitor(C, table.leaf_desc);
-        visitor.TraverseType(F->getType());
+        if (!visitor.TraverseType(F->getType()))
+          return false;
+        BScanfVisitor scanf_visitor(C);
+        if (!scanf_visitor.TraverseType(F->getType()))
+          return false;
+        scanf_visitor.finalize(table.leaf_reader);
       } else if (F->getName() == "data") {
         table.max_entries = sz / table.leaf_size;
       }

src/cc/frontends/clang/b_frontend_action.h

@@ -43,23 +43,44 @@ struct BPFTable {
   size_t max_entries;
   std::string key_desc;
   std::string leaf_desc;
+  std::string key_reader;
+  std::string leaf_reader;
 };
 
 // Helper visitor for constructing a string representation of a key/leaf decl
 class BMapDeclVisitor : public clang::RecursiveASTVisitor<BMapDeclVisitor> {
  public:
   explicit BMapDeclVisitor(clang::ASTContext &C, std::string &result);
+  bool TraverseRecordDecl(clang::RecordDecl *Decl);
   bool VisitRecordDecl(clang::RecordDecl *Decl);
   bool VisitFieldDecl(clang::FieldDecl *Decl);
   bool VisitBuiltinType(const clang::BuiltinType *T);
   bool VisitTypedefType(const clang::TypedefType *T);
   bool VisitTagType(const clang::TagType *T);
-  const std::string & str() const { return result_; }
  private:
   clang::ASTContext &C;
   std::string &result_;
 };
 
+// Helper visitor for constructing a fscanf routine for key/leaf decl
+class BScanfVisitor : public clang::RecursiveASTVisitor<BScanfVisitor> {
+ public:
+  explicit BScanfVisitor(clang::ASTContext &C);
+  bool TraverseRecordDecl(clang::RecordDecl *Decl);
+  bool VisitRecordDecl(clang::RecordDecl *Decl);
+  bool VisitFieldDecl(clang::FieldDecl *Decl);
+  bool VisitBuiltinType(const clang::BuiltinType *T);
+  bool VisitTypedefType(const clang::TypedefType *T);
+  bool VisitTagType(const clang::TagType *T);
+  void finalize(std::string &result);
+ private:
+  clang::ASTContext &C;
+  size_t n_args_;
+  std::string fmt_;
+  std::string args_;
+  std::string type_;
+};
+
 // Type visitor and rewriter for B programs.
 // It will look for B-specific features and rewrite them into a valid
 // C program. As part of the processing, open the necessary BPF tables

tests/cc/test_clang.py

@@ -46,5 +46,16 @@ int count_foo(struct pt_regs *ctx, unsigned long a, unsigned long b) {
         b = BPF(text=text, debug=0)
         fn = b.load_func("count_foo", BPF.KPROBE)
 
+    def test_scanf(self):
+        text = """
+BPF_TABLE("hash", int, struct { int a; int b; }, stats, 10);
+int foo(void *ctx) {
+    return 0;
+}
+"""
+        b = BPF(text=text, debug=0)
+        fn = b.load_func("foo", BPF.KPROBE)
+        # todo: the actual test
+
 if __name__ == "__main__":
     main()