bpf: print out the src debug info to a temporary file

Currently, for C++ API and for each func, the original src
and the rewritten source has been stored in
BCC_PROG_TAG_DIR directory. This patch allows the
source debug info (bytecode embedded by source code)
also stored in the same BCC_PROG_TAG_DIR directory.

This feature is not turned on by default now.
It requires non-zero (debug_flag & DEBUG_SOURCE).
The DEBUG_SOURCE enables "-g", with which a lot of more
llvm insns executed and it may increase application
RSS overhead by 4M (in my test).

As an example, if you modify examples/cpp/RandomRead.cc
to enable DEBUG_SOURCE, as below
-  bpf = new ebpf::BPF();
+  bpf = new ebpf::BPF(8);

After running the application, you can see:
-bash-4.3$ ls /var/tmp/bcc/bpf_prog_7f01346289a53cc3/
on_urandom_read.c  on_urandom_read.dis.txt  on_urandom_read.rewritten.c
-bash-4.3$ cat /var/tmp/bcc/bpf_prog_7f01346289a53cc3/on_urandom_read.dis.txt
; int on_urandom_read(struct urandom_read_args* attr) { // Line  23
   0:	bf 16 00 00 00 00 00 00 	r6 = r1
   1:	b7 01 00 00 00 00 00 00 	r1 = 0
; struct event_t event = {}; // Line  24
   2:	63 1a f8 ff 00 00 00 00 	*(u32 *)(r10 - 8) = r1
   3:	63 1a f4 ff 00 00 00 00 	*(u32 *)(r10 - 12) = r1
.....
Signed-off-by: Yonghong Song <yhs@fb.com>

src/cc/bcc_debug.cc

@@ -77,14 +77,15 @@ vector<string> SourceDebugger::buildLineCache() {
 
 void SourceDebugger::dumpSrcLine(const vector<string> &LineCache,
                                  const string &FileName, uint32_t Line,
-                                 uint32_t &CurrentSrcLine) {
+                                 uint32_t &CurrentSrcLine,
+                                 llvm::raw_ostream &os) {
   if (Line != 0 && Line != CurrentSrcLine && Line < LineCache.size() &&
       FileName == mod_->getSourceFileName()) {
-    errs() << "; " << StringRef(LineCache[Line - 1]).ltrim()
-           << format(
-                  " // Line"
-                  "%4" PRIu64 "\n",
-                  Line);
+    os << "; " << StringRef(LineCache[Line - 1]).ltrim()
+       << format(
+              " // Line"
+              "%4" PRIu64 "\n",
+              Line);
     CurrentSrcLine = Line;
   }
 }
@@ -179,16 +180,19 @@ void SourceDebugger::dump() {
       uint64_t FuncSize = get<1>(section.second);
       ArrayRef<uint8_t> Data(FuncStart, FuncSize);
       uint32_t CurrentSrcLine = 0;
+      string func_name = section.first.substr(fn_prefix_.size());
 
       errs() << "Disassembly of section " << section.first << ":\n"
-             << section.first.substr(fn_prefix_.size()) << ":\n";
+             << func_name << ":\n";
 
+      string src_dbg_str;
+      llvm::raw_string_ostream os(src_dbg_str);
       for (uint64_t Index = 0; Index < FuncSize; Index += Size) {
         S = DisAsm->getInstruction(Inst, Size, Data.slice(Index), Index,
                                    nulls(), nulls());
         if (S != MCDisassembler::Success) {
-          errs() << "Debug Error: disassembler failed: " << std::to_string(S)
-                 << '\n';
+          os << "Debug Error: disassembler failed: " << std::to_string(S)
+             << '\n';
           break;
         } else {
           DILineInfo LineInfo;
@@ -199,14 +203,16 @@ void SourceDebugger::dump() {
 
           adjustInstSize(Size, Data[Index], Data[Index + 1]);
           dumpSrcLine(LineCache, LineInfo.FileName, LineInfo.Line,
-                      CurrentSrcLine);
-          errs() << format("%4" PRIu64 ":", Index >> 3) << '\t';
-          dumpBytes(Data.slice(Index, Size), errs());
-          IP->printInst(&Inst, errs(), "", *STI);
-          errs() << '\n';
+                      CurrentSrcLine, os);
+          os << format("%4" PRIu64 ":", Index >> 3) << '\t';
+          dumpBytes(Data.slice(Index, Size), os);
+          IP->printInst(&Inst, os, "", *STI);
+          os << '\n';
         }
       }
-      errs() << '\n';
+      os.flush();
+      errs() << src_dbg_str << '\n';
+      src_dbg_fmap_[func_name] = src_dbg_str;
     }
 }
 

src/cc/bcc_debug.h

@@ -20,11 +20,13 @@ class SourceDebugger {
   SourceDebugger(
       llvm::Module *mod,
       std::map<std::string, std::tuple<uint8_t *, uintptr_t>> &sections,
-      const std::string &fn_prefix, const std::string &mod_src)
+      const std::string &fn_prefix, const std::string &mod_src,
+      std::map<std::string, std::string> &src_dbg_fmap)
       : mod_(mod),
         sections_(sections),
         fn_prefix_(fn_prefix),
-        mod_src_(mod_src) {}
+        mod_src_(mod_src),
+        src_dbg_fmap_(src_dbg_fmap) {}
 // Only support dump for llvm 6.x and later.
 //
 // The llvm 5.x, but not earlier versions, also supports create
@@ -40,7 +42,7 @@ class SourceDebugger {
   std::vector<std::string> buildLineCache();
   void dumpSrcLine(const std::vector<std::string> &LineCache,
                    const std::string &FileName, uint32_t Line,
-                   uint32_t &CurrentSrcLine);
+                   uint32_t &CurrentSrcLine, llvm::raw_ostream &os);
   void getDebugSections(
       llvm::StringMap<std::unique_ptr<llvm::MemoryBuffer>> &DebugSections);
 #else
@@ -53,6 +55,7 @@ class SourceDebugger {
   const std::map<std::string, std::tuple<uint8_t *, uintptr_t>> &sections_;
   const std::string &fn_prefix_;
   const std::string &mod_src_;
+  std::map<std::string, std::string> &src_dbg_fmap_;
 };
 
 }  // namespace ebpf

src/cc/bpf_module.cc

@@ -616,7 +616,8 @@ int BPFModule::finalize() {
       function_names_.push_back(section.first);
 
   if (flags_ & DEBUG_SOURCE) {
-    SourceDebugger src_debugger(mod, sections_, FN_PREFIX, mod_src_);
+    SourceDebugger src_debugger(mod, sections_, FN_PREFIX, mod_src_,
+                                src_dbg_fmap_);
     src_debugger.dump();
   }
 
@@ -690,7 +691,7 @@ int BPFModule::annotate_prog_tag(const string &name, int prog_fd,
 
   ::snprintf(buf, sizeof(buf), BCC_PROG_TAG_DIR "/bpf_prog_%llx/%s.c",
              tag1, name.data());
-  FileDesc fd(open(buf, O_CREAT | O_WRONLY | O_TRUNC,  0644));
+  FileDesc fd(open(buf, O_CREAT | O_WRONLY | O_TRUNC, 0644));
   if (fd < 0) {
     fprintf(stderr, "cannot create %s\n", buf);
     return -1;
@@ -701,7 +702,7 @@ int BPFModule::annotate_prog_tag(const string &name, int prog_fd,
 
   ::snprintf(buf, sizeof(buf), BCC_PROG_TAG_DIR "/bpf_prog_%llx/%s.rewritten.c",
              tag1, name.data());
-  fd = open(buf, O_CREAT | O_WRONLY | O_TRUNC,  0644);
+  fd = open(buf, O_CREAT | O_WRONLY | O_TRUNC, 0644);
   if (fd < 0) {
     fprintf(stderr, "cannot create %s\n", buf);
     return -1;
@@ -709,6 +710,20 @@ int BPFModule::annotate_prog_tag(const string &name, int prog_fd,
 
   src = function_source_rewritten(name);
   write(fd, src, strlen(src));
+
+  if (!src_dbg_fmap_[name].empty()) {
+    ::snprintf(buf, sizeof(buf), BCC_PROG_TAG_DIR "/bpf_prog_%llx/%s.dis.txt",
+               tag1, name.data());
+    fd = open(buf, O_CREAT | O_WRONLY | O_TRUNC, 0644);
+    if (fd < 0) {
+      fprintf(stderr, "cannot create %s\n", buf);
+      return -1;
+    }
+
+    const char *src = src_dbg_fmap_[name].c_str();
+    write(fd, src, strlen(src));
+  }
+
   return 0;
 }
 

src/cc/bpf_module.h

@@ -122,6 +122,7 @@ class BPFModule {
   std::map<llvm::Type *, std::string> writers_;
   std::string id_;
   std::string mod_src_;
+  std::map<std::string, std::string> src_dbg_fmap_;
   TableStorage *ts_;
   std::unique_ptr<TableStorage> local_ts_;
 };