Merge pull request #1053 from palmtenor/bufferepoll

Use epoll in BPFPerfBuffer

src/cc/BPFTable.cc

@@ -14,18 +14,20 @@
  * limitations under the License.
  */
 
-#include <errno.h>
+#include <sys/epoll.h>
 #include <unistd.h>
+#include <cerrno>
 #include <cstring>
 #include <iostream>
+#include <memory>
 
 #include "BPFTable.h"
 
 #include "bcc_exception.h"
 #include "bcc_syms.h"
+#include "common.h"
 #include "libbpf.h"
 #include "perf_reader.h"
-#include "common.h"
 
 namespace ebpf {
 
@@ -70,27 +72,42 @@ StatusTuple BPFPerfBuffer::open_on_cpu(perf_reader_raw_cb cb, int cpu,
                                        void* cb_cookie, int page_cnt) {
   if (cpu_readers_.find(cpu) != cpu_readers_.end())
     return StatusTuple(-1, "Perf buffer already open on CPU %d", cpu);
-  auto reader =
-      static_cast<perf_reader*>(bpf_open_perf_buffer(cb, cb_cookie, -1, cpu, page_cnt));
+
+  auto reader = static_cast<perf_reader*>(
+      bpf_open_perf_buffer(cb, cb_cookie, -1, cpu, page_cnt));
   if (reader == nullptr)
     return StatusTuple(-1, "Unable to construct perf reader");
+
   int reader_fd = perf_reader_fd(reader);
   if (!update(&cpu, &reader_fd)) {
     perf_reader_free(static_cast<void*>(reader));
     return StatusTuple(-1, "Unable to open perf buffer on CPU %d: %s", cpu,
-                       strerror(errno));
+                       std::strerror(errno));
+  }
+
+  struct epoll_event event = {};
+  event.events = EPOLLIN;
+  event.data.ptr = static_cast<void*>(reader);
+  if (epoll_ctl(epfd_, EPOLL_CTL_ADD, reader_fd, &event) != 0) {
+    perf_reader_free(static_cast<void*>(reader));
+    return StatusTuple(-1, "Unable to add perf_reader FD to epoll: %s",
+                       std::strerror(errno));
   }
+
   cpu_readers_[cpu] = reader;
-  readers_.push_back(reader);
   return StatusTuple(0);
 }
 
-StatusTuple BPFPerfBuffer::open_all_cpu(perf_reader_raw_cb cb,
-                                        void* cb_cookie, int page_cnt) {
-  if (cpu_readers_.size() != 0 || readers_.size() != 0)
+StatusTuple BPFPerfBuffer::open_all_cpu(perf_reader_raw_cb cb, void* cb_cookie,
+                                        int page_cnt) {
+  if (cpu_readers_.size() != 0 || epfd_ != -1)
     return StatusTuple(-1, "Previously opened perf buffer not cleaned");
 
-  for (int i: get_online_cpus()) {
+  std::vector<int> cpus = get_online_cpus();
+  ep_events_.reset(new epoll_event[cpus.size()]);
+  epfd_ = epoll_create1(EPOLL_CLOEXEC);
+
+  for (int i : cpus) {
     auto res = open_on_cpu(cb, i, cb_cookie, page_cnt);
     if (res.code() != 0) {
       TRY2(close_all_cpu());
@@ -114,7 +131,19 @@ StatusTuple BPFPerfBuffer::close_on_cpu(int cpu) {
 StatusTuple BPFPerfBuffer::close_all_cpu() {
   std::string errors;
   bool has_error = false;
-  for (int i: get_online_cpus()) {
+
+  int close_res = close(epfd_);
+  epfd_ = -1;
+  ep_events_.reset();
+  if (close_res != 0) {
+    has_error = true;
+    errors += std::string(std::strerror(errno)) + "\n";
+  }
+
+  std::vector<int> opened_cpus;
+  for (auto it : cpu_readers_)
+    opened_cpus.push_back(it.first);
+  for (int i : opened_cpus) {
     auto res = close_on_cpu(i);
     if (res.code() != 0) {
       errors += "Failed to close CPU" + std::to_string(i) + " perf buffer: ";
@@ -122,23 +151,27 @@ StatusTuple BPFPerfBuffer::close_all_cpu() {
       has_error = true;
     }
   }
-  readers_.clear();
+
   if (has_error)
     return StatusTuple(-1, errors);
   return StatusTuple(0);
 }
 
 void BPFPerfBuffer::poll(int timeout) {
-  if (readers_.empty())
+  if (epfd_ < 0)
+    return;
+  int cnt = epoll_wait(epfd_, ep_events_.get(), cpu_readers_.size(), timeout);
+  if (cnt <= 0)
     return;
-  perf_reader_poll(readers_.size(), readers_.data(), timeout);
+  for (int i = 0; i < cnt; i++)
+    perf_reader_event_read(static_cast<perf_reader*>(ep_events_[i].data.ptr));
 }
 
 BPFPerfBuffer::~BPFPerfBuffer() {
   auto res = close_all_cpu();
   if (res.code() != 0)
-    std::cerr << "Failed to close all perf buffer on destruction: "
-              << res.msg() << std::endl;
+    std::cerr << "Failed to close all perf buffer on destruction: " << res.msg()
+              << std::endl;
 }
 
 }  // namespace ebpf

src/cc/BPFTable.h

@@ -16,6 +16,7 @@
 
 #pragma once
 
+#include <sys/epoll.h>
 #include <exception>
 #include <map>
 #include <memory>
@@ -123,7 +124,7 @@ private:
 class BPFPerfBuffer : protected BPFTableBase<int, int> {
 public:
   BPFPerfBuffer(BPFModule* bpf_module, const std::string& name)
-      : BPFTableBase<int, int>(bpf_module, name) {}
+      : BPFTableBase<int, int>(bpf_module, name), epfd_(-1) {}
   ~BPFPerfBuffer();
 
   StatusTuple open_all_cpu(perf_reader_raw_cb cb, void* cb_cookie,
@@ -137,7 +138,9 @@ private:
   StatusTuple close_on_cpu(int cpu);
 
   std::map<int, perf_reader*> cpu_readers_;
-  std::vector<perf_reader*> readers_;
+
+  int epfd_;
+  std::unique_ptr<epoll_event[]> ep_events_;
 };
 
 }  // namespace ebpf

src/cc/perf_reader.c

@@ -204,7 +204,7 @@ static void write_data_tail(struct perf_event_mmap_page *perf_header, uint64_t d
   perf_header->data_tail = data_tail;
 }
 
-static void event_read(struct perf_reader *reader) {
+void perf_reader_event_read(struct perf_reader *reader) {
   struct perf_event_mmap_page *perf_header = reader->base;
   uint64_t buffer_size = (uint64_t)reader->page_size * reader->page_cnt;
   uint64_t data_head;
@@ -261,7 +261,7 @@ int perf_reader_poll(int num_readers, struct perf_reader **readers, int timeout)
   if (poll(pfds, num_readers, timeout) > 0) {
     for (i = 0; i < num_readers; ++i) {
       if (pfds[i].revents & POLLIN)
-        event_read(readers[i]);
+        perf_reader_event_read(readers[i]);
     }
   }
   return 0;

src/cc/perf_reader.h

@@ -29,6 +29,7 @@ struct perf_reader * perf_reader_new(perf_reader_cb cb,
     perf_reader_raw_cb raw_cb, void *cb_cookie, int page_cnt);
 void perf_reader_free(void *ptr);
 int perf_reader_mmap(struct perf_reader *reader, unsigned type, unsigned long sample_type);
+void perf_reader_event_read(struct perf_reader *reader);
 int perf_reader_poll(int num_readers, struct perf_reader **readers, int timeout);
 int perf_reader_fd(struct perf_reader *reader);
 void perf_reader_set_fd(struct perf_reader *reader, int fd);