Merge pull request #953 from Appdynamics/master

Debian jessie .deb build config and automated test fixes.

.gitignore

@@ -2,6 +2,10 @@
 *.swp
 *.swo
 *.pyc
+.idea
 
-# Build artefacts
+# Build artifacts
 /build/
+cmake-build-debug
+debian/**/*.log
+obj-x86_64-linux-gnu

INSTALL.md

@@ -7,6 +7,7 @@
   - [Arch](#arch---aur)
   - [Gentoo](#gentoo---portage)
 * [Source](#source)
+  - [Debian](#debian---source)
   - [Ubuntu](#ubuntu---source)
   - [Fedora](#fedora---source)
 * [Older Instructions](#older-instructions)
@@ -153,6 +154,90 @@ The appropriate dependencies (e.g., ```clang```, ```llvm``` with BPF backend) wi
 
 # Source
 
+## Debian - Source
+### Jessie
+#### Repositories
+
+The automated tests that run as part of the build process require `netperf`.  Since netperf's license is not "certified"
+as an open-source license, it is in Debian's `non-free` repository.
+
+`/etc/apt/sources.list` should include the `non-free` repository and look something like this:
+
+```
+deb http://httpredir.debian.org/debian/ jessie main non-free
+deb-src http://httpredir.debian.org/debian/ jessie main non-free
+
+deb http://security.debian.org/ jessie/updates main non-free
+deb-src http://security.debian.org/ jessie/updates main non-free
+
+# wheezy-updates, previously known as 'volatile'
+deb http://ftp.us.debian.org/debian/ jessie-updates main non-free
+deb-src http://ftp.us.debian.org/debian/ jessie-updates main non-free
+```
+
+BCC also requires kernel version 4.1 or above.  Those kernels are available in the `jessie-backports` repository.  To
+add the `jessie-backports` repository to your system create the file `/etc/apt/sources.list.d/jessie-backports.list`
+with the following contents:
+
+```
+deb http://httpredir.debian.org/debian jessie-backports main
+deb-src http://httpredir.debian.org/debian jessie-backports main
+```
+
+#### Install Build Dependencies
+
+Note, check for the latest `linux-image-4.x` version in `jessie-backports` before proceeding.  Also, have a look at the
+`Build-Depends:` section in `debian/control` file.
+
+```
+# Before you begin
+apt-get update
+
+# Update kernel and linux-base package
+apt-get -t jessie-backports install linux-base linux-image-4.8.0-0.bpo.2-amd64
+
+# BCC build dependencies:
+apt-get install debhelper cmake libllvm3.8 llvm-3.8-dev libclang-3.8-dev \
+  libelf-dev bison flex libedit-dev clang-format-3.8 python python-netaddr \
+  python-pyroute2 luajit libluajit-5.1-dev arping iperf netperf ethtool \
+  devscripts
+```
+
+#### Sudo
+
+Adding eBPF probes to the kernel and removing probes from it requires root privileges.  For the build to complete
+successfully, you must build from an account with `sudo` access.  (You may also build as root, but it is bad style.)
+
+`/etc/sudoers` or `/etc/sudoers.d/build-user` should contain
+
+```
+build-user ALL = (ALL) NOPASSWD: ALL
+```
+
+or
+
+```
+build-user ALL = (ALL) ALL
+```
+
+If using the latter sudoers configuration, please keep an eye out for sudo's password prompt while the build is running.
+
+#### Build
+
+```
+cd <preferred development directory>
+git clone https://github.com/iovisor/bcc.git
+cd bcc
+debuild -b -uc -us
+```
+
+#### Install
+
+```
+cd ..
+sudo dpkg -i *bcc*.deb
+```
+
 ## Ubuntu - Source
 
 To build the toolchain from source, one needs:

debian/control

@@ -3,7 +3,12 @@ Maintainer: Brenden Blanco <bblanco@plumgrid.com>
 Section: misc
 Priority: optional
 Standards-Version: 3.9.5
-Build-Depends: debhelper (>= 9), cmake, libllvm3.7 | libllvm3.8, llvm-3.7-dev | llvm-3.8-dev, libclang-3.7-dev | libclang-3.8-dev, libelf-dev, bison, flex, libedit-dev, clang-format | clang-format-3.7, python-netaddr, python-pyroute2, luajit, libluajit-5.1-dev
+Build-Depends: debhelper (>= 9), cmake, libllvm3.7 | libllvm3.8,
+    llvm-3.7-dev | llvm-3.8-dev, libclang-3.7-dev | libclang-3.8-dev,
+    libelf-dev, bison, flex, libedit-dev,
+    clang-format | clang-format-3.7 | clang-format-3.8, python (>= 2.7),
+    python-netaddr, python-pyroute2, luajit, libluajit-5.1-dev, arping,
+    inetutils-ping | iputils-ping, iperf, netperf, ethtool, devscripts
 Homepage: https://github.com/iovisor/bcc
 
 Package: libbcc

src/lua/bcc/bpf.lua

@@ -180,7 +180,7 @@ end
 function Bpf:attach_uprobe(args)
   Bpf.check_probe_quota(1)
 
-  local path, addr = Sym.check_path_symbol(args.name, args.sym, args.addr)
+  local path, addr = Sym.check_path_symbol(args.name, args.sym, args.addr, args.pid)
   local fn = self:load_func(args.fn_name, 'BPF_PROG_TYPE_KPROBE')
   local ptype = args.retprobe and "r" or "p"
   local ev_name = string.format("%s_%s_0x%p", ptype, path:gsub("[^%a%d]", "_"), addr)

src/lua/bcc/sym.lua

@@ -30,10 +30,10 @@ local function create_cache(pid)
   }
 end
 
-local function check_path_symbol(module, symname, addr)
+local function check_path_symbol(module, symname, addr, pid)
   local sym = SYM()
   local module_path
-  if libbcc.bcc_resolve_symname(module, symname, addr or 0x0, 0, sym) < 0 then
+  if libbcc.bcc_resolve_symname(module, symname, addr or 0x0, pid or 0, sym) < 0 then
     if sym[0].module == nil then
       error("could not find library '%s' in the library path" % module)
     else

tests/lua/test_standalone.sh

@@ -19,23 +19,9 @@ if ldd bcc-lua | grep -q luajit; then
     fail "bcc-lua depends on libluajit"
 fi
 
-rm -f libbcc.so probe.lua
+rm -f probe.lua
 echo "return function(BPF) print(\"Hello world\") end" > probe.lua
 
-if ./bcc-lua "probe.lua"; then
-    fail "bcc-lua runs without libbcc.so"
-fi
-
-if ! env LIBBCC_SO_PATH=../cc/libbcc.so ./bcc-lua "probe.lua"; then
-    fail "bcc-lua cannot load libbcc.so through the environment"
-fi
-
-ln -s ../cc/libbcc.so
-
-if ! ./bcc-lua "probe.lua"; then
-    fail "bcc-lua cannot find local libbcc.so"
-fi
-
 PROBE="../../../examples/lua/offcputime.lua"
 
 if ! sudo ./bcc-lua "$PROBE" -d 1 >/dev/null 2>/dev/null; then

tests/lua/test_uprobes.lua

@@ -27,8 +27,8 @@ int count(struct pt_regs *ctx) {
   local text = text:gsub("PID", tostring(pid))
 
   local b = BPF:new{text=text}
-  b:attach_uprobe{name="c", sym="malloc_stats", fn_name="count"}
-  b:attach_uprobe{name="c", sym="malloc_stats", fn_name="count", retprobe=true}
+  b:attach_uprobe{name="c", sym="malloc_stats", fn_name="count", pid=pid}
+  b:attach_uprobe{name="c", sym="malloc_stats", fn_name="count", pid=pid, retprobe=true}
 
   assert_equals(BPF.num_open_uprobes(), 2)
 

tests/python/test_brb2.py

@@ -68,6 +68,16 @@ ipr = IPRoute()
 ipdb = IPDB(nl=ipr)
 sim = Simulation(ipdb)
 
+allocated_interfaces = set(ipdb.interfaces.keys())
+
+def get_next_iface(prefix):
+    i = 0
+    while True:
+        iface = "{0}{1}".format(prefix, i)
+        if iface not in allocated_interfaces:
+            allocated_interfaces.add(iface)
+            return iface
+        i += 1
 
 class TestBPFSocket(TestCase):
     def setup_br(self, br, veth_rt_2_br, veth_pem_2_br, veth_br_2_pem):
@@ -84,15 +94,15 @@ class TestBPFSocket(TestCase):
             br1.add_port(ipdb.interfaces[veth_rt_2_br])
             br1.up()
         subprocess.call(["sysctl", "-q", "-w", "net.ipv6.conf." + br + ".disable_ipv6=1"])
-            
+
     def set_default_const(self):
         self.ns1            = "ns1"
         self.ns2            = "ns2"
         self.ns_router      = "ns_router"
-        self.br1            = "br1"
+        self.br1            = get_next_iface("br")
         self.veth_pem_2_br1 = "v20"
         self.veth_br1_2_pem = "v21"
-        self.br2            = "br2"
+        self.br2            = get_next_iface("br")
         self.veth_pem_2_br2 = "v22"
         self.veth_br2_2_pem = "v23"
 

tests/python/test_uprobes.py

@@ -18,22 +18,24 @@ static void incr(int idx) {
         ++(*ptr);
 }
 int count(struct pt_regs *ctx) {
+    bpf_trace_printk("count() uprobe fired");
     u32 pid = bpf_get_current_pid_tgid();
     if (pid == PID)
         incr(0);
     return 0;
 }"""
-        text = text.replace("PID", "%d" % os.getpid())
+        test_pid = os.getpid()
+        text = text.replace("PID", "%d" % test_pid)
         b = bcc.BPF(text=text)
-        b.attach_uprobe(name="c", sym="malloc_stats", fn_name="count")
-        b.attach_uretprobe(name="c", sym="malloc_stats", fn_name="count")
+        b.attach_uprobe(name="c", sym="malloc_stats", fn_name="count", pid=test_pid)
+        b.attach_uretprobe(name="c", sym="malloc_stats", fn_name="count", pid=test_pid)
         libc = ctypes.CDLL("libc.so.6")
         libc.malloc_stats.restype = None
         libc.malloc_stats.argtypes = []
         libc.malloc_stats()
         self.assertEqual(b["stats"][ctypes.c_int(0)].value, 2)
-        b.detach_uretprobe(name="c", sym="malloc_stats")
-        b.detach_uprobe(name="c", sym="malloc_stats")
+        b.detach_uretprobe(name="c", sym="malloc_stats", pid=test_pid)
+        b.detach_uprobe(name="c", sym="malloc_stats", pid=test_pid)
 
     def test_simple_binary(self):
         text = """