Merge pull request #81 from iovisor/bblanco_dev

Update README with install info, few cleanups

CMakeLists.txt

@@ -46,6 +46,7 @@ set(CMAKE_CXX_FLAGS "-std=c++11 -Wall")
 endif()
 
 add_subdirectory(scripts)
+add_subdirectory(examples)
 add_subdirectory(src)
 add_subdirectory(tests)
 
@@ -59,6 +60,6 @@ else()
 endif()
 set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}")
 set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "amd64")
-set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libstdc++6, python")
+set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libstdc++6, python, make, gcc")
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Shared Library for BPF Compiler Collection (BCC)")
 include(CPack)

Dockerfile.fedora

@@ -11,7 +11,7 @@ RUN git clone https://github.com/llvm-mirror/llvm.git
 RUN git clone https://github.com/llvm-mirror/clang.git llvm/tools/clang
 RUN mkdir -p /root/llvm/build
 WORKDIR /root/llvm/build
-RUN cmake .. -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="ARM;CppBackend;X86;BPF"
+RUN cmake .. -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="X86;BPF"
 RUN make -j$(grep -c ^process /proc/cpuinfo)
 
 RUN mkdir -p /root/bcc/build

README.md

@@ -21,84 +21,168 @@ receive feedback from the compiler on the validity as it pertains to a BPF
 backend. This toolkit aims to provide a frontend that can only create valid BPF
 programs while still harnessing its full flexibility.
 
+Furthermore, current integrations with BPF have a kludgy workflow, sometimes
+involving compiling directly in a linux kernel source tree. This toolchain aims
+to minimize the time that a developer spends getting BPF compiled, and instead
+focus on the applications that can be written and the problems that can be
+solved with BPF.
+
 The features of this toolkit include:
 * End-to-end BPF workflow in a shared library
-  * The B language - a C-like language for BPF backends
+  * A modified C language for BPF backends
   * Integration with llvm-bpf backend for JIT
   * Dynamic (un)loading of JITed programs
   * Support for BPF kernel hooks: socket filters, tc classifiers,
       tc actions, and kprobes
 * Bindings for Python
 * Examples for socket filters, tc classifiers, and kprobes
-* Test cases!
 
-## Requirements
+In the future, more bindings besides python will likely be supported. Feel free
+to add support for the language of your choice and send a pull request!
 
-To get started using this toolchain, one needs:
-* Linux kernel 4.1 or newer, with these flags enabled:
-  * `CONFIG_BPF=y`
-  * `CONFIG_BPF_SYSCALL=y`
-  * `CONFIG_NET_CLS_BPF=m` [optional, for tc filters]
-  * `CONFIG_NET_ACT_BPF=m` [optional, for tc actions]
-  * `CONFIG_BPF_JIT=y`
-  * `CONFIG_HAVE_BPF_JIT=y`
-  * `CONFIG_BPF_EVENTS=y` [optional, for kprobes]
-* Linux kernel headers, 4.1 or newer
-* LLVM 3.7 or newer, compiled with BPF support (default=on)
-* Clang 3.7, built from the same tree as LLVM
-* pyroute2, version X.X (currently master, tag TBD) or newer
-* cmake, gcc-4.7, flex, bison
+## Examples
 
-## Getting started
+This toolchain is currently composed of two parts: a C wrapper around LLVM, and
+a Python API to interact with the running program. Later, we will go into more
+detail of how this all works.
 
-### Demo VM
+### Hello, World
 
-See https://github.com/iovisor/bcc/scripts/README.md for a script that can
-be used to set up a libvirt VM with the required dependencies.
+First, we should include the BPF class from the bpf module:
+```python
+from bpf import BPF
+```
+
+Since the C code is so short, we will embed it inside the python script.
+
+The BPF program always takes at least one argument, which is a pointer to the
+context for this type of program. Different program types have different calling
+conventions, but for this one we don't care so `void *` is fine.
+```python
+prog = """
+int hello(void *ctx) {
+  bpf_trace_printk("Hello, World!\\n");
+  return 0;
+};
+"""
+b = BPF(text=prog)
+```
 
-### Quick Setup
+For this example, we will call the program every time `fork()` is called by a
+userspace process. Underneath the hood, fork translates to the `clone` syscall,
+so we will attach our program to the kernel symbol `sys_clone`.
+```python
+fn = b.load_func("hello", BPF.KPROBE)
+BPF.attach_kprobe(fn, "sys_clone")
+```
 
-If the LLVM and Linux kernel requirements are satisfied, testing out this
-package should be as simple as:
+The python process will then print the trace printk circular buffer until ctrl-c
+is pressed. The BPF program is removed from the kernel when the userspace
+process that loaded it closes the fd (or exits).
+```python
+from subprocess import call
+try:
+    call(["cat", "/sys/kernel/debug/tracing/trace_pipe"])
+except KeyboardInterrupt:
+    pass
+```
 
+Output:
 ```
-git clone https://github.com/iovisor/bcc.git
-cd bcc; mkdir build; cd build
-cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_PREFIX_PATH=/opt/local/llvm
-make -j$(grep -c ^processor /proc/cpuinfo)
-sudo make install
-cd ../
-sudo python examples/hello_world.py
-<ctrl-C>
+bcc/examples$ sudo python hello_world.py 
+          python-7282  [002] d...  3757.488508: : Hello, World!
 ```
 
-Change `CMAKE_PREFIX_PATH` if llvm is installed elsewhere.
+[Source code listing](examples/hello_world.py)
 
-### Cleaning up
+### Networking
 
-Since packaging is currently not available, one can cleanup the collateral of
-bcc by doing:
+Walkthrough TBD, see
+[Neighbor Sharing example](examples/tc_neighbor_sharing.py) for longer
+example.
 
-```
-sudo rm -rf /usr/{lib/libbpf.prog.so,include/bcc,share/bcc}
-sudo pip uninstall bpf
-```
+### Tracing
 
-### Building LLVM
+## Requirements
 
-See http://llvm.org/docs/GettingStarted.html for the full guide.
+To get started using this toolchain in binary format, one needs:
+* Linux kernel 4.1 or newer, with these flags enabled:
+  * `CONFIG_BPF=y`
+  * `CONFIG_BPF_SYSCALL=y`
+  * `CONFIG_NET_CLS_BPF=m` [optional, for tc filters]
+  * `CONFIG_NET_ACT_BPF=m` [optional, for tc actions]
+  * `CONFIG_BPF_JIT=y`
+  * `CONFIG_HAVE_BPF_JIT=y`
+  * `CONFIG_BPF_EVENTS=y` [optional, for kprobes]
+* Headers for the above kernel
+* gcc, make, python
+* python-pyroute2 (for some networking features only)
 
-The short version:
+## Getting started
 
-```
-git clone https://github.com/llvm-mirror/llvm.git llvm
-git clone https://github.com/llvm-mirror/clang.git llvm/tools/clang
-mkdir llvm/build/
-cd llvm/build/
-cmake .. -DCMAKE_INSTALL_PREFIX=/opt/local/llvm
-make -j$(grep -c ^processor /proc/cpuinfo)
-sudo make install
-```
+As of this writing, binary packages for the above requirements are available
+in unstable formats. Both Ubuntu and Fedora have 4.2-rcX builds with the above
+flags defaulted to on. LLVM provides 3.7 Ubuntu packages (but not Fedora yet).
+
+### Ubuntu - Docker edition
+
+The build dependencies are captured in a [Dockerfile](Dockerfile.ubuntu), the
+output of which is a .deb for easy installation.
+
+* Start with a recent Ubuntu install (tested with 14.04 LTS)
+* Install a [>= 4.2 kernel](http://kernel.ubuntu.com/~kernel-ppa/mainline/)
+  with headers
+* Reboot
+* Install [docker](https://docs.docker.com/installation/ubuntulinux/)
+  (`wget -qO- https://get.docker.com/ | sh`)
+* Run the Dockerfile for Ubuntu - results in an installable .deb
+  * `git clone https://github.com/iovisor/bcc; cd bcc`
+  * `docker build -t bcc -f Dockerfile.ubuntu .`
+  * `docker run --rm -v /tmp:/mnt bcc sh -c "cp /root/bcc/build/*.deb /mnt"`
+  * `sudo dpkg -i /tmp/libbcc*.deb`
+* Run the example
+  * `sudo python /usr/share/bcc/examples/hello_world.py`
+
+### Fedora - Docker edition
+
+The build dependencies are captured in a [Dockerfile](Dockerfile.fedora), the
+output of which is a .rpm for easy installation. This version takes longer since
+LLVM needs to be compiled from source.
+
+* Start with a recent Fedora install (tested with F22)
+* Install a [>= 4.2 kernel](http://alt.fedoraproject.org/pub/alt/rawhide-kernel-nodebug/x86_64/)
+  with headers
+* Reboot
+* Install [docker](https://docs.docker.com/installation/fedora/)
+* Run the Dockerfile for Fedora - results in an installable .rpm
+  * `git clone https://github.com/iovisor/bcc; cd bcc`
+  * `docker build -t bcc -f Dockerfile.fedora .`
+  * `docker run --rm -v /tmp:/mnt bcc sh -c "cp /root/bcc/build/*.rpm /mnt"`
+  * `sudo rpm -ivh /tmp/libbcc*.rpm`
+* Run the example
+  * `sudo python /usr/share/bcc/examples/hello_world.py`
+
+### Ubuntu - From source
+
+To build the toolchain from source, one needs:
+* LLVM 3.7 or newer, compiled with BPF support (default=on)
+* Clang 3.7, built from the same tree as LLVM
+* cmake, gcc (>=4.7), flex, bison
+
+* Add the [LLVM binary repo](http://llvm.org/apt/) to your apt sources
+  * `echo "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty main" | sudo tee /etc/apt/sources.list.d/llvm.list`
+  * `wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | sudo apt-key add -`
+  * `sudo apt-get update`
+* Install build dependencies
+  * `sudo apt-get -y install bison build-essential cmake flex git libedit-dev python zlib1g-dev`
+* Install LLVM and Clang development libs
+  * `sudo apt-get -y install libllvm3.7 llvm-3.7-dev libclang-3.7-dev`
+* Install and compile BCC
+  * `git clone https://github.com/iovisor/bcc.git`
+  * `mkdir bcc/build; cd bcc/build`
+  * `cmake .. -DCMAKE_INSTALL_PREFIX=/usr`
+  * `make -j$(grep -c ^process /proc/cpuinfo)`
+  * `sudo make install`
 
 ## Release notes
 

examples/CMakeLists.txt

+set(EXAMPLE_FILES hello_world.py task_switch.py task_switch.c simple_tc.py)
+install(FILES ${EXAMPLE_FILES} DESTINATION share/bcc/examples)

src/cc/b_frontend_action.cc

@@ -222,7 +222,9 @@ bool BTypeVisitor::VisitCallExpr(CallExpr *Call) {
         } else if (Decl->getName() == "bpf_trace_printk") {
           //  #define bpf_trace_printk(fmt, args...)
           //    ({ char _fmt[] = fmt; bpf_trace_printk_(_fmt, sizeof(_fmt), args...); })
-          text = "({ char _fmt[] = " + args[0] + "; bpf_trace_printk_(_fmt, sizeof(_fmt), ";
+          text = "({ char _fmt[] = " + args[0] + "; bpf_trace_printk_(_fmt, sizeof(_fmt)";
+          if (args.size() > 1)
+            text += ", ";
           for (auto arg = args.begin() + 1; arg != args.end(); ++arg) {
             text += *arg;
             if (arg + 1 != args.end())

src/python/bpf/__init__.py

@@ -155,12 +155,26 @@ class BPF(object):
                 raise StopIteration()
             return next_key
 
+    @staticmethod
+    def _find_file(filename):
+        """ If filename is invalid, search in ./ of argv[0] """
+        if filename:
+            if not os.path.isfile(filename):
+                t = "/".join([os.path.abspath(os.path.dirname(sys.argv[0])), filename])
+                if os.path.isfile(t):
+                    filename = t
+                else:
+                    raise Exception("Could not find file %s" % filename)
+        return filename
+
     def __init__(self, src_file="", hdr_file="", text=None, debug=0):
         self.debug = debug
         self.funcs = {}
         if text:
             self.module = lib.bpf_module_create_from_string(text.encode("ascii"), self.debug)
         else:
+            src_file = BPF._find_file(src_file)
+            hdr_file = BPF._find_file(hdr_file)
             self.module = lib.bpf_module_create(src_file.encode("ascii"),
                     hdr_file.encode("ascii"), self.debug)