Merge pull request #75 from iovisor/bblanco_dev

Make tunnel example more user friendly

examples/tunnel_monitor/README.md

+## Tunnel Monitor Example
+
+This example shows how to use a BPF program to parse packets across an
+encapsulation boundary. It uses this ability to record inner+outer ip addresses
+as well as vxlan id into a hash table. The entries in that table store bytes
+and packets received/transmitted. One novel part of this program is its use of
+`bpf_tail_call` to parse two different IP headers (inner/outer) using the same
+state machine logic.
+
+Also part of this example is a simulation of a multi-host environment with an
+overlay network (using vxlan in this case), and each host contains multiple
+clients in different segments of the overlay network. The script `traffic.sh`
+can be used to simulate a subset of clients on host0 talking to various other
+clients+hosts at different traffic rates.
+
+![Overlay Diagram](vxlan.jpg)
+
+Once the simulation is running, the statistics kept by the BPF program can be
+displayed to give a visual clue as to the nature of the traffic flowing over
+the physical interface, post-encapsulation.
+
+![Chord Diagram](chord.png)
+
+To get the example running, change into the examples/tunnel_monitor directory.
+If this is the first time, run `setup.sh` to pull in the UI component and
+dependencies. You will need nodejs+npm installed on the system to run this, but
+the setup script will only install packages in the local directory.
+
+```
+[user@localhost tunnel_monitor]$ ./setup.sh 
+Cloning into 'chord-transitions'...
+remote: Counting objects: 294, done.
+...
+jquery#2.1.4 bower_components/jquery
+modernizr#2.8.3 bower_components/modernizr
+fastclick#1.0.6 bower_components/fastclick
+[user@localhost tunnel_monitor]$
+```
+
+Then, start the simulation by running main.py:
+
+```
+[root@bcc-dev tunnel_monitor]# python main.py 
+Launching host 1 of 9
+Launching host 2 of 9
+...
+Starting tunnel 8 of 9
+Starting tunnel 9 of 9
+HTTPServer listening on 0.0.0.0:8080
+Press enter to quit:
+```
+
+The prompt will remain until you choose to exit. In the background, the script
+has started a python SimpleHTTPServer on port 8080, which you may now try to
+connect to from your browser. There will likely be a blank canvas until traffic
+is sent through the tunnels.
+
+To simulate traffic, use the traffic.sh script to generate a distribution of
+pings between various clients and hosts. Check back on the chord diagram to
+see a visualization. Try clicking on a host IP address to see a breakdown of
+the inner IP addresses sent to/from that host.
+
+As an exercise, try modifying the traffic.sh script to cause one of the clients
+to send much more traffic than the others, and use the chord diagram to identify
+the culprit.

examples/tunnel_stats.py → examples/tunnel_monitor/main.py

@@ -3,11 +3,14 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 
 from builtins import input
+from http.server import HTTPServer, SimpleHTTPRequestHandler
 from netaddr import IPNetwork
+from os import chdir
 from pyroute2 import IPRoute, NetNS, IPDB, NSPopen
 from random import choice, randint
 from simulation import Simulation
 from socket import htons
+from threading import Thread
 import sys
 
 ipr = IPRoute()
@@ -27,6 +30,7 @@ class TunnelSimulation(Simulation):
         # each entry is tuple of ns_ipdb, out_ifc, in_ifc
         host_info = []
         for i in range(0, num_hosts):
+            print("Launching host %i of %i" % (i + 1, num_hosts))
             ipaddr = "172.16.1.%d/24" % (100 + i)
             host_info.append(self._create_ns("host%d" % i, ipaddr=ipaddr))
         with self.ipdb.create(ifname="br100", kind="bridge") as br100:
@@ -34,12 +38,15 @@ class TunnelSimulation(Simulation):
             br100.up()
         # create a vxlan device inside each namespace
         for host in host_info:
+            print("Starting tunnel %i of %i" % (len(self.processes) + 1, num_hosts))
             cmd = ["netserver", "-D"]
             self.processes.append(NSPopen(host[0].nl.netns, cmd, stdout=null))
             for i in range(0, num_vnis):
-                with host[0].create(ifname="vxlan%d" % i, kind="vxlan", vxlan_id=10000 + i,
+                with host[0].create(ifname="vxlan%d" % i, kind="vxlan",
+                                    vxlan_id=10000 + i,
                                     vxlan_link=host[0].interfaces.eth0,
-                                    vxlan_port=htons(4789), vxlan_group="239.1.1.%d" % (1 + i)) as vx:
+                                    vxlan_port=htons(4789),
+                                    vxlan_group="239.1.1.%d" % (1 + i)) as vx:
                     vx.up()
                 with host[0].create(ifname="br%d" % i, kind="bridge") as br:
                     br.add_port(host[0].interfaces["vxlan%d" % i])
@@ -54,13 +61,24 @@ class TunnelSimulation(Simulation):
 
         # pick one host to start the monitor in
         host = host_info[0]
-        cmd = ["python", "tunnel_monitor.py"]
+        cmd = ["python", "monitor.py"]
         p = NSPopen(host[0].nl.netns, cmd)
         self.processes.append(p)
 
+    def serve_http(self):
+        chdir("chord-transitions")
+        # comment below line to see http server log messages
+        SimpleHTTPRequestHandler.log_message = lambda self, format, *args: None
+        self.srv = HTTPServer(("", 8080), SimpleHTTPRequestHandler)
+        self.t = Thread(target=self.srv.serve_forever)
+        self.t.setDaemon(True)
+        self.t.start()
+        print("HTTPServer listening on 0.0.0.0:8080")
+
 try:
     sim = TunnelSimulation(ipdb)
     sim.start()
+    sim.serve_http()
     input("Press enter to quit:")
 finally:
     if "br100" in ipdb.interfaces: ipdb.interfaces.br100.remove().commit()

examples/tunnel_monitor.py → examples/tunnel_monitor/monitor.py

@@ -14,19 +14,19 @@ from time import sleep
 ipr = IPRoute()
 ipdb = IPDB(nl=ipr)
 
-b = BPF(src_file="tunnel_monitor.c", debug=0)
+b = BPF(src_file="monitor.c", debug=0)
 ingress_fn = b.load_func("handle_ingress", BPF.SCHED_CLS)
 egress_fn = b.load_func("handle_egress", BPF.SCHED_CLS)
 outer_fn = b.load_func("handle_outer", BPF.SCHED_CLS)
 inner_fn = b.load_func("handle_inner", BPF.SCHED_CLS)
 stats = b.get_table("stats")
+# using jump table for inner and outer packet split
 parser = b.get_table("parser")
 parser[c_int(1)] = c_int(outer_fn.fd)
 parser[c_int(2)] = c_int(inner_fn.fd)
 
 ifc = ipdb.interfaces.eth0
 
-# monitor one host...move this inside the netns to be more realistic
 ipr.tc("add", "ingress", ifc.index, "ffff:")
 ipr.tc("add-filter", "bpf", ifc.index, ":1", fd=ingress_fn.fd,
        name=ingress_fn.name, parent="ffff:", action="ok", classid=1)
@@ -69,12 +69,12 @@ while True:
 
     prev = tmp
 
-    with open("/root/chord-transitions/data/tunnel.json.new", "w") as f:
+    with open("./chord-transitions/data/tunnel.json.new", "w") as f:
         json.dump(result_total, f)
-    rename("/root/chord-transitions/data/tunnel.json.new", "/root/chord-transitions/data/tunnel.json")
-    with open("/root/chord-transitions/data/tunnel-delta.json.new", "w") as f:
+    rename("./chord-transitions/data/tunnel.json.new", "./chord-transitions/data/tunnel.json")
+    with open("./chord-transitions/data/tunnel-delta.json.new", "w") as f:
         json.dump(result_delta, f)
-    rename("/root/chord-transitions/data/tunnel-delta.json.new", "/root/chord-transitions/data/tunnel-delta.json")
+    rename("./chord-transitions/data/tunnel-delta.json.new", "./chord-transitions/data/tunnel-delta.json")
     sleep(5)
 ipdb.release()
 

examples/tunnel_monitor/setup.sh

+#!/bin/bash
+# Copyright (c) PLUMgrid, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+# this script:
+#  1. checks for bower to be installed
+#  2. clones the chord-transitions UI from github
+#  3. installs locally the packages required by the UI
+
+function which_() { hash "$1" &>/dev/null; }
+
+if [[ ! -d chord-transitions ]]; then
+  git clone https://github.com/iovisor/chord-transitions.git
+fi
+
+cd chord-transitions
+
+export PATH=node_modules/.bin:$PATH
+
+if ! which_ bower; then
+  if ! which_ npm; then
+    echo "Error: required binary 'npm' not found, please install nodejs"
+    exit 1
+  fi
+  npm install bower
+fi
+
+if [[ "$(id -u)" = "0" ]]; then
+  args="--allow-root"
+fi
+
+bower install $args

examples/tunnel_monitor/simulation.py

+../simulation.py
\ No newline at end of file

examples/tunnel_traffic.sh → examples/tunnel_monitor/traffic.sh

 #!/bin/bash
 
+cmd="ip netns exec host0"
+if [[ "$(id -u)" != "0" ]]; then
+  cmd="sudo $cmd"
+fi
+
 B=/usr/bin/byobu
 S=tunnel1
 
@@ -7,31 +12,31 @@ tmux has-session -t $S &> /dev/null
 
 if [[ $? != 0 ]]; then
   $B new-session -s $S -n "c1" -d
-  tmux send -t $S "ip netns exec host0 ping 192.168.0.1 -s512" C-m
+  tmux send -t $S "$cmd ping 192.168.0.1 -s512" C-m
   tmux new-window -t $S -n "c2"
-  tmux send -t $S "ip netns exec host0 ping 192.168.0.2 -s128" C-m
+  tmux send -t $S "$cmd ping 192.168.0.2 -s128" C-m
   tmux new-window -t $S -n "c3"
-  tmux send -t $S "ip netns exec host0 ping 192.168.0.3 -s1024" C-m
+  tmux send -t $S "$cmd ping 192.168.0.3 -s1024" C-m
   tmux new-window -t $S -n "c3"
-  tmux send -t $S "ip netns exec host0 ping 192.168.0.4 -s128" C-m
+  tmux send -t $S "$cmd ping 192.168.0.4 -s128" C-m
   tmux new-window -t $S -n "c3"
-  tmux send -t $S "ip netns exec host0 ping 192.168.0.5 -s128" C-m
+  tmux send -t $S "$cmd ping 192.168.0.5 -s128" C-m
   tmux new-window -t $S -n "c3"
-  tmux send -t $S "ip netns exec host0 ping 192.168.0.6 -s128" C-m
+  tmux send -t $S "$cmd ping 192.168.0.6 -s128" C-m
   tmux new-window -t $S -n "c4"
-  tmux send -t $S "ip netns exec host0 ping 192.168.1.2 -s128" C-m
+  tmux send -t $S "$cmd ping 192.168.1.2 -s128" C-m
   tmux new-window -t $S -n "c5"
-  tmux send -t $S "ip netns exec host0 ping 192.168.1.4 -s768" C-m
+  tmux send -t $S "$cmd ping 192.168.1.4 -s768" C-m
   tmux new-window -t $S -n "c2"
-  tmux send -t $S "ip netns exec host0 ping 192.168.2.2 -s128" C-m
+  tmux send -t $S "$cmd ping 192.168.2.2 -s128" C-m
   tmux new-window -t $S -n "c3"
-  tmux send -t $S "ip netns exec host0 ping 192.168.2.7 -s1024" C-m
+  tmux send -t $S "$cmd ping 192.168.2.7 -s1024" C-m
   tmux new-window -t $S -n "c4"
-  tmux send -t $S "ip netns exec host0 ping 192.168.2.2 -s128" C-m
+  tmux send -t $S "$cmd ping 192.168.2.2 -s128" C-m
   tmux new-window -t $S -n "c5"
-  tmux send -t $S "ip netns exec host0 ping 192.168.3.8 -s768" C-m
+  tmux send -t $S "$cmd ping 192.168.3.8 -s768" C-m
   tmux new-window -t $S -n "c5"
-  tmux send -t $S "ip netns exec host0 ping 192.168.3.9 -s768" C-m
+  tmux send -t $S "$cmd ping 192.168.3.9 -s768" C-m
 fi
 
 exec tmux attach -t $S