Add distributed bridge example, using vxlan mcast

Signed-off-by: Brenden Blanco <bblanco@plumgrid.com>

examples/distributed_bridge/main.py

+#!/usr/bin/env python
+# Copyright (c) PLUMgrid, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+from pyroute2 import IPRoute, NetNS, IPDB, NSPopen
+from simulation import Simulation
+from subprocess import PIPE
+
+ipr = IPRoute()
+ipdb = IPDB(nl=ipr)
+
+num_hosts = 3
+null = open("/dev/null", "w")
+
+class TunnelSimulation(Simulation):
+    def __init__(self, ipdb):
+        super(TunnelSimulation, self).__init__(ipdb)
+
+    def start(self):
+        # each entry is tuple of ns_ipdb, out_ifc, in_ifc
+        host_info = []
+        for i in range(0, num_hosts):
+            print("Launching host %i of %i" % (i + 1, num_hosts))
+            ipaddr = "172.16.1.%d/24" % (100 + i)
+            host_info.append(self._create_ns("host%d" % i, ipaddr=ipaddr,
+                disable_ipv6=True))
+            cmd = ["python", "tunnel.py"]
+            p = NSPopen(host_info[i][0].nl.netns, cmd, stdin=PIPE)
+            self.processes.append(p)
+        with self.ipdb.create(ifname="br-fabric", kind="bridge") as br:
+            for host in host_info: br.add_port(host[1])
+            br.up()
+
+try:
+    sim = TunnelSimulation(ipdb)
+    sim.start()
+    input("Press enter to quit:")
+finally:
+    if "br-fabric" in ipdb.interfaces: ipdb.interfaces["br-fabric"].remove().commit()
+    for p in sim.processes: p.communicate(b"\n")
+    sim.release()
+    ipdb.release()
+    null.close()

examples/distributed_bridge/simulation.py

+../simulation.py
\ No newline at end of file

examples/distributed_bridge/tunnel.c

+// Copyright (c) PLUMgrid, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License")
+#include <bcc/proto.h>
+
+BPF_TABLE("hash", u32, int, vni2if, 1024);
+
+struct vni_key {
+  u64 mac;
+  int ifindex;
+  int pad;
+};
+struct host {
+  u32 tunnel_id;
+  u32 remote_ipv4;
+  u64 rx_pkts;
+  u64 tx_pkts;
+};
+BPF_TABLE("hash", struct vni_key, struct host, mac2host, 10240);
+
+struct config {
+  int tunnel_ifindex;
+};
+BPF_TABLE("hash", int, struct config, conf, 1);
+
+// Handle packets from the encap device, demux into the dest tenant
+int handle_ingress(struct __sk_buff *skb) {
+  u8 *cursor = 0;
+
+  struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));
+
+  struct bpf_tunnel_key tkey = {};
+  bpf_skb_get_tunnel_key(skb, &tkey, sizeof(tkey));
+
+  int *ifindex = vni2if.lookup(&tkey.tunnel_id);
+  if (ifindex) {
+    //bpf_trace_printk("ingress tunnel_id=%d ifindex=%d\n", tkey.tunnel_id, *ifindex);
+    struct vni_key vk = {ethernet->src, *ifindex, 0};
+    struct host *src_host = mac2host.lookup_or_init(&vk,
+        &(struct host){tkey.tunnel_id, tkey.remote_ipv4, 0, 0});
+    lock_xadd(&src_host->rx_pkts, 1);
+    bpf_clone_redirect(skb, *ifindex, 1/*ingress*/);
+  } else {
+    bpf_trace_printk("ingress invalid tunnel_id=%d\n", tkey.tunnel_id);
+  }
+
+  return 1;
+}
+
+// Handle packets from the tenant, mux into the encap device
+int handle_egress(struct __sk_buff *skb) {
+  u8 *cursor = 0;
+
+  int one = 1;
+  struct config *cfg = conf.lookup(&one);
+  if (!cfg) return 1;
+
+  struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));
+
+  struct vni_key vk = {ethernet->dst, skb->ifindex, 0};
+  struct host *dst_host = mac2host.lookup(&vk);
+  struct bpf_tunnel_key tkey = {};
+  if (dst_host) {
+    u32 zero = 0;
+    tkey.tunnel_id = dst_host->tunnel_id;
+    tkey.remote_ipv4 = dst_host->remote_ipv4;
+    bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey));
+    lock_xadd(&dst_host->tx_pkts, 1);
+  } else {
+    struct bpf_tunnel_key tkey = {};
+    vk.mac = 0xFFFFFFFFFFFFull;
+    dst_host = mac2host.lookup(&vk);
+    if (!dst_host)
+      return 1;
+    tkey.tunnel_id = dst_host->tunnel_id;
+    tkey.remote_ipv4 = dst_host->remote_ipv4;
+    bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey));
+  }
+  bpf_clone_redirect(skb, cfg->tunnel_ifindex, 0/*egress*/);
+  return 1;
+}

examples/distributed_bridge/tunnel.py

+#!/usr/bin/env python
+# Copyright (c) PLUMgrid, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+from bpf import BPF
+from builtins import input
+from ctypes import c_int, c_uint
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+import json
+from netaddr import EUI, IPAddress
+from pyroute2 import IPRoute, NetNS, IPDB, NSPopen
+from socket import htons, AF_INET
+from threading import Thread
+
+b = BPF(src_file="tunnel.c")
+ingress_fn = b.load_func("handle_ingress", BPF.SCHED_CLS)
+egress_fn = b.load_func("handle_egress", BPF.SCHED_CLS)
+mac2host = b.get_table("mac2host")
+vni2if = b.get_table("vni2if")
+conf = b.get_table("conf")
+
+ipr = IPRoute()
+ipdb = IPDB(nl=ipr)
+
+ifc = ipdb.interfaces.eth0
+mcast = IPAddress("239.1.1.1")
+
+# ifcs to cleanup at the end
+ifc_gc = []
+
+def run():
+    ipdb.routes.add({"dst": "224.0.0.0/4", "oif": ifc.index}).commit()
+    with ipdb.create(ifname="vxlan0", kind="vxlan", vxlan_id=0,
+                     vxlan_link=ifc, vxlan_port=htons(4789),
+                     vxlan_group=str(mcast), vxlan_flowbased=True,
+                     vxlan_collect_metadata=True,
+                     vxlan_learning=False) as vx:
+        vx.up()
+        ifc_gc.append(vx.ifname)
+
+    conf[c_int(1)] = c_int(vx.index)
+
+    ipr.tc("add", "ingress", vx.index, "ffff:")
+    ipr.tc("add-filter", "bpf", vx.index, ":1", fd=ingress_fn.fd,
+           name=ingress_fn.name, parent="ffff:", action="drop", classid=1)
+
+    for i in range(0, 2):
+        vni = 10000 + i
+        with ipdb.create(ifname="br%d" % i, kind="bridge") as br:
+            v = ipdb.create(ifname="dummy%d" % i, kind="dummy").up().commit()
+            mcast_key = mac2host.Key(0xFFFFFFFFFFFF, v.index, 0)
+            mcast_leaf = mac2host.Leaf(vni, mcast.value, 0, 0)
+            mac2host[mcast_key] = mcast_leaf
+
+            ipr.tc("add", "sfq", v.index, "1:")
+            ipr.tc("add-filter", "bpf", v.index, ":1", fd=egress_fn.fd,
+                   name=egress_fn.name, parent="1:", action="drop", classid=1)
+            br.add_port(v)
+            br.up()
+            ifc_gc.append(v.ifname)
+            ifc_gc.append(br.ifname)
+            vni2if[c_uint(vni)] = c_int(v.index)
+
+try:
+    run()
+    input("")
+    print("---")
+    for k, v in mac2host.items():
+        print(EUI(k.mac), k.ifindex, IPAddress(v.remote_ipv4),
+              v.tunnel_id, v.rx_pkts, v.tx_pkts)
+finally:
+    for v in ifc_gc: ipdb.interfaces[v].remove().commit()
+    ipdb.release()