full mesh vxlan example without multicast

Run 'sudo python ./main.py mesh'
Signed-off-by: Wei-Chun Chao <weichunc@plumgrid.com>

examples/distributed_bridge/main.py

@@ -2,10 +2,16 @@
 # Copyright (c) PLUMgrid, Inc.
 # Licensed under the Apache License, Version 2.0 (the "License")
 
+from sys import argv
 from pyroute2 import IPRoute, NetNS, IPDB, NSPopen
 from simulation import Simulation
 from subprocess import PIPE
 
+if len(argv) > 1 and argv[1] == "mesh":
+  multicast = 0
+else:
+  multicast = 1
+
 ipr = IPRoute()
 ipdb = IPDB(nl=ipr)
 
@@ -24,7 +30,10 @@ class TunnelSimulation(Simulation):
             ipaddr = "172.16.1.%d/24" % (100 + i)
             host_info.append(self._create_ns("host%d" % i, ipaddr=ipaddr,
                 disable_ipv6=True))
-            cmd = ["python", "tunnel.py"]
+            if multicast:
+              cmd = ["python", "tunnel.py"]
+            else:
+              cmd = ["python", "tunnel_mesh.py", str(num_hosts), str(i)]
             p = NSPopen(host_info[i][0].nl.netns, cmd, stdin=PIPE)
             self.processes.append(p)
         with self.ipdb.create(ifname="br-fabric", kind="bridge") as br:

examples/distributed_bridge/tunnel_mesh.c

+// Copyright (c) PLUMgrid, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License")
+#include <bcc/proto.h>
+
+struct config {
+  int tunnel_ifindex;
+};
+BPF_TABLE("hash", int, struct config, conf, 1);
+
+BPF_TABLE("hash", struct bpf_tunnel_key, int, tunkey2if, 1024);
+
+BPF_TABLE("hash", int, struct bpf_tunnel_key, if2tunkey, 1024);
+
+// Handle packets from the encap device, demux into the dest tenant
+int handle_ingress(struct __sk_buff *skb) {
+  struct bpf_tunnel_key tkey = {};
+  bpf_skb_get_tunnel_key(skb, &tkey, sizeof(tkey));
+
+  int *ifindex = tunkey2if.lookup(&tkey);
+  if (ifindex) {
+    //bpf_trace_printk("ingress tunnel_id=%d remote_ip=%08x ifindex=%d\n",
+    //                 tkey.tunnel_id, tkey.remote_ipv4, *ifindex);
+    // mark from external
+    skb->tc_index = 1;
+    bpf_clone_redirect(skb, *ifindex, 1/*ingress*/);
+  } else {
+    bpf_trace_printk("ingress invalid tunnel_id=%d\n", tkey.tunnel_id);
+  }
+
+  return 1;
+}
+
+// Handle packets from the tenant, mux into the encap device
+int handle_egress(struct __sk_buff *skb) {
+  int ifindex = skb->ifindex;
+  struct bpf_tunnel_key *tkey_p, tkey = {};
+  int one = 1;
+  struct config *cfg = conf.lookup(&one);
+
+  if (!cfg) return 1;
+
+  if (skb->tc_index) {
+    //bpf_trace_printk("from external\n");
+    // don't send it back out to encap device
+    return 1;
+  }
+
+  tkey_p = if2tunkey.lookup(&ifindex);
+  if (tkey_p) {
+    tkey.tunnel_id = tkey_p->tunnel_id;
+    tkey.remote_ipv4 = tkey_p->remote_ipv4;
+    bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey));
+    bpf_clone_redirect(skb, cfg->tunnel_ifindex, 0/*egress*/);
+  }
+  return 1;
+}

examples/distributed_bridge/tunnel_mesh.py

+#!/usr/bin/env python
+# Copyright (c) PLUMgrid, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License")
+
+from sys import argv
+from bpf import BPF
+from builtins import input
+from ctypes import c_int, c_uint
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+import json
+from netaddr import EUI, IPAddress
+from pyroute2 import IPRoute, NetNS, IPDB, NSPopen
+from socket import htons, AF_INET
+from threading import Thread
+
+num_hosts = int(argv[1])
+host_id = int(argv[2])
+
+b = BPF(src_file="tunnel_mesh.c")
+ingress_fn = b.load_func("handle_ingress", BPF.SCHED_CLS)
+egress_fn = b.load_func("handle_egress", BPF.SCHED_CLS)
+tunkey2if = b.get_table("tunkey2if")
+if2tunkey = b.get_table("if2tunkey")
+conf = b.get_table("conf")
+
+ipr = IPRoute()
+ipdb = IPDB(nl=ipr)
+
+ifc = ipdb.interfaces.eth0
+
+# ifcs to cleanup at the end
+ifc_gc = []
+
+def run():
+    with ipdb.create(ifname="vxlan0", kind="vxlan", vxlan_id=0,
+                     vxlan_link=ifc, vxlan_port=htons(4789),
+                     vxlan_flowbased=True,
+                     vxlan_collect_metadata=True,
+                     vxlan_learning=False) as vx:
+        vx.up()
+        ifc_gc.append(vx.ifname)
+
+    conf[c_int(1)] = c_int(vx.index)
+
+    ipr.tc("add", "ingress", vx.index, "ffff:")
+    ipr.tc("add-filter", "bpf", vx.index, ":1", fd=ingress_fn.fd,
+           name=ingress_fn.name, parent="ffff:", action="drop", classid=1)
+
+    for j in range(0, 2):
+        vni = 10000 + j
+        with ipdb.create(ifname="br%d" % j, kind="bridge") as br:
+            for i in range(0, num_hosts):
+                if i != host_id:
+                    v = ipdb.create(ifname="dummy%d%d" % (j , i), kind="dummy").up().commit()
+                    ipaddr = "172.16.1.%d" % (100 + i)
+                    tunkey2if_key = tunkey2if.Key(vni, IPAddress(ipaddr))
+                    tunkey2if_leaf = tunkey2if.Leaf(v.index)
+                    tunkey2if[tunkey2if_key] = tunkey2if_leaf
+
+                    if2tunkey_key = if2tunkey.Key(v.index)
+                    if2tunkey_leaf = if2tunkey.Leaf(vni, IPAddress(ipaddr))
+                    if2tunkey[if2tunkey_key] = if2tunkey_leaf
+
+                    ipr.tc("add", "sfq", v.index, "1:")
+                    ipr.tc("add-filter", "bpf", v.index, ":1", fd=egress_fn.fd,
+                       name=egress_fn.name, parent="1:", action="drop", classid=1)
+                    br.add_port(v)
+                    br.up()
+                    ifc_gc.append(v.ifname)
+            ifc_gc.append(br.ifname)
+
+try:
+    run()
+    input("")
+    print("---")
+finally:
+    for v in ifc_gc: ipdb.interfaces[v].remove().commit()
+    ipdb.release()