Use local key/leaf definition instead of bpf_tunnel_key

Even after fixing the nested union issue in the preceding commits, this
example was failing. The reason was that the kernel was filling in all
bits of the bpf_tunnel_key, which in recent kernels includes non-zero
fields such as ttl. That non-zero field would break the lookup in the
tunkey2if table, which was populated assuming all extra bytes would be
zero.

Fix it by defining a simpler key structure, and copy the required fields
out from the local variable.

Fixes: #510

examples/networking/distributed_bridge/tunnel_mesh.c

@@ -7,24 +7,31 @@ struct config {
 };
 BPF_TABLE("hash", int, struct config, conf, 1);
 
-BPF_TABLE("hash", struct bpf_tunnel_key, int, tunkey2if, 1024);
+struct tunnel_key {
+  u32 tunnel_id;
+  u32 remote_ipv4;
+};
+BPF_TABLE("hash", struct tunnel_key, int, tunkey2if, 1024);
 
-BPF_TABLE("hash", int, struct bpf_tunnel_key, if2tunkey, 1024);
+BPF_TABLE("hash", int, struct tunnel_key, if2tunkey, 1024);
 
 // Handle packets from the encap device, demux into the dest tenant
 int handle_ingress(struct __sk_buff *skb) {
   struct bpf_tunnel_key tkey = {};
+  struct tunnel_key key;
   bpf_skb_get_tunnel_key(skb, &tkey, sizeof(tkey), 0);
 
-  int *ifindex = tunkey2if.lookup(&tkey);
+  key.tunnel_id = tkey.tunnel_id;
+  key.remote_ipv4 = tkey.remote_ipv4;
+  int *ifindex = tunkey2if.lookup(&key);
   if (ifindex) {
     //bpf_trace_printk("ingress tunnel_id=%d remote_ip=%08x ifindex=%d\n",
-    //                 tkey.tunnel_id, tkey.remote_ipv4, *ifindex);
+    //                 key.tunnel_id, key.remote_ipv4, *ifindex);
     // mark from external
     skb->tc_index = 1;
     bpf_clone_redirect(skb, *ifindex, 1/*ingress*/);
   } else {
-    bpf_trace_printk("ingress invalid tunnel_id=%d\n", tkey.tunnel_id);
+    bpf_trace_printk("ingress invalid tunnel_id=%d\n", key.tunnel_id);
   }
 
   return 1;
@@ -33,7 +40,8 @@ int handle_ingress(struct __sk_buff *skb) {
 // Handle packets from the tenant, mux into the encap device
 int handle_egress(struct __sk_buff *skb) {
   int ifindex = skb->ifindex;
-  struct bpf_tunnel_key *tkey_p, tkey = {};
+  struct bpf_tunnel_key tkey = {};
+  struct tunnel_key *key_p;
   int one = 1;
   struct config *cfg = conf.lookup(&one);
 
@@ -45,10 +53,10 @@ int handle_egress(struct __sk_buff *skb) {
     return 1;
   }
 
-  tkey_p = if2tunkey.lookup(&ifindex);
-  if (tkey_p) {
-    tkey.tunnel_id = tkey_p->tunnel_id;
-    tkey.remote_ipv4 = tkey_p->remote_ipv4;
+  key_p = if2tunkey.lookup(&ifindex);
+  if (key_p) {
+    tkey.tunnel_id = key_p->tunnel_id;
+    tkey.remote_ipv4 = key_p->remote_ipv4;
     bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey), 0);
     bpf_clone_redirect(skb, cfg->tunnel_ifindex, 0/*egress*/);
   }

examples/networking/distributed_bridge/tunnel_mesh.py

@@ -47,7 +47,7 @@ def run():
             ifc_gc.append(vx.ifname)
     else:
         with ipdb.create(ifname="vxlan0", kind="vxlan", vxlan_id=0,
-                         vxlan_link=ifc, vxlan_port=htons(4789),
+                         vxlan_link=ifc, vxlan_port=4789,
                          vxlan_collect_metadata=True,
                          vxlan_learning=False) as vx:
             vx.up()
@@ -66,12 +66,14 @@ def run():
                 if i != host_id:
                     v = ipdb.create(ifname="dummy%d%d" % (j , i), kind="dummy").up().commit()
                     ipaddr = "172.16.1.%d" % (100 + i)
-                    tunkey2if_key = tunkey2if.Key(vni, IPAddress(ipaddr))
+                    tunkey2if_key = tunkey2if.Key(vni)
+                    tunkey2if_key.remote_ipv4 = IPAddress(ipaddr)
                     tunkey2if_leaf = tunkey2if.Leaf(v.index)
                     tunkey2if[tunkey2if_key] = tunkey2if_leaf
 
                     if2tunkey_key = if2tunkey.Key(v.index)
-                    if2tunkey_leaf = if2tunkey.Leaf(vni, IPAddress(ipaddr))
+                    if2tunkey_leaf = if2tunkey.Leaf(vni)
+                    if2tunkey_leaf.remote_ipv4 = IPAddress(ipaddr)
                     if2tunkey[if2tunkey_key] = if2tunkey_leaf
 
                     ipr.tc("add", "sfq", v.index, "1:")
@@ -124,7 +126,7 @@ def run():
             while retry < 0:
                 check = Popen(["ip", "addr", "show", "br%d" % j], stdout=PIPE, stderr=PIPE)
                 out = check.stdout.read()
-                checkip = "99.1.%d" % j
+                checkip = b"99.1.%d" % j
                 retry = out.find(checkip)
 
 try: