Added vlan_filter application. (#1773)

* Added vlan_filter application.

* Added demo application and changed timestamp to human readable format.

* changed files to executable and updated README.md file dependencies part.

* Fixed header printout to match actual output and README.

examples/networking/vlan_filter/README.md

+# VLAN Filter #
+This is eBPF application to parse VXLAN packets and then extracts encapsulated VLAN packets to monitor traffic from each VLAN. Extracted packet header fields can be stored in a file or sent to remote server via Apache Kafka messaging system.
+
+Also part of this example is a simulation of a multi-host environment. Simulation environment can be setup by using test_setup.sh script. Then a sample script (traffic.sh) can be used to send traffic from one client (VLAN=100) from host1 talking to another client at host2 and one client (VLAN=200) from host2 talking to another client at host1 while running vlan_filter application in parallel by using command 'python data-plane-tracing -i veth7'.
+
+![picture](scenario.jpg)
+
+### Usage Example ###
+* $ sudo python data-plane-tracing.py
+
+Timestamp | Host Name  | Host IP   | IP Version   | Source Host IP   | Dest Host IP   | Source Host Port   | Dest Host Port   | VNI   | Source VM MAC  | Dest VM MAC  | VLAN ID  | Source VM IP   | Dest VM IP   | Protocol   | Source VM Port   | Dest VM Port   | Packet Length   |
+---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---
+ 2018-05-24 18:43:30.386228 | Box1 | x.x.x.x  | 4 | 10.1.1.11 | 10.1.1.12 | 54836 | 4789 | 10 | fa:16:3e:ec:22:99 | fa:16:3e:1c:6f:2d | 100 | 192.168.100.11 | 192.168.100.12 | 6 | 1285 | 20302 | 1200
+
+
+# Implementation overview #
+Example application implementation is split into two parts: the former that exploits eBPF code, the latter that performs some additional processing in user space (python wrapper).
+
+### First part: eBPF Filter ###
+This component filters VXLAN packets.
+The program is loaded as PROG_TYPE_SOCKET_FILTER and attached to a socket, bind to eth0.
+Packets matching VXLAN filter are forwarded to the user space, while other packets are dropped.
+
+### Python code in user space ###
+The Python script reads filtered raw packets from the socket, extracts all the useful header fields and stores extracted packet into a file by default or can be sent to a remote server via Apache Kafka messaging system.
+
+# How to execute this example application #
+VLAN Filter application can be executed by using one of the below commands:
+* $ sudo python data-plane-tracing.py
+* $ sudo python data-plane-tracing -i eth2 -k vc.manage.overcloud:9092
+
+# How to install Required Dependencies
+* $ pip install kafka-python
+* $ pip install netifaces

examples/networking/vlan_filter/data-plane-tracing.c

+#include <uapi/linux/ptrace.h>
+#include <net/sock.h>
+#include <bcc/proto.h>
+
+#define IP_TCP 	6
+#define IP_UDP 17
+#define IP_ICMP 1
+/* 
+  In 802.3, both the source and destination addresses are 48 bits (4 bytes) MAC address.
+  6 bytes (src) + 6 bytes (dst) + 2 bytes (type) = 14 bytes 
+*/
+#define ETH_HLEN 14
+
+/*eBPF program.
+  Filter TCP/UDP/ICMP packets, having payload not empty
+  if the program is loaded as PROG_TYPE_SOCKET_FILTER
+  and attached to a socket
+  return  0 -> DROP the packet
+  return -1 -> KEEP the packet and return it to user space (userspace can read it from the socket_fd )
+*/
+int vlan_filter(struct __sk_buff *skb) { 
+	u8 *cursor = 0;	
+
+	struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));
+	
+	//filter IP packets (ethernet type = 0x0800) 0x0800 is IPv4 packet
+	switch(ethernet->type){
+		case 0x0800: goto IP;
+	    	default: goto DROP;
+	}
+
+	
+	IP: ;
+		struct ip_t *ip = cursor_advance(cursor, sizeof(*ip));  // IP header (datagram)
+	        switch (ip->nextp){
+			case 17: goto UDP;
+			default: goto DROP;
+		}
+
+	UDP: ;
+		struct udp_t *udp = cursor_advance(cursor, sizeof(*udp));
+		switch (udp->dport) {
+    			case 4789: goto KEEP;
+    			default: goto DROP;
+  		}
+
+	//keep the packet and send it to userspace retruning -1
+	KEEP:
+		return -1;
+
+	//drop the packet returning 0
+	DROP:
+		return 0;
+}
\ No newline at end of file

examples/networking/vlan_filter/data-plane-tracing.py

+#!/usr/bin/python
+from __future__ import print_function
+from bcc import BPF
+
+import sys
+import socket
+import os
+import argparse
+import time
+import netifaces as ni
+
+from sys import argv
+from kafka import KafkaProducer
+from kafka.errors import KafkaError
+from datetime import datetime
+
+#args
+def usage():
+    print("USAGE: %s [-i <if_name>]" % argv[0])
+    print("")
+    print("Try '%s -h' for more options." % argv[0])
+    exit()
+
+#help
+def help():
+    print("USAGE: %s [-i <if_name>][-k <kafka_server_name:kafka_port>]" % argv[0])
+    print("")
+    print("optional arguments:")
+    print("   -h                       print this help")
+    print("   -i if_name               select interface if_name. Default is eth0")
+    print("   -k kafka_server_name     select kafka server name. Default is save to file")
+    print("                            If -k option is not specified data will be saved to file.")
+    
+    print("")
+    print("examples:")
+    print("    data-plane-tracing                                      # bind socket to eth0")
+    print("    data-plane-tracing -i eno2 -k vc.manage.overcloud:9092  # bind socket to eno2 and send data to kafka server in iovisor-topic.")
+    exit()
+
+#arguments
+interface="eth0"
+kafkaserver=''
+        
+#check provided arguments
+if len(argv) == 2:
+    if str(argv[1]) == '-h':
+        help()
+    else:
+        usage()
+
+if len(argv) == 3:
+    if str(argv[1]) == '-i':
+        interface = argv[2]
+    elif str(argv[1]) == '-k':
+        kafkaserver = argv[2] 
+    else:
+        usage()
+    
+if len(argv) == 5:
+    if str(argv[1]) == '-i':
+        interface = argv[2]
+        kafkaserver = argv[4]
+    elif str(argv[1]) == '-k':
+        kafkaserver = argv[2] 
+        interface = argv[4]
+    else:
+        usage()
+
+if len(argv) > 5:
+    usage()
+
+print ("binding socket to '%s'" % interface)	
+ 
+#initialize BPF - load source code from http-parse-simple.c
+bpf = BPF(src_file = "data-plane-tracing.c", debug = 0)
+
+#load eBPF program http_filter of type SOCKET_FILTER into the kernel eBPF vm
+#more info about eBPF program types http://man7.org/linux/man-pages/man2/bpf.2.html
+function_vlan_filter = bpf.load_func("vlan_filter", BPF.SOCKET_FILTER)
+
+#create raw socket, bind it to eth0
+#attach bpf program to socket created
+BPF.attach_raw_socket(function_vlan_filter, interface)
+
+#get file descriptor of the socket previously created inside BPF.attach_raw_socket
+socket_fd = function_vlan_filter.sock
+
+#create python socket object, from the file descriptor
+sock = socket.fromfd(socket_fd,socket.PF_PACKET,socket.SOCK_RAW,socket.IPPROTO_IP)
+
+#set it as blocking socket
+sock.setblocking(True)
+
+#get interface ip address. In case ip is not set then just add 127.0.0.1.
+ni.ifaddresses(interface)
+try:
+    ip = ni.ifaddresses(interface)[ni.AF_INET][0]['addr']
+except:
+    ip = '127.0.0.1'    
+
+print("| Timestamp | Host Name | Host IP | IP Version | Source Host IP | Dest Host IP | Source Host Port | Dest Host Port | VNI | Source VM MAC | Dest VM MAC | VLAN ID | Source VM IP | Dest VM IP | Protocol | Source VM Port | Dest VM Port | Packet Length |")
+
+while 1:
+    #retrieve raw packet from socket
+    packet_str = os.read(socket_fd, 2048)
+    
+    #convert packet into bytearray
+    packet_bytearray = bytearray(packet_str)
+    
+    #ethernet header length
+    ETH_HLEN = 14 
+    
+    #VXLAN header length
+    VXLAN_HLEN = 8
+    
+    #VLAN header length
+    VLAN_HLEN = 4
+    
+    #Inner TCP/UDP header length
+    TCP_HLEN = 20
+    UDP_HLEN = 8
+    
+    #calculate packet total length
+    total_length = packet_bytearray[ETH_HLEN + 2]               #load MSB
+    total_length = total_length << 8                            #shift MSB
+    total_length = total_length + packet_bytearray[ETH_HLEN+3]  #add LSB
+    
+    #calculate ip header length
+    ip_header_length = packet_bytearray[ETH_HLEN]               #load Byte
+    ip_header_length = ip_header_length & 0x0F                  #mask bits 0..3
+    ip_header_length = ip_header_length << 2                    #shift to obtain length
+    
+    #calculate payload offset
+    payload_offset = ETH_HLEN + ip_header_length + UDP_HLEN + VXLAN_HLEN
+    
+    #parsing ip version from ip packet header
+    ipversion = str(bin(packet_bytearray[14])[2:5])
+    
+    #parsing source ip address, destination ip address from ip packet header
+    src_host_ip = str(packet_bytearray[26]) + "." + str(packet_bytearray[27]) + "." + str(packet_bytearray[28]) + "." + str(packet_bytearray[29])
+    dest_host_ip = str(packet_bytearray[30]) + "." + str(packet_bytearray[31]) + "." + str(packet_bytearray[32]) + "." + str(packet_bytearray[33])
+    
+    #parsing source port and destination port
+    src_host_port = packet_bytearray[34] << 8 | packet_bytearray[35]
+    dest_host_port = packet_bytearray[36] << 8 | packet_bytearray[37]
+    
+    #parsing VNI from VXLAN header
+    VNI = str((packet_bytearray[46])+(packet_bytearray[47])+(packet_bytearray[48]))
+    
+    #parsing source mac address and destination mac address
+    mac_add = [packet_bytearray[50], packet_bytearray[51], packet_bytearray[52], packet_bytearray[53], packet_bytearray[54], packet_bytearray[55]]
+    src_vm_mac = ":".join(map(lambda b: format(b, "02x"), mac_add))
+    mac_add = [packet_bytearray[56], packet_bytearray[57], packet_bytearray[58], packet_bytearray[59], packet_bytearray[60], packet_bytearray[61]]
+    dest_vm_mac = ":".join(map(lambda b: format(b, "02x"), mac_add))
+    
+    #parsing VLANID from VLAN header
+    VLANID=""
+    VLANID = str((packet_bytearray[64])+(packet_bytearray[65]))
+
+    #parsing source vm ip address, destination vm ip address from encapsulated ip packet header
+    src_vm_ip = str(packet_bytearray[80]) + "." + str(packet_bytearray[81]) + "." + str(packet_bytearray[82]) + "." + str(packet_bytearray[83])
+    dest_vm_ip = str(packet_bytearray[84]) + "." + str(packet_bytearray[85]) + "." + str(packet_bytearray[86]) + "." + str(packet_bytearray[87]) 
+    
+    #parsing source port and destination port
+    if (packet_bytearray[77]==6 or packet_bytearray[77]==17):
+        src_vm_port = packet_bytearray[88] << 8 | packet_bytearray[88]
+        dest_vm_port = packet_bytearray[90] << 8 | packet_bytearray[91]
+    elif (packet_bytearray[77]==1):
+        src_vm_port = -1
+        dest_vm_port = -1
+        type = str(packet_bytearray[88])
+    else:
+        continue
+    
+    timestamp = str(datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S.%f'))
+    
+    #send data to remote server via Kafka Messaging Bus
+    if kafkaserver:
+        MESSAGE = (timestamp, socket.gethostname(),ip, str(int(ipversion, 2)), str(src_host_ip), str(dest_host_ip), str(src_host_port), str(dest_host_port), str(int(VNI)), str(src_vm_mac), str(dest_vm_mac), str(int(VLANID)), src_vm_ip, dest_vm_ip, str(packet_bytearray[77]), str(src_vm_port), str(dest_vm_port), str(total_length))
+        print (MESSAGE)
+        MESSAGE = ','.join(MESSAGE)
+        MESSAGE = MESSAGE.encode() 
+        producer = KafkaProducer(bootstrap_servers=[kafkaserver])
+        producer.send('iovisor-topic', key=b'iovisor', value=MESSAGE)
+    
+    #save data to files
+    else:
+        MESSAGE = timestamp+","+socket.gethostname()+","+ip+","+str(int(ipversion, 2))+","+src_host_ip+","+dest_host_ip+","+str(src_host_port)+","+str(dest_host_port)+","+str(int(VNI))+","+str(src_vm_mac)+","+str(dest_vm_mac)+","+str(int(VLANID))+","+src_vm_ip+","+dest_vm_ip+","+str(packet_bytearray[77])+","+str(src_vm_port)+","+str(dest_vm_port)+","+str(total_length)
+        print (MESSAGE)
+        #save data to a file on hour basis 
+        filename = "./vlan-data-"+time.strftime("%Y-%m-%d-%H")+"-00"
+        with open(filename, "a") as f:
+            f.write("%s\n" % MESSAGE)

examples/networking/vlan_filter/test_setup.sh

+#!/bin/bash
+
+# This script must be executed by root user
+if [ "$(id -u)" != "0" ]; then
+   echo "This script must be run as root" 1>&2
+   exit 1
+fi
+
+# add namespaces
+ip netns add netns11
+ip netns add netns12
+ip netns add netns21
+ip netns add netns22
+ip netns add netns3
+ip netns add netns4
+
+# set up veth devices in netns11 to netns21 with connection to netns3  
+ip link add veth11 type veth peer name veth13
+ip link add veth21 type veth peer name veth23
+ip link set veth11 netns netns11
+ip link set veth21 netns netns21
+ip link set veth13 netns netns3
+ip link set veth23 netns netns3
+
+# set up veth devices in netns12 and netns22 with connection to netns4 
+ip link add veth12 type veth peer name veth14
+ip link add veth22 type veth peer name veth24
+ip link set veth12 netns netns12
+ip link set veth22 netns netns22
+ip link set veth14 netns netns4
+ip link set veth24 netns netns4
+  
+# assign IP addresses and set the devices up 
+ip netns exec netns11 ifconfig veth11 192.168.100.11/24 up
+ip netns exec netns11 ip link set lo up
+ip netns exec netns12 ifconfig veth12 192.168.100.12/24 up
+ip netns exec netns12 ip link set lo up
+ip netns exec netns21 ifconfig veth21 192.168.200.21/24 up
+ip netns exec netns21 ip link set lo up
+ip netns exec netns22 ifconfig veth22 192.168.200.22/24 up
+ip netns exec netns22 ip link set lo up
+
+# set up bridge brx and its ports 
+ip netns exec netns3 brctl addbr brx  
+ip netns exec netns3 ip link set brx up
+ip netns exec netns3 ip link set veth13 up
+ip netns exec netns3 ip link set veth23 up
+ip netns exec netns3 brctl addif brx veth13
+ip netns exec netns3 brctl addif brx veth23
+
+# set up bridge bry and its ports 
+ip netns exec netns4 brctl addbr bry  
+ip netns exec netns4 ip link set bry up
+ip netns exec netns4 ip link set veth14 up
+ip netns exec netns4 ip link set veth24 up
+ip netns exec netns4 brctl addif bry veth14
+ip netns exec netns4 brctl addif bry veth24
+
+# create veth devices to connect the bridges
+ip link add vethx type veth peer name vethx11
+ip link add vethy type veth peer name vethy11
+ip link set vethx netns netns3
+ip link set vethx11 netns netns3
+ip link set vethy netns netns4
+ip link set vethy11 netns netns4
+
+ip netns exec netns3 brctl addif brx vethx
+ip netns exec netns3 ip link set vethx up
+ip netns exec netns3 bridge vlan add vid 100 tagged dev vethx
+ip netns exec netns3 bridge vlan add vid 200 tagged dev vethx
+ip netns exec netns3 bridge vlan del vid 1 dev vethx
+ip netns exec netns3 bridge vlan show
+
+ip netns exec netns4 brctl addif bry vethy
+ip netns exec netns4 ip link set vethy up
+ip netns exec netns4 bridge vlan add vid 100 tagged dev vethy
+ip netns exec netns4 bridge vlan add vid 200 tagged dev vethy
+ip netns exec netns4 bridge vlan del vid 1 dev vethy
+ip netns exec netns4 bridge vlan show
+
+ip netns exec netns3 ip link set dev brx type bridge vlan_filtering 1
+ip netns exec netns4 ip link set dev bry type bridge vlan_filtering 1
+ip netns exec netns3 bridge vlan del vid 1 dev brx self
+ip netns exec netns4 bridge vlan del vid 1 dev bry self
+ip netns exec netns3 bridge vlan show
+ip netns exec netns4 bridge vlan show
+
+ip netns exec netns3 bridge vlan add vid 100 pvid untagged dev veth13
+ip netns exec netns3 bridge vlan add vid 200 pvid untagged dev veth23
+ip netns exec netns4 bridge vlan add vid 100 pvid untagged dev veth14
+ip netns exec netns4 bridge vlan add vid 200 pvid untagged dev veth24
+
+ip netns exec netns3 bridge vlan del vid 1 dev veth13
+ip netns exec netns3 bridge vlan del vid 1 dev veth23
+ip netns exec netns4 bridge vlan del vid 1 dev veth14
+ip netns exec netns4 bridge vlan del vid 1 dev veth24
+
+# set up bridge brvx and its ports 
+ip netns exec netns3 brctl addbr brvx  
+ip netns exec netns3 ip link set brvx up
+ip netns exec netns3 ip link set vethx11 up
+ip netns exec netns3 brctl addif brvx vethx11
+
+# set up bridge brvy and its ports 
+ip netns exec netns4 brctl addbr brvy  
+ip netns exec netns4 ip link set brvy up
+ip netns exec netns4 ip link set vethy11 up
+ip netns exec netns4 brctl addif brvy vethy11
+
+# create veth devices to connect the vxlan bridges
+ip link add veth3 type veth peer name veth4
+ip link add veth5 type veth peer name veth6
+ip link set veth3 netns netns3
+ip link set veth5 netns netns4
+ip netns exec netns3 ip link set veth3 up
+ip netns exec netns4 ip link set veth5 up
+ip link set veth4 up
+ip link set veth6 up
+ip netns exec netns3 ifconfig veth3 10.1.1.11/24 up
+ip netns exec netns4 ifconfig veth5 10.1.1.12/24 up
+
+# add vxlan ports
+ip netns exec netns3 ip link add vxlan-10 type vxlan id 10 remote 10.1.1.12 dstport 4789 dev veth3
+ip netns exec netns4 ip link add vxlan-10 type vxlan id 10 remote 10.1.1.11 dstport 4789 dev veth5
+ip netns exec netns3 ip link set vxlan-10 up
+ip netns exec netns4 ip link set vxlan-10 up
+ip netns exec netns3 brctl addif brvx vxlan-10
+ip netns exec netns4 brctl addif brvy vxlan-10
+
+# create veth devices to connect the vxlan bridges
+ip link add veth7 type veth peer name veth8
+ip link set veth7 up
+ip link set veth8 up
+
+# set up bridge brjx and its ports 
+brctl addbr brjx  
+ip link set brjx up
+ip link set veth4 up
+brctl addif brjx veth4
+brctl addif brjx veth7
+
+# set up bridge brjy and its ports 
+brctl addbr brjy  
+ip link set brjy up
+ip link set veth6 up
+brctl addif brjy veth6
+brctl addif brjy veth8

examples/networking/vlan_filter/test_traffic.sh

+#!/bin/bash
+
+ip netns exec netns11 ping 192.168.100.12 -c 10
+ip netns exec netns22 ping 192.168.200.21 -c 10