Commit a85c8c02 authored by Julien Muchembled's avatar Julien Muchembled

Change protocol to discover addresses of peers to connect to

The previous broadcast model is replaced by a query-response one.
During normal operation, the cache of peers is not used anymore to select
peers to connect to. It now only used for bootstrapping and avoid querying
an already known address.
parent 09ec1327
......@@ -12,19 +12,3 @@
the --private option
- Put a section about how to build the package from the sources in the README
- review protocol to discover pair:
A node that that wants to establish a new tunnel just have to look at the
routing table and pick a random destination node, actually its re6nst subnet.
If its address is unknown (not in local cache), re6st queries it by sending a
UDP packet to the subnet (anycast, i.e. IP suffix is 0): the reply is also an
UDP packet containing the address string to establish a tunnel. Then, the
tunnel can be established.
For bootstrapping, no destination is known so one has to query the registry
as already implemented. The way the registry discovers node addresses should
be changed so that nodes don't need to declare themselves anymore.
The registry should look at the local peer DB cache.
This new protocol should produce less network trafic than the current one.
......@@ -3,7 +3,7 @@ import math, nemu, os, signal, socket, subprocess, sys, time
from collections import defaultdict
IPTABLES = 'iptables'
SCREEN = 'screen'
# registry
# |.2
......@@ -132,16 +132,17 @@ for ip in '', '', '', '':
print "Connectivity IPv4 OK!"
registry.screen('../re6stnet @registry/re6stnet.conf --ip -v%u' % VERBOSE,
'../re6st-registry @registry/re6st-registry.conf')
gateway1.screen('miniupnpd -d -f miniupnpd.conf -P -a'
' -i %s' % g1_if_0_name)
machine1.screen('../re6stnet @m1/re6stnet.conf -v%u' % VERBOSE)
machine2.screen('../re6stnet @m2/re6stnet.conf -v%u' % VERBOSE)
machine3.screen('../re6stnet @m3/re6stnet.conf -v%u -i%s' % (VERBOSE,
machine4.screen('../re6stnet @m4/re6stnet.conf -v%u -i%s' % (VERBOSE,
machine5.screen('../re6stnet @m5/re6stnet.conf -v%u' % VERBOSE)
machine6.screen('../re6stnet @m6/re6stnet.conf -v%u' % VERBOSE)
if 1:
registry.screen('../re6stnet @registry/re6stnet.conf --ip -v%u' % VERBOSE,
'../re6st-registry @registry/re6st-registry.conf')
machine1.screen('../re6stnet @m1/re6stnet.conf -v%u' % VERBOSE)
machine2.screen('../re6stnet @m2/re6stnet.conf -v%u' % VERBOSE)
machine3.screen('../re6stnet @m3/re6stnet.conf -v%u -i%s' % (VERBOSE,
machine4.screen('../re6stnet @m4/re6stnet.conf -v%u -i%s' % (VERBOSE,
machine5.screen('../re6stnet @m5/re6stnet.conf -v%u' % VERBOSE)
machine6.screen('../re6stnet @m6/re6stnet.conf -v%u' % VERBOSE)
nodes = registry, machine1, machine2, machine3, machine4, machine5, machine6
_ll = {}
PRAGMA foreign_keys=OFF;
prefix text primary key not null,
address text not null,
date integer default (strftime('%s','now')));
token text primary key not null,
email text not null,
......@@ -190,5 +186,4 @@ rlyT1Q==
INSERT INTO "cert" VALUES('0000000000001001',NULL,NULL);
INSERT INTO "cert" VALUES('000000000000101',NULL,NULL);
INSERT INTO "cert" VALUES('00000000000011',NULL,NULL);
CREATE INDEX peers_ping ON peers(date);
......@@ -4,7 +4,7 @@ import subprocess, time, threading, traceback, errno, logging, os, xmlrpclib
from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
from email.mime.text import MIMEText
from OpenSSL import crypto
from re6st import utils
from re6st import tunnel, utils
# To generate server ca and key with serial for 2001:db8:42::/48
# openssl req -nodes -new -x509 -key ca.key -set_serial 0x120010db80042 -days 365 -out ca.crt
......@@ -64,18 +64,15 @@ class main(object):
help='VPN IP of the node on which runs the registry')
self.config = parser.parse_args()
if not self.config.private:
if self.config.private:
self.sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
logging.warning('You have declared no private address'
', either this is the first start, or you should'
'check you configuration')
# Database initializing
self.db = sqlite3.connect(self.config.db, isolation_level=None)
self.db.execute("""CREATE TABLE IF NOT EXISTS peers (
prefix text primary key not null,
address text not null,
date integer default (strftime('%s','now')))""")
self.db.execute("CREATE INDEX IF NOT EXISTS peers_ping ON peers(date)")
self.db.execute("""CREATE TABLE IF NOT EXISTS token (
token text primary key not null,
email text not null,
......@@ -196,40 +193,35 @@ class main(object):
return crypto.dump_certificate(crypto.FILETYPE_PEM,
def getPrivateAddress(self, handler):
return 'http://[%s]:%u' % (self.config.private, self.config.port)
return self.config.private
def getBootstrapPeer(self, handler, client_prefix):
cert, = self.db.execute("SELECT cert FROM cert WHERE prefix = ?",
prefix, address = self.db.execute("""SELECT prefix, address FROM peers
WHERE prefix != ? ORDER BY random() LIMIT 1""", (client_prefix,)).next()
except StopIteration:'No peer to send for bootstrap')
address = self.config.private, tunnel.PORT
self.sock.sendto('\2', address)
peer = None
while[self.sock], [], [], peer is None)[0]:
msg = self.sock.recv(1<<16)
if msg[0] == '\1':
peer = msg[1:].split('\n')[-2]
except IndexError:
peer = ''
if peer is None:
raise EnvironmentError("Timeout while querying [%s]:%u", *address)
if not peer or peer.split()[0] == client_prefix:
raise LookupError("No bootstrap peer found")"Sending bootstrap peer: %s", peer)
r, w = os.pipe()
threading.Thread(target=os.write, args=(w, cert)).start()
p = subprocess.Popen(('openssl', 'rsautl', '-encrypt', '-certin', '-inkey', '/proc/self/fd/%u' % r),
stdin=subprocess.PIPE, stdout=subprocess.PIPE)"Sending bootstrap peer (%s, %s)" % (prefix, address))
return xmlrpclib.Binary(p.communicate('%s %s' % (prefix, address))[0])
return xmlrpclib.Binary(p.communicate(peer)[0])
def declare(self, handler, address):
client_address, _, _, _ = handler.client_address
client_ip = utils.binFromIp(client_address)
if client_ip.startswith(
prefix = client_ip[len(]
prefix, = self.db.execute("SELECT prefix FROM cert WHERE prefix <= ? ORDER BY prefix DESC LIMIT 1", (prefix,)).next()
self.db.execute("INSERT OR REPLACE INTO peers (prefix, address) VALUES (?,?)", (prefix, address))
return True
logging.warning("Unauthorized connection from %s which does not start with %s"
% (utils.ipFromBin(client_ip), utils.ipFromBin(, '0'))))
return False
if __name__ == "__main__":
This diff is collapsed.
......@@ -5,4 +5,4 @@ if os.environ['script_type'] == 'up':
os.execlp('ip', 'ip', 'link', 'set', os.environ['dev'], 'up')
# Write into pipe external ip address received
os.write(int(sys.argv[1]), '%(script_type)s %(OPENVPN_external_ip)s\n' % os.environ)
os.write(int(sys.argv[1]), '%(script_type)s %(common_name)s %(OPENVPN_external_ip)s\n' % os.environ)
......@@ -74,7 +74,7 @@ def router(network, subnet, subnet_size, interface_list,
'-C', 'out local ip %s/%s le %s' % (subnet, subnet_size, subnet_size),
'-C', 'out local deny',
# Route VIFIB ip adresses
'-C', 'in ip %s::/%u' % (utils.ipFromBin(network), len(network)),
'-C', 'in ip %s/%u' % (utils.ipFromBin(network), len(network)),
# Route only addresse in the 'local' network,
# or other entire networks
#'-C', 'in ip %s' % (config.internal_ip),
This diff is collapsed.
......@@ -4,10 +4,13 @@ from OpenSSL import crypto
logging_levels = logging.WARNING, logging.INFO, logging.DEBUG, 5
def setupLog(log_level):
def setupLog(log_level, **kw):
if log_level:
format='%(asctime)s %(levelname)-9s %(message)s',
datefmt='%d-%m-%Y %H:%M:%S')
datefmt='%d-%m-%Y %H:%M:%S', **kw)
logging.addLevelName(5, 'TRACE')
logging.trace = lambda *args, **kw: logging.log(5, *args, **kw)
......@@ -28,19 +31,10 @@ def binFromIp(ip):
return bin(ip1)[2:].rjust(64, '0') + bin(ip2)[2:].rjust(64, '0')
def ipFromBin(prefix):
prefix = hex(int(prefix, 2))[2:]
ip = ''
for i in xrange(0, len(prefix) - 1, 4):
ip += prefix[i:i + 4] + ':'
return ip.rstrip(':')
def ipFromPrefix(re6stnet, prefix, prefix_len):
prefix = bin(int(prefix))[2:].rjust(prefix_len, '0')
ip_t = (re6stnet + prefix).ljust(127, '0').ljust(128, '1')
return ipFromBin(ip_t), prefix
def ipFromBin(prefix, suffix=''):
ip = prefix + suffix.rjust(128 - len(prefix), '0')
return socket.inet_ntop(socket.AF_INET6,
struct.pack('>QQ', int(ip[:64], 2), int(ip[64:], 2)))
def networkFromCa(ca_path):
# Get network prefix from ca.crt
......@@ -48,15 +42,14 @@ def networkFromCa(ca_path):
ca = crypto.load_certificate(crypto.FILETYPE_PEM,
return bin(ca.get_serial_number())[3:]
def ipFromCert(network, cert_path):
# Get ip from cert.crt
with open(cert_path, 'r') as f:
cert = crypto.load_certificate(crypto.FILETYPE_PEM,
subject = cert.get_subject()
prefix, prefix_len = subject.CN.split('/')
return ipFromPrefix(network, prefix, int(prefix_len))
prefix = bin(int(prefix))[2:].rjust(int(prefix_len), '0')
return ipFromBin(network + prefix, '1'), prefix
def address_str(address):
return ';'.join(map(','.join, address))
......@@ -68,7 +61,5 @@ def address_list(address_list):
def binFromSubnet(subnet):
prefix, subnet_size = subnet.split('/')
binary = bin(int(prefix))[2:]
binary = ('0' * (int(subnet_size) - len(binary))) + binary
return binary
p, l = subnet.split('/')
return bin(int(p))[2:].rjust(int(l), '0')
#!/usr/bin/env python
import atexit, os, sys, select, socket, time
import atexit, os, sys, select, time
import argparse, signal, subprocess, sqlite3, logging, traceback
from re6st import plib, utils, db, tunnel
......@@ -27,13 +27,11 @@ def getConfig():
_('--registry', required=True,
help="HTTP URL of the discovery peer server,"
" with public host (default port: 80)")
_('--peers-db-refresh', default=43200, type=int,
help='the time (seconds) to wait before refreshing the peers db')
_('-l', '--log', default='/var/log',
help='Path to re6stnet logs directory')
_('-s', '--state', default='/var/lib/re6stnet',
help='Path to re6stnet state directory')
_('-v', '--verbose', default=0, type=int,
_('-v', '--verbose', default=1, type=int,
help='Log level of re6st itself')
_('-i', '--interface', action='append', dest='iface_list', default=[],
help='Extra interface for LAN discovery')
......@@ -83,7 +81,8 @@ def main():
db_path = os.path.join(config.state, 'peers.db')
# Set logging
filename=os.path.join(config.log, 're6stnet.log'))
logging.trace("Configuration :\n%s" % config)
......@@ -124,13 +123,11 @@ def main():
if address:
ip_changed = None
peer_db = db.PeerManager(db_path, config.registry, config.key,
config.peers_db_refresh, address, internal_ip, prefix,
ip_changed, 200)
peer_db = db.PeerDB(db_path, config.registry, config.key, prefix)
tunnel_manager = tunnel.TunnelManager(write_pipe, peer_db, openvpn_args,
config.hello, config.tunnel_refresh, config.connection_count,
config.iface_list, network, prefix, 2, config.encrypt)
peer_db.tunnel_manager = tunnel_manager
config.hello, config.tunnel_refresh, config.connection_count,
config.iface_list, network, prefix, address, ip_changed,
# Launch routing protocol. WARNING : you have to be root to start babeld
server_tunnels = {}
......@@ -156,24 +153,24 @@ def main():
config.connection_count, config.dh, write_pipe, port,
proto, config.hello, config.encrypt, *openvpn_args))
while True:
nextUpdate = min(tunnel_manager.next_refresh, peer_db.next_refresh)
next = tunnel_manager.next_refresh
if forwarder:
nextUpdate = min(nextUpdate, forwarder.next_refresh)
nextUpdate = max(0, nextUpdate - time.time())
select_list = [read_pipe]
if peer_db.socket_file:
ready, tmp1, tmp2 =, [], [], nextUpdate)
if read_pipe in ready:
if time.time() >= peer_db.next_refresh:
if time.time() >= tunnel_manager.next_refresh:
next = min(next, forwarder.next_refresh)
r = [read_pipe, tunnel_manager.sock]
r =, [], [], max(0, next - time.time()))[0]
if read_pipe in r:
if tunnel_manager.sock in r:
t = time.time()
if t >= tunnel_manager.next_refresh:
if forwarder and time.time() > forwarder.next_refresh:
if forwarder and t >= forwarder.next_refresh:
if peer_db.socket_file in ready:
except Exception:
f = traceback.format_exception(*sys.exc_info())
logging.error('%s%s', f.pop(), ''.join(f))
for p in server_process:
......@@ -186,7 +183,6 @@ def main():
except sqlite3.Error:
os.rename(db_path, db_path + '.bak')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment