Speed up bootstrap and reconnection

The way peer addresses were exchanged polluted caches with information about
dead nodes. In particular, bootstrapping often took a long time because the
cache of the primary node was mostly useless.

This also fixes bootstrap of registry.

TODO

@@ -5,12 +5,6 @@
   - the ip address of the network being built
   - the creator of the network ( add option in registry ? )
 
-- Fix bootstrap problem:
-
-  registry & --private option ( see re6stnet man page HOW TO ).
-  one have to start the registry twice, the first time without
-  the --private option
-
 - Babel limitations:
 
   - The metric does not take latency into account.
@@ -27,4 +21,4 @@
 - Filter non-routable IPs. Add an option not to do it.
 
 - Abort in case of import child process failure (babel, openvpn server,
-  openvpn client if run with --client).
\ No newline at end of file
+  openvpn client if run with --client).

demo/registry/re6st-registry.conf

 ca ca.crt
 key registry/ca.key
-private 2001:db8:42::1
 logfile registry/registry.log

docs/re6stnet.rst

@@ -117,25 +117,20 @@ certificates, as follows: translate the significant part to hexadecimal
               -days 365 -out ca.crt
 
 The CA email will be used as sender for mails containing tokens.
-
-Now start the registry in order to setup the main re6st node, which should be
-on the same machine::
+The registry can now be started::
 
   re6st-registry --ca ca.crt --key ca.key --mailhost smtp.example.com
-  re6st-conf --registry http://localhost/
-
-If `re6st-conf` is run in the directory containing CA files, ca.crt will be
-overridden without harm.
 
-Note that the registry was started without specifying the re6st IP of the main
-node, because it was not known yet. For your network to work, it has to be
-restarted with appropriate --private option.
+Like the registry, the first registered node should be always up because its
+presence is used by all other nodes to garantee they are connected to the
+network. It is therefore recommended to run it on the same machine as the
+registry::
 
-Let's suppose your first node is allocated subnet 2001:db8:42::/64.
-Its IP is the first unicast address::
+  re6st-conf --registry http://localhost/
 
-  re6st-registry --private 2001:db8:42::1 ...
-  re6stnet --registry http://localhost/ --ip re6st.example.com ...
+If `re6st-conf` is run in the directory containing CA files, ca.crt will be
+overridden without harm. See previous section for more information to create
+a node.
 
 TROUBLESHOOTING
 ===============

re6st-registry

@@ -75,9 +75,6 @@ def main():
             help="SMTP host to send confirmation emails. For debugging"
                  " purpose, it can also be an absolute or existing path to"
                  " a mailbox file")
-    _('--private',
-            help="re6stnet IP of the node on which runs the registry."
-                 " Required for normal operation.")
     _('--prefix-length', default=16, type=int,
             help="Default length of allocated prefixes.")
     _('--anonymous-prefix-length', type=int,

re6st/db.py

@@ -31,29 +31,24 @@ class PeerDB(object):
         q("INSERT INTO volatile.stat (peer) SELECT prefix FROM peer")
         try:
             a = q("SELECT value FROM config WHERE name='registry'").next()[0]
-        except StopIteration:
-            a = self._updateRegistryIP()
-        else:
-            self.registry_ip = utils.binFromIp(a)
-            if not self.registry_ip.startswith(network):
-                a = self._updateRegistryIP()
-        logging.info("Cache initialized. Registry IP is %s", a)
-
-    def _updateRegistryIP(self):
-        logging.info("Asking registry its private IP...")
-        retry = 1
-        while True:
-            try:
-                a = self._registry.getPrivateAddress(self._prefix)
-                break
-            except socket.error, e:
-                logging.warning(e)
-                time.sleep(retry)
-                retry = min(60, retry * 2)
-        self._db.execute("INSERT OR REPLACE INTO config VALUES ('registry',?)",
-                         (a,))
-        self.registry_ip = utils.binFromIp(a)
-        return a
+            int(a, 2)
+        except (StopIteration, ValueError):
+            logging.info("Asking registry its private IP...")
+            retry = 1
+            while True:
+                try:
+                    a = self._registry.getPrefix(self._prefix)
+                    int(a, 2)
+                    break
+                except socket.error, e:
+                    logging.warning(e)
+                    time.sleep(retry)
+                    retry = min(60, retry * 2)
+            q("INSERT OR REPLACE INTO config VALUES ('registry',?)", (a,))
+            self._db.commit()
+        self.registry_prefix = a
+        logging.info("Cache initialized. Prefix of registry node is %s/%u",
+                     int(a, 2), len(a))
 
     def log(self):
         if logging.getLogger().isEnabledFor(5):
@@ -64,6 +59,19 @@ class PeerDB(object):
                 logging.trace("- %s: %s%s", prefix, address,
                               ' (blacklisted)' if _try else '')
 
+    def cacheMinimize(self, size):
+        with self._db:
+            self._cacheMinimize(size)
+
+    def _cacheMinimize(self, size):
+        a = self._db.execute(
+            "SELECT peer FROM volatile.stat ORDER BY try, RANDOM() LIMIT ?,-1",
+            (size,)).fetchall()
+        if a:
+            q = self._db.executemany
+            q("DELETE FROM peer WHERE prefix IN (?)", a)
+            q("DELETE FROM volatile.stat WHERE peer IN (?)", a)
+
     def connecting(self, prefix, connecting):
         self._db.execute("UPDATE volatile.stat SET try=? WHERE peer=?",
                          (connecting, prefix))
@@ -119,13 +127,8 @@ class PeerDB(object):
                         return len(preferred)
                 address = ';'.join(sorted(address.split(';'), key=key))
             except ValueError:
-                a = q("SELECT peer FROM volatile.stat ORDER BY try, RANDOM()"
-                      " LIMIT ?,-1", (self._db_size,)).fetchall()
-                if a:
-                    qq = self._db.executemany
-                    qq("DELETE FROM peer WHERE prefix IN (?)", a)
-                    qq("DELETE FROM volatile.stat WHERE peer IN (?)", a)
-                # 'a != address' will evaluate to True because types differs
+                self._cacheMinimize(self._db_size)
+                a = None
             if a != address:
                 q("INSERT OR REPLACE INTO peer VALUES (?,?)", (prefix, address))
             q("INSERT OR REPLACE INTO volatile.stat VALUES (?,0)", (prefix,))

re6st/registry.py

@@ -30,24 +30,27 @@ class getcallargs(type):
 class RegistryServer(object):
 
     __metaclass__ = getcallargs
+    _peers = 0, ()
 
     def __init__(self, config):
         self.config = config
         self.cert_duration = 365 * 86400
         self.lock = threading.Lock()
         self.sessions = {}
-
-        if self.config.private:
-            self.sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
-        else:
-            logging.warning('You have declared no private address'
-                    ', either this is the first start, or you should'
-                    'check you configuration')
+        self.sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
 
         # Database initializing
         utils.makedirs(os.path.dirname(self.config.db))
         self.db = sqlite3.connect(self.config.db, isolation_level=None,
                                                   check_same_thread=False)
+        self.db.execute("""CREATE TABLE IF NOT EXISTS config (
+                        name text primary key,
+                        value text)""")
+        try:
+            (self.prefix,), = self.db.execute(
+                "SELECT value FROM config WHERE name='prefix'")
+        except ValueError:
+            self.prefix = None
         self.db.execute("""CREATE TABLE IF NOT EXISTS token (
                         token text primary key not null,
                         email text not null,
@@ -157,7 +160,7 @@ class RegistryServer(object):
             s.sendmail(self._email, email, msg.as_string())
             s.quit()
 
-    def _getPrefix(self, prefix_len):
+    def _newPrefix(self, prefix_len):
         max_len = 128 - len(self.network)
         assert 0 < prefix_len <= max_len
         try:
@@ -173,7 +176,7 @@ class RegistryServer(object):
         if len(prefix) < max_len or '1' in prefix:
             return prefix
         self.db.execute("UPDATE cert SET cert = 'reserved' WHERE prefix = ?", (prefix,))
-        return self._getPrefix(prefix_len)
+        return self._newPrefix(prefix_len)
 
     def requestCertificate(self, token, req):
         req = crypto.load_certificate_request(crypto.FILETYPE_PEM, req)
@@ -193,9 +196,13 @@ class RegistryServer(object):
                     if not prefix_len:
                         return
                     email = None
-                prefix = self._getPrefix(prefix_len)
+                prefix = self._newPrefix(prefix_len)
                 self.db.execute("UPDATE cert SET email = ? WHERE prefix = ?",
                                 (email, prefix))
+                if self.prefix is None:
+                    self.prefix = prefix
+                    self.db.execute(
+                        "INSERT INTO config VALUES ('prefix',?)", (prefix,))
                 return self._createCertificate(prefix, req.get_subject(),
                                                        req.get_pubkey())
 
@@ -227,37 +234,51 @@ class RegistryServer(object):
     def getCa(self):
         return crypto.dump_certificate(crypto.FILETYPE_PEM, self.ca)
 
+    def getPrefix(self, cn):
+        return self.prefix
+
     def getPrivateAddress(self, cn):
-        return self.config.private
+        # BBB: Deprecated by getPrefix.
+        return utils.ipFromBin(self.network + self.prefix)
 
     def getBootstrapPeer(self, cn):
         with self.lock:
             cert = self._getCert(cn)
-            address = self.config.private, tunnel.PORT
+            age, peers = self._peers
+            if time.time() < age or not peers:
+                peers = [x[1] for x in utils.iterRoutes(self.network)]
+                random.shuffle(peers)
+                self._peers = time.time() + 60, peers
+            peer = peers.pop()
+            if peer == cn:
+                # Very unlikely (e.g. peer restarted with empty cache),
+                # so don't bother looping over above code
+                # (in case 'peers' is empty).
+                peer = self.prefix
+            address = utils.ipFromBin(self.network + peer), tunnel.PORT
             self.sock.sendto('\2', address)
-            peer = None
-            while select.select([self.sock], [], [], peer is None)[0]:
+            start = time.time()
+            timeout = 1
+            # Loop because there may be answers from previous requests.
+            while select.select([self.sock], [], [], timeout)[0]:
                 msg = self.sock.recv(1<<16)
                 if msg[0] == '\1':
                     try:
-                        peer = msg[1:].split('\n')[-2]
-                    except IndexError:
-                        peer = ''
-        if peer is None:
-            raise EnvironmentError("Timeout while querying [%s]:%u" % address)
-        if not peer or peer.split()[0] == cn:
-            raise LookupError("No bootstrap peer found")
-        logging.info("Sending bootstrap peer: %s", peer)
-        return utils.encrypt(cert, peer)
+                        msg = msg[1:msg.index('\n')]
+                    except ValueError:
+                        continue
+                    if msg.split()[0] == peer:
+                        break
+                timeout = max(0, time.time() - start)
+            else:
+                raise EnvironmentError("Timeout while querying [%s]:%u"
+                                       % address)
+        logging.info("Sending bootstrap peer: %s", msg)
+        return utils.encrypt(cert, msg)
 
     def topology(self):
         with self.lock:
-            is_registry = utils.binFromIp(self.config.private
-                )[len(self.network):].startswith
-            peers = deque('%u/%u' % (int(x, 2), len(x))
-                for x, in self.db.execute("SELECT prefix FROM cert")
-                if is_registry(x))
-            assert len(peers) == 1
+            peers = deque(('%u/%u' % (int(self.prefix, 2), len(self.prefix)),))
             cookie = hex(random.randint(0, 1<<32))[2:]
             graph = dict.fromkeys(peers)
             asked = 0

re6st/tunnel.py

 import logging, random, socket, subprocess, time
 from collections import deque
-from itertools import chain
 from . import plib, utils
 
 PORT = 326
-RTF_CACHE = 0x01000000  # cache entry
 
 # Be careful the refresh interval should let the routes be established
 
@@ -278,8 +276,6 @@ class TunnelManager(object):
                     count -= self._makeTunnel(peer, address)
                 else:
                     ip = utils.ipFromBin(self._network + peer)
-                    # TODO: Send at least 1 address. This helps the registry
-                    #       node filling its cache when building a new network.
                     try:
                         self.sock.sendto('\2', (ip, PORT))
                     except socket.error, e:
@@ -317,36 +313,21 @@ class TunnelManager(object):
         del self._distant_peers[:]
         for conn in self._connection_dict.itervalues():
             conn.routes = 0
-        a = len(self._network)
-        b = a + len(self._prefix)
         other = []
-        with open('/proc/net/ipv6_route') as f:
-            self._last_routing_table = f.read()
-            for line in self._last_routing_table.splitlines():
-                line = line.split()
-                iface = line[-1]
-                if iface == 'lo' or int(line[-2], 16) & RTF_CACHE:
-                    continue
-                ip = bin(int(line[0], 16))[2:].rjust(128, '0')
-                if ip[:a] != self._network or ip[a:b] == self._prefix:
-                    continue
-                prefix_len = int(line[1], 16)
-                prefix = ip[a:prefix_len]
-                logging.trace('Route on iface %s detected to %s/%u',
-                              iface, utils.ipFromBin(ip), prefix_len)
-                nexthop = self._iface_to_prefix.get(iface)
-                if nexthop:
-                    self._connection_dict[nexthop].routes += 1
-                if prefix in self._served or prefix in self._connection_dict:
-                    continue
-                if iface in self._iface_list:
-                    other.append(prefix)
-                else:
-                    self._distant_peers.append(prefix)
-        is_registry = self._peer_db.registry_ip[a:].startswith
-        if is_registry(self._prefix) or any(is_registry(peer)
-              for peer in chain(self._distant_peers, other,
-                                self._served, self._connection_dict)):
+        for iface, prefix in utils.iterRoutes(self._network, self._prefix):
+            assert iface != 'lo', (iface, prefix)
+            nexthop = self._iface_to_prefix.get(iface)
+            if nexthop:
+                self._connection_dict[nexthop].routes += 1
+            if prefix in self._served or prefix in self._connection_dict:
+                continue
+            if iface in self._iface_list:
+                other.append(prefix)
+            else:
+                self._distant_peers.append(prefix)
+        registry = self._peer_db.registry_prefix
+        if registry == self._prefix or any(registry in x for x in (
+              self._distant_peers, other, self._served, self._connection_dict)):
             self._disconnected = None
             # XXX: When there is no new peer to connect when looking at routes
             #      coming from tunnels, we'd like to consider those discovered
@@ -410,37 +391,34 @@ class TunnelManager(object):
             return
         code = ord(msg[0])
         if code == 1: # answer
-            # We parse the message in a way to discard a truncated line.
-            for peer in msg[1:].split('\n')[:-1]:
-                try:
-                    prefix, address = peer.split()
-                    int(prefix, 2)
-                except ValueError:
-                    break
+            # Old versions may send additional and obsolete addresses.
+            # Ignore them, as well as truncated lines.
+            try:
+                prefix, address = msg[1:msg.index('\n')].split()
+                int(prefix, 2)
+            except ValueError:
+                pass
+            else:
                 if prefix != self._prefix:
                     self._peer_db.addPeer(prefix, address)
                     try:
                         self._connecting.remove(prefix)
                     except KeyError:
-                        continue
-                    self._makeTunnel(prefix, address)
+                        pass
+                    else:
+                        self._makeTunnel(prefix, address)
         elif code == 2: # request
-            encode = '%s %s\n'.__mod__
             if self._address:
-                msg = [encode((self._prefix, self._address))]
-            else: # I don't know my IP yet!
-                msg = []
-            # Add an extra random peer, mainly for the registry.
-            if random.randint(0, self._peer_db.getPeerCount()):
-                msg.append(encode(self._peer_db.getPeerList().next()))
-            if msg:
+                msg = '\1%s %s\n' % (self._prefix, self._address)
                 try:
-                    self.sock.sendto('\1' + ''.join(msg), address[:2])
+                    self.sock.sendto(msg, address[:2])
                 except socket.error, e:
                     logging.info('Failed to reply to %s (%s)', address, e)
+            #else: # I don't know my IP yet!
         elif code == 255:
             # the registry wants to know the topology for debugging purpose
-            if utils.binFromIp(address[0]) == self._peer_db.registry_ip:
+            if utils.binFromIp(address[0])[len(self._network):].startswith(
+                  self._peer_db.registry_prefix):
                 msg = ['\xfe%s%u/%u\n%u\n' % (msg[1:],
                     int(self._prefix, 2), len(self._prefix),
                     len(self._connection_dict))]

re6st/utils.py

@@ -155,6 +155,33 @@ def binFromSubnet(subnet):
     p, l = subnet.split('/')
     return bin(int(p))[2:].rjust(int(l), '0')
 
+if 1:
+    def _iterRoutes():
+        with open('/proc/net/ipv6_route') as f:
+            routing_table = f.read()
+        for line in routing_table.splitlines():
+            line = line.split()
+            iface = line[-1]
+            if 0 < int(line[5], 16) < 1 << 31: # positive metric
+                yield (iface, bin(int(line[0], 16))[2:].rjust(128, '0'),
+                              int(line[1], 16))
+
+_iterRoutes.__doc__ = """Iterates over all routes
+
+    Amongst all returned routes starting with re6st prefix:
+    - one is the local one with our prefix
+    - any route with null prefix will be ignored
+    - other are reachable routes installed by babeld
+    """
+
+def iterRoutes(network, exclude_prefix=None):
+    a = len(network)
+    for iface, ip, prefix_len in _iterRoutes():
+        if ip[:a] == network:
+            prefix = ip[a:prefix_len]
+            if prefix and prefix != exclude_prefix:
+                yield iface, prefix
+
 def decrypt(key_path, data):
     p = subprocess.Popen(
         ('openssl', 'rsautl', '-decrypt', '-inkey', key_path),

re6stnet

@@ -278,6 +278,7 @@ def main():
             r_pipe, write_pipe = os.pipe()
             read_pipe = os.fdopen(r_pipe)
             peer_db = db.PeerDB(db_path, registry, config.key, network, prefix)
+            cleanup.append(lambda: peer_db.cacheMinimize(config.client_count))
             tunnel_manager = tunnel.TunnelManager(write_pipe, peer_db,
                 config.openvpn_args, timeout, config.tunnel_refresh,
                 config.client_count, config.iface_list, network, prefix,