Add --clone option

clones separate re6stnet babeld git repos for the different machines,
according to clone.conf. Specific commits or branches can be specified
in clone.conf

demo/clone.conf

+# Re6st
+
+# | Commit    | Commit |     Date       | babeld   | babeld      | OpenVPN |
+# |           | number |                | commit   | version     |         |
+# | --------- | ------ | -------------- | -------- | ----------- | ------- |
+# |  cc3b779  |  451   | 21/01/2016     | 284612ac | v1.6.2-nxd1 |         |
+# |  4fda2dc  |  501   | 03/07/2018     |          | v1.6.2-nxd1 | 2.3.18  |
+# |  111a5e4  |  513   | 17/10/2018     |          | v1.6.2-nxd1 | 2.3.18  |
+# |  4397261  |  551   | 13/01/2020     |          | hmac-nxd2   | 2.4.7   |
+
+#re6stnet m1 commit 4397261
+#re6stnet m2 commit 4397261
+#re6stnet m4 commit 4397261
+#re6stnet m5 commit 4397261
+#re6stnet m6 commit 4397261
+#re6stnet m7 commit 4397261
+#re6stnet m8 commit 4397261
+
+# Babeld
+
+#babeld m2 commit v1.6.2-nxd1
+#babeld m3 commit v1.6.2-nxd1
+#babeld m6 commit v1.6.2-nxd1
+
+# OpenVPN
+
+#openvpn m2 commit v2.3.18
+#openvpn m3 commit v2.3.18
+#openvpn m6 commit v2.3.18

demo/demo

@@ -4,6 +4,7 @@ import socket, sqlite3, subprocess, sys, time, weakref
 from collections import defaultdict
 from contextlib import contextmanager
 from threading import Thread
+import thread
 IPTABLES = 'iptables'
 SCREEN = 'screen'
 VERBOSE = 4
@@ -12,11 +13,17 @@ REGISTRY2 = '10.3.0.2'
 REGISTRY_SERIAL = '0x120010db80042'
 REGISTRY2_SERIAL = '0x120010db80043'
 CA_DAYS = 1000
+# global variables for the --clone option
+REPOSITORY_URL = {'re6stnet': 'https://lab.nexedi.com/jhuge/re6stnet.git',
+                  'babeld': 'https://lab.nexedi.com/nexedi/babeld.git',
+                  'openvpn': 'https://github.com/OpenVPN/openvpn.git'}
+BINARY_PATH = {'babeld': 'babeld',
+               'openvpn': 'openvpn/src'}
 
 # Quick check to avoid wasting time if there is an error.
 with open(os.devnull, "wb") as f:
     for x in 're6stnet', 're6st-conf', 're6st-registry':
-        subprocess.check_call(('./py', x, '--help'), stdout=f)
+        subprocess.check_call(('./py', '.', '-', '-', x, '--help'), stdout=f)
 #
 # Underlying network:
 #
@@ -72,6 +79,8 @@ parser.add_argument('-p', '--ping', action = 'store_true',
     help = 'execute ping utility')
 parser.add_argument('-m', '--hmac', action = 'store_true',
     help = 'execute HMAC test')
+parser.add_argument('-c', '--clone', action = 'store_true',
+    help = 'clone separate re6st, babel or openvpn repository as specified in clone.conf')
 args = parser.parse_args()
 
 def handler(signum, frame):
@@ -83,6 +92,18 @@ if args.duration:
 
 execfile("fixnemu.py")
 
+# Dict of machines which needs to clone one of the three repo
+clone_conf = {'re6stnet': {}, 'babeld': {}, 'openvpn': {}}
+if args.clone:
+    # Parse clone configuration file
+    with open('clone.conf', 'r') as clone_conf_file:
+        for line in clone_conf_file:
+            if line[0] == '#' or line[0] == '\n':
+                continue
+            soft, machine, object_type, git_object = line.split()
+            if soft in clone_conf:
+                clone_conf[soft][machine] = (object_type,git_object)
+
 # create nodes
 for name in """internet=I registry=R
                           gateway1=g1 machine1=1 machine2=2
@@ -94,6 +115,71 @@ for name in """internet=I registry=R
     globals()[name] = node = nemu.Node()
     node.name = name
     node.short = short
+
+    # Clone repos specified in clone.conf if --clone option is present
+    if args.clone:
+        if 'machine' in name or 'registry' in name:
+            # Clone the repository if needed
+            short_name = name.replace('machine', 'm')
+
+            # Clone the repository if there isn't a git repository already with
+            # the correct repository URL as specified in REPOSITORY_URL
+            def init_repo(repo):
+
+                if short_name not in clone_conf[repo]:
+                    return
+                repo_folder = '%s/%s' % (short_name, repo)
+
+                def init_repo_cmd():
+                    subprocess.Popen(('rm', '-r', '-f', repo_folder)).wait()
+                    subprocess.Popen(('git', 'clone', REPOSITORY_URL[repo],
+                                      repo_folder)).wait()
+
+                init_needed = False
+                try:
+                    remote_list = subprocess.Popen(('git', 'remote', '-v'),
+                                              stdout=subprocess.PIPE,
+                                              cwd=repo_folder).communicate()[0]
+                    git_toplevel = subprocess.Popen(('git', 'rev-parse', '--show-toplevel'),
+                                              stdout=subprocess.PIPE,
+                                              cwd=repo_folder).communicate()[0][:-1]
+                except OSError:
+                    # Init repo if 'git remote -v' failed
+                    init_needed = True
+                else:
+                    # Check if we are in the correct repository
+                    if git_toplevel != os.path.abspath(repo_folder):
+                        init_needed = True
+                    else:
+                        # Check if repository url is in 'git remote -v' output
+                        remote_list = list(filter(lambda x: REPOSITORY_URL[repo] in x,
+                                             remote_list.split('\n')))
+                        if not remote_list:
+                            init_needed = True
+                        else:
+                            remote_name = remote_list[0].split()[0]
+
+                if init_needed:
+                    init_repo_cmd()
+                    remote_name = 'origin'
+
+                subprocess.Popen(('git', 'fetch', remote_name), cwd=repo_folder).wait()
+                object_type, git_object = clone_conf[repo][short_name]
+                object_path = ('%s/' % remote_name if object_type == 'branch'
+                               else '') + git_object
+                subprocess.Popen(('git', 'checkout', object_path), cwd=repo_folder).wait()
+
+                if repo == 'babeld':
+                    subprocess.Popen(('make'), cwd=repo_folder).wait()
+                elif repo == 'openvpn':
+                    subprocess.Popen(('autoreconf', '-i' ,'-v' ,'-f'), cwd=repo_folder).wait()
+                    subprocess.Popen(('./configure'), cwd=repo_folder).wait()
+                    subprocess.Popen(('make'), cwd=repo_folder).wait()
+
+            init_repo('re6stnet')
+            init_repo('babeld')
+            init_repo('openvpn')
+
     node.Popen(('sysctl', '-q',
         'net.ipv4.icmp_echo_ignore_broadcasts=0')).wait()
     node._screen = node.Popen((SCREEN, '-DmS', name))
@@ -239,9 +325,20 @@ def new_network(registry, reg_addr, serial, ca):
     fingerprint = "sha256:" + hashlib.sha256(
         crypto.dump_certificate(crypto.FILETYPE_ASN1, cert)).hexdigest()
     db_path = "%s/registry.db" % registry.name
-    registry.screen("./py re6st-registry @%s/re6st-registry.conf"
-        " --db %s --mailhost %s -v%u"
-        % (registry.name, db_path, os.path.abspath('mbox'), VERBOSE))
+
+    re6stnet_path = ('demo/%s/re6stnet' % registry.name
+                      if registry.name in clone_conf['re6stnet']
+                      else '.')
+    babeld_path = ('demo/%s/%s' % (registry.name, BINARY_PATH['babeld'])
+                      if registry.name in clone_conf['babeld']
+                      else '-')
+    openvpn_path = ('demo/%s/%s' % (registry.name, BINARY_PATH['openvpn'])
+                      if registry.name in clone_conf['openvpn']
+                      else '-')
+    registry.screen('./py %s %s %s re6st-registry @%s/re6st-registry.conf'
+        ' --db %s --mailhost %s -v%u'
+        % (re6stnet_path, babeld_path, openvpn_path, registry.name, db_path,
+           os.path.abspath('mbox'), VERBOSE))
     registry_url = 'http://%s/' % reg_addr
     registry.Popen(('python', '-c', """if 1:
         import socket, time
@@ -254,14 +351,23 @@ def new_network(registry, reg_addr, serial, ca):
                 time.sleep(.1)
         """)).wait()
     db = sqlite3.connect(db_path, isolation_level=None)
-    def new_node(node, folder, args='', prefix_len=None, registry=registry_url):
+    def new_node(node, folder, node_args='', prefix_len=None, registry=registry_url):
         nodes.append(node)
+        re6stnet_path = ('demo/%s/re6stnet' % folder
+                          if folder in clone_conf['re6stnet']
+                          else '.')
+        babeld_path = ('demo/%s/%s' % (registry.name, BINARY_PATH['babeld'])
+                          if folder in clone_conf['babeld']
+                          else '-')
+        openvpn_path = ('demo/%s/%s' % (registry.name, BINARY_PATH['openvpn'])
+                          if folder in clone_conf['openvpn']
+                          else '-')
         if not os.path.exists(folder + '/cert.crt'):
             dh_path = folder + '/dh2048.pem'
             if not os.path.exists(dh_path):
                 os.symlink('../dh2048.pem', dh_path)
             email = node.name + '@example.com'
-            p = node.Popen(('../py', 're6st-conf', '--registry', registry,
+            p = node.Popen(('../py', re6stnet_path, babeld_path, openvpn_path, 're6st-conf', '--registry', registry,
                             '--email', email, '--fingerprint', fingerprint),
                 stdin=subprocess.PIPE, cwd=folder)
             token = None
@@ -276,9 +382,10 @@ def new_network(registry, reg_addr, serial, ca):
             os.remove(dh_path)
             os.remove(folder + '/ca.crt')
         node.re6st_cmdline = (
-            './py re6stnet @%s/re6stnet.conf -v%u --registry %s'
+            './py %s %s %s re6stnet @%s/re6stnet.conf -v%u --registry %s'
             ' --console %s/run/console.sock %s'
-            ) % (folder, VERBOSE, registry, folder, args)
+            ) % (re6stnet_path, babeld_path, openvpn_path, folder, VERBOSE, registry,
+                 folder, node_args)
         node.screen(node.re6st_cmdline)
     new_node(registry, registry.name, '--ip ' + reg_addr, registry='http://localhost/')
     yield new_node

demo/py

 #!/usr/bin/env python
+# argv[1] : re6stnet repository path
+# argv[2] : babeld repository path
+# argv[3] : openVPN repository path
+# argv[4] : re6st script
+# argv[5:] : re6st script arguments
+
+import os, sys
 def __file__():
-    import argparse, os, sys
+    import argparse
     sys.dont_write_bytecode = True
-    sys.path[0] = os.path.dirname(sys.path[0])
+    # Set path to re6stnet repository root folder
+    sys.path[0] = os.path.join(os.path.dirname(sys.path[0]), sys.argv[1])
     from argparse import ArgumentParser
     _parse_args = ArgumentParser.parse_args
-    ArgumentParser.parse_args = lambda self: _parse_args(self, sys.argv[2:])
+    ArgumentParser.parse_args = lambda self: _parse_args(self, sys.argv[5:])
     # Always import to prevent re6st.node.cli from altering the first sys.path
     from re6st import registry
 
@@ -25,9 +33,16 @@ def __file__():
 
         # Simulate the case of a node that does
         # not renew its certificate in time.
-        if sys.argv[2] == "@m7/re6stnet.conf":
+        if sys.argv[5] == "@m7/re6stnet.conf":
             registry.RENEW_PERIOD = -30
 
-    return os.path.join(sys.path[0], sys.argv[1])
+    return os.path.join(sys.path[0], sys.argv[4])
+
 __file__ = __file__()
+# Export babeld binary folder in $PATH
+if sys.argv[2] != '-':
+    os.environ["PATH"] = "%s:%s" % (os.path.join(sys.path[0], sys.argv[2]), os.environ["PATH"])
+# Export openvpn binary folder in $PATH
+if sys.argv[3] != '-':
+    os.environ["PATH"] = "%s:%s" % (os.path.join(sys.path[0], sys.argv[3]), os.environ["PATH"])
 execfile(__file__)