software/theia: Add import script

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 6ce2d9f0c32e3b781fab5d59685dc866
+md5sum = e0fb65c34b2d3d524cbca85579728e54
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
@@ -23,7 +23,7 @@ md5sum = a7d78b4002266c69ece05a476df82791
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = d1181f788461a31026e36677ab56398d
+md5sum = b1e1a31b0ca3fc0be7de259f05f80c5d
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in
@@ -41,6 +41,10 @@ md5sum = e57396473b4b6a17d26a747f0030293c
 _update_hash_filename_ = theia_export.py
 md5sum = b5f5ac1924b27d3f2be2e5ea291c119e
 
+[theia-import]
+_update_hash_filename_ = theia_import.py
+md5sum = 9e8c17a4b2d802695caf0c2c052f0d11
+
 [yarn.lock]
 _update_hash_filename_ = yarn.lock
 md5sum = 80e7ad91deea54cebcccef5a83fdb380

software/theia/instance-import.cfg.jinja.in

@@ -59,16 +59,64 @@ seed = Import {{ root_title }}
 # in post-notification-run:output and in importer:wrapper.
 
 [post-notification-run]
-recipe = slapos.cookbook:wrapper
-command-line = echo "Import Post Notification Run Not Implemented Yet"
-wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-post-run
-wrapper = $${:wrapper-path}
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/post-notification-run-script
+output = $${:rendered}
+mode = 0700
+template =
+  inline:#!${software-info:bash}
+  # Do nothing because the backup signature will
+  # be verified by the import script itself
+  exit 0
 
 [importer]
-recipe = slapos.cookbook:wrapper
-command-line = echo "Import Not Implemented Yet"
-wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-importer
-wrapper = $${:wrapper-path}
+wrapper = $${theia-import-script:rendered}
+
+[theia-import-script]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/theia-import-script
+mode = 0700
+exitcode-file = $${directory:srv}/import-exitcode-file
+error-file = $${directory:srv}/import-errormessage-file
+context =
+  raw python ${software-info:python-with-eggs}
+  raw theia_import ${software-info:theia-import}
+  raw bash ${software-info:bash}
+  raw rsync ${software-info:rsync}
+  raw sqlite3 ${software-info:sqlite3}
+  raw slapos ${software-info:slapos}
+  raw slapos_node_software_log $${directory:runner}/var/log/slapos-node-software.log
+  raw slapos_node_instance_log $${directory:runner}/var/log/slapos-node-instance.log
+  raw supervisorctl ${software-info:supervisorctl}
+  raw supervisord_conf $${directory:runner}/etc/supervisord.conf
+  raw root_path $${buildout:directory}
+  raw backup_path $${directory:backup}
+  raw slapos_cfg $${directory:runner}/etc/slapos.cfg
+  raw project_path $${directory:project}
+  raw public_path $${directory:frontend-static-public}
+  key exitfile :exitcode-file
+  key errorfile :error-file
+{%- raw %}
+template =
+  inline:#!{{ bash }}
+  . $${common-environment:rendered}
+  . $${slapos-standalone-activate:rendered}
+  {{ python }} {{ theia_import }} \
+  --rsync {{ rsync }} \
+  --sqlite3 {{ sqlite3 }} \
+  --slapos {{ slapos }} \
+  --srlog {{ slapos_node_software_log }} \
+  --cplog {{ slapos_node_instance_log }} \
+  --supervisorctl {{ supervisorctl }} \
+  --supervisordconf {{ supervisord_conf }} \
+  --root {{ root_path }} \
+  --backup {{ backup_path }} \
+  --cfg {{ slapos_cfg }} \
+  --dirs {{ project_path }} \
+  --dirs {{ public_path }} \
+  --exitfile {{ exitfile }} \
+  --errorfile {{ errorfile }}
+{%- endraw %}
 
 
 # Resilient connection parameters of import instance are published

software/theia/instance-theia.cfg.jinja.in

@@ -332,7 +332,7 @@ mode = 0700
 template =
   inline:
   #!/bin/sh
-  export HOME=$${buildout:directory}
+  export HOME=$${directory:home}
   export PATH=${python-language-server:location}/bin:${java-jdk:location}/bin:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
 
 
@@ -441,6 +441,7 @@ ip = {{ ipv4_random }}
 ipv4 = {{ ipv4_random }}
 ipv6 = {{ ipv6_random }}
 port = $${slapos-standalone-port:port}
+local-software-release-root = $${directory:home}
 slapos-configuration = $${directory:runner}/etc/slapos.cfg
 computer-id = slaprunner
 
@@ -469,6 +470,7 @@ template =
       $${slapos-standalone-config:ipv4} \
       $${slapos-standalone-config:ipv6} \
       $${slapos-standalone-config:port} \
+      $${slapos-standalone-config:local-software-release-root} \
       $${slapos-standalone-config:computer-id} \
       {%- if parameter_dict.get('embedded-sr') %}
       --sr='{{ parameter_dict['embedded-sr'] }}' \

software/theia/software.cfg

@@ -74,6 +74,7 @@ initialization =
   parser.add_argument('ipv4')
   parser.add_argument('ipv6')
   parser.add_argument('server_port', type=int)
+  parser.add_argument('local_software_release_root')
   parser.add_argument('computer_id')
   parser.add_argument('--sr')
   parser.add_argument('--srtype')
@@ -109,6 +110,7 @@ initialization =
       instance_root="%s/instance" % args.base_directory,
       partition_forward_configuration=partition_forward_configuration,
       slapos_bin="${buildout:bin-directory}/slapos",
+      local_software_release_root=args.local_software_release_root,
   )
 
   def signal_handler(signum, frame):
@@ -395,10 +397,17 @@ destination = ${buildout:directory}/theia_common.py
 <= download-base
 destination = ${buildout:directory}/theia_export.py
 
+[theia-import]
+<= download-base
+destination = ${buildout:directory}/theia_import.py
+
 [software-info]
+slapos = ${buildout:bin-directory}/slapos
 python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
 python = ${python:location}/bin/python
 rsync = ${rsync:location}/bin/rsync
 sqlite3 = ${sqlite3:location}/bin/sqlite3
 bash = ${bash:location}/bin/bash
+supervisorctl = ${buildout:bin-directory}/supervisorctl
 theia-export = ${theia-export:output}
+theia-import = ${theia-import:output}

software/theia/theia_import.py

+import argparse
+import glob
+import itertools
+import os
+import sys
+import subprocess as sp
+import traceback
+
+import six
+from six.moves import configparser
+
+sys.path.append(os.path.dirname(__file__))
+from theia_common import copytree, copydb, hashwalk, parse_installed, remove
+
+
+os.environ['LC_ALL'] = 'C'
+os.umask(0o77)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--rsync', required=True)
+  parser.add_argument('--sqlite3', required=True)
+  parser.add_argument('--slapos', required=True)
+  parser.add_argument('--srlog', required=True)
+  parser.add_argument('--cplog', required=True)
+  parser.add_argument('--supervisorctl', required=True)
+  parser.add_argument('--supervisordconf', required=True)
+  parser.add_argument('--root', required=True)
+  parser.add_argument('--backup', required=True)
+  parser.add_argument('--cfg', required=True)
+  parser.add_argument('--dirs', action='append')
+  parser.add_argument('--exitfile', required=True)
+  parser.add_argument('--errorfile', required=True)
+  args = parser.parse_args()
+
+  TheiaImport(args)()
+
+
+class TheiaImport(object):
+
+  def __init__(self, args):
+    self.rsync_bin = args.rsync
+    self.sqlite3_bin = args.sqlite3
+    self.slapos_bin = args.slapos
+    self.sr_log = args.srlog
+    self.cp_log = args.cplog
+    self.supervisorctl_bin = args.supervisorctl
+    self.supervisord_conf = args.supervisordconf
+    self.root_dir = args.root
+    self.backup_dir = args.backup
+    self.slapos_cfg = cfg = args.cfg
+    self.dirs = args.dirs
+    self.exit_file = args.exitfile
+    self.error_file = args.errorfile
+    configp = configparser.SafeConfigParser()
+    configp.read(cfg)
+    self.proxy_db = configp.get('slapproxy', 'database_uri')
+    self.instance_dir = configp.get('slapos', 'instance_root')
+    mirror_dir = self.mirrorpath(self.instance_dir)
+    partitions = glob.glob(os.path.join(mirror_dir, 'slappart*'))
+    self.mirror_partition_dirs = [p for p in partitions if os.path.isdir(p)]
+    self.logs = []
+
+  def mirrorpath(self, dst):
+    return os.path.abspath(os.path.join(
+      self.backup_dir, os.path.relpath(dst, start=self.root_dir)))
+
+  def dstpath(self, src):
+    return os.path.abspath(os.path.join(
+      self.root_dir, os.path.relpath(src, start=self.backup_dir)))
+
+  def restoretree(self, dst, exclude=[], extrargs=[], verbosity='-v'):
+    src = self.mirrorpath(dst)
+    return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
+
+  def restoredb(self):
+    copydb(self.sqlite3_bin, self.mirrorpath(self.proxy_db), self.proxy_db)
+
+  def restorepartition(self, mirror_partition):
+    p = self.dstpath(mirror_partition)
+    installed = parse_installed(p) if os.path.exists(p) else []
+    copytree(self.rsync_bin, mirror_partition, p, exclude=installed)
+
+  def supervisorctl(self, *args):
+    supervisor_command = (self.supervisorctl_bin, '-c', self.supervisord_conf)
+    command = supervisor_command + args
+    print(' '.join(command))
+    sp.check_call(command)
+
+  def slapos(self, *args):
+    command = (self.slapos_bin,) + args + ('--cfg', self.slapos_cfg)
+    print(' '.join(command))
+    sp.check_call(command)
+
+  def verify(self, signaturefile):
+    pardir = os.path.abspath(os.path.join(self.backup_dir, os.pardir))
+    moved = os.path.join(pardir, 'backup.signature.moved')
+    proof = os.path.join(pardir, 'backup.signature.proof')
+    if os.path.exists(signaturefile):
+      os.rename(signaturefile, moved)
+    if not os.path.exists(moved):
+      msg = 'ERROR the backup signature file is missing'
+      print(msg)
+      raise Exception(msg)
+    with open(proof, 'w') as f:
+      for s in hashwalk(self.backup_dir, self.mirror_partition_dirs):
+        f.write(s + '\n')
+    diffcommand = ('diff', moved, proof)
+    print(' '.join(diffcommand))
+    try:
+      sp.check_output(
+        diffcommand, stderr=sp.STDOUT, universal_newlines=True)
+    except sp.CalledProcessError as e:
+      template = 'ERROR the backup signatures do not match\n\n%s'
+      msg = template % e.output
+      print(msg)
+      raise Exception(msg)
+
+  def loginfo(self, msg):
+    print(msg)
+    self.logs.append(msg)
+
+  def __call__(self):
+    remove(self.error_file)
+    exitcode = 0
+    try:
+      self.restore()
+    except Exception:
+      exitcode = 1
+      exc = traceback.format_exc()
+      with open(self.error_file, 'w') as f:
+        f.write('\n ... OK\n\n'.join(self.logs))
+        f.write('\n ... ERROR !\n\n')
+        f.write(exc)
+      print('\n\nERROR\n\n' + exc)
+    finally:
+      with open(self.exit_file, 'w') as f:
+        f.write(str(exitcode))
+      sys.exit(exitcode)
+
+  def restore(self):
+    self.loginfo('Verify backup signature')
+    self.verify(os.path.join(self.backup_dir, 'backup.signature'))
+
+    self.loginfo('Stop slapproxy')
+    self.supervisorctl('stop', 'slapos-proxy')
+
+    self.loginfo('Restore partitions')
+    for m in self.mirror_partition_dirs:
+      self.restorepartition(m)
+
+    for d in self.dirs:
+      self.loginfo('Restore directory ' + d)
+      self.restoretree(d)
+
+    self.loginfo('Restore slapproxy database')
+    self.restoredb()
+
+    etc_dir = os.path.join(self.root_dir, 'etc')
+    self.loginfo('Restore directory ' + etc_dir)
+    self.restoretree(etc_dir, extrargs=('--filter=- */', '--filter=-! .*'))
+
+    custom_script = os.path.join(self.root_dir, 'srv', 'runner-import-restore')
+    if os.path.exists(custom_script):
+      self.loginfo('Run custom restore script %s' % custom_script)
+      sp.check_call(custom_script)
+
+    self.loginfo('Start slapproxy again')
+    self.supervisorctl('start', 'slapos-proxy')
+
+    self.loginfo('Reformat partitions')
+    self.slapos('node', 'format', '--now')
+
+    self.loginfo('Remove old supervisord configuration files')
+    conf_dir = os.path.join(self.instance_dir, 'etc', 'supervisor.conf.d')
+    for f in glob.glob(os.path.join(conf_dir, '*')):
+      os.remove(f)
+
+    self.loginfo('Build Software Releases')
+    for i in range(3):
+      try:
+        self.slapos('node', 'software', '--all', '--logfile', self.sr_log)
+      except sp.CalledProcessError:
+        if i == 2:
+          raise
+      else:
+        break
+
+    self.loginfo('Remove old custom instance scripts')
+    partitions_glob = os.path.join(self.instance_dir, 'slappart*')
+    scripts = os.path.join(partitions_glob, 'srv', 'runner-import-restore')
+    for f in glob.glob(scripts):
+      remove(f)
+
+    self.loginfo('Remove partition timestamps')
+    timestamps = os.path.join(partitions_glob, '.timestamp')
+    for f in glob.glob(timestamps):
+      remove(f)
+
+    self.loginfo('Build Instances')
+    cp_log = self.cp_log
+    for i in range(3):
+      try:
+        self.slapos('node', 'instance', '--force-stop', '--logfile', cp_log)
+      except sp.CalledProcessError:
+        if i == 2:
+          raise
+      else:
+        break
+
+    for custom_script in glob.glob(scripts):
+      self.loginfo('Running custom instance script %s' % custom_script)
+      sp.check_call(custom_script)
+
+    self.loginfo('Done')
+
+
+if __name__ == '__main__':
+  main()