Commit e3a6f032 authored by Xavier Thompson's avatar Xavier Thompson

software/theia: Add export script

parent c52d568a
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[instance-theia] [instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in _update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 001440693adfb9583d1479aae4bfa970 md5sum = 6ce2d9f0c32e3b781fab5d59685dc866
[instance] [instance]
_update_hash_filename_ = instance.cfg.in _update_hash_filename_ = instance.cfg.in
...@@ -27,12 +27,20 @@ md5sum = d1181f788461a31026e36677ab56398d ...@@ -27,12 +27,20 @@ md5sum = d1181f788461a31026e36677ab56398d
[instance-export] [instance-export]
_update_hash_filename_ = instance-export.cfg.jinja.in _update_hash_filename_ = instance-export.cfg.jinja.in
md5sum = 4f52e4cff8e72369b38e47df297df7e2 md5sum = 802c13e0cdbdc4dffa606df1344eeb6b
[instance-resilient] [instance-resilient]
_update_hash_filename_ = instance-resilient.cfg.jinja _update_hash_filename_ = instance-resilient.cfg.jinja
md5sum = d78a9f885bdebf6720197209e0c21aa0 md5sum = d78a9f885bdebf6720197209e0c21aa0
[theia-common]
_update_hash_filename_ = theia_common.py
md5sum = e57396473b4b6a17d26a747f0030293c
[theia-export]
_update_hash_filename_ = theia_export.py
md5sum = b5f5ac1924b27d3f2be2e5ea291c119e
[yarn.lock] [yarn.lock]
_update_hash_filename_ = yarn.lock _update_hash_filename_ = yarn.lock
md5sum = 80e7ad91deea54cebcccef5a83fdb380 md5sum = 80e7ad91deea54cebcccef5a83fdb380
......
...@@ -24,10 +24,42 @@ namebase = {{ parameter_dict['namebase'] }} ...@@ -24,10 +24,42 @@ namebase = {{ parameter_dict['namebase'] }}
# to be pulled from the export instance. # to be pulled from the export instance.
# All it expects is that a script be available in exporter:wrapper. # All it expects is that a script be available in exporter:wrapper.
[exporter] [exporter]
recipe = slapos.cookbook:wrapper wrapper = $${theia-export-script:rendered}
command-line = echo "Export Not Implemented Yet"
wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-exporter [theia-export-script]
wrapper = $${:wrapper-path} recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/theia-export-script
mode = 0700
exitcode-file = $${directory:srv}/export-exitcode-file
error-file = $${directory:srv}/export-errormessage-file
context =
raw python ${software-info:python-with-eggs}
raw theia_export ${software-info:theia-export}
raw bash ${software-info:bash}
raw rsync ${software-info:rsync}
raw sqlite3 ${software-info:sqlite3}
raw root_path $${buildout:directory}
raw backup_path $${directory:backup}
raw slapos_cfg $${directory:runner}/etc/slapos.cfg
raw project_path $${directory:project}
raw public_path $${directory:frontend-static-public}
key exitfile :exitcode-file
key errorfile :error-file
{%- raw %}
template =
inline:#!{{ bash }}
{{ python }} {{ theia_export }} \
--rsync {{ rsync }} \
--sqlite3 {{ sqlite3 }} \
--root {{ root_path }} \
--backup {{ backup_path }} \
--cfg {{ slapos_cfg }} \
--dirs {{ project_path }} \
--dirs {{ public_path }} \
--exitfile {{ exitfile }} \
--errorfile {{ errorfile }}
{%- endraw %}
# Extend resilient parameters with normal theia connection parameters # Extend resilient parameters with normal theia connection parameters
[resilient-publish-connection-parameter] [resilient-publish-connection-parameter]
......
...@@ -41,23 +41,26 @@ backend-url = $${frontend-instance:url} ...@@ -41,23 +41,26 @@ backend-url = $${frontend-instance:url}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc home = $${buildout:directory}
var = $${buildout:directory}/var etc = $${:home}/etc
srv = $${buildout:directory}/srv var = $${:home}/var
bin = $${buildout:directory}/bin srv = $${:home}/srv
tmp = $${buildout:directory}/tmp bin = $${:home}/bin
dot-theia = $${buildout:directory}/.theia/ tmp = $${:home}/tmp
dot-theia = $${:home}/.theia/
pidfiles = $${:var}/run pidfiles = $${:var}/run
services = $${:etc}/service services = $${:etc}/service
runner = $${:srv}/runner runner = $${:srv}/runner
backup = $${:srv}/backup/theia
project = $${:srv}/project project = $${:srv}/project
frontend-static = $${:srv}/frontend-static frontend-static = $${:srv}/frontend-static
frontend-static-public = $${:frontend-static}/public frontend-static-public = $${:frontend-static}/public
frontend-static-css = $${:frontend-static}/css frontend-static-css = $${:frontend-static}/css
bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/ bash-completions = $${:home}/.local/share/bash-completion/completions/
fish-completions = $${buildout:directory}/.config/fish/completions/ fish-completions = $${:home}/.config/fish/completions/
# Promises # Promises
......
...@@ -23,11 +23,14 @@ extends = ...@@ -23,11 +23,14 @@ extends =
parts = parts =
theia-wrapper theia-wrapper
slapos-cookbook slapos-cookbook
python-with-eggs
instance-theia instance-theia
instance instance
instance-import instance-import
instance-export instance-export
instance-resilient instance-resilient
theia-common
theia-export
# default for slapos-standalone # default for slapos-standalone
shared-part-list = shared-part-list =
...@@ -353,6 +356,18 @@ template = ...@@ -353,6 +356,18 @@ template =
#!/bin/sh #!/bin/sh
exec ${nodejs:location}/bin/node ${theia:location}/node_modules/.bin/theia-open "$@" exec ${nodejs:location}/bin/node ${theia:location}/node_modules/.bin/theia-open "$@"
[python-with-eggs]
recipe = zc.recipe.egg
interpreter = ${:_buildout_section_name_}
eggs =
${slapos-toolbox:eggs}
six
zc.buildout
# Only generate the interpreter script to avoid conflicts with scripts
# for eggs that are also generated by another section, like slapos.toolbox
scripts = ${:interpreter}
[instance-theia] [instance-theia]
<= template-base <= template-base
output = ${buildout:directory}/instance-theia.cfg.jinja output = ${buildout:directory}/instance-theia.cfg.jinja
...@@ -371,3 +386,19 @@ output = ${buildout:directory}/instance-export.cfg.jinja ...@@ -371,3 +386,19 @@ output = ${buildout:directory}/instance-export.cfg.jinja
[instance-resilient] [instance-resilient]
<= download-base <= download-base
[theia-common]
<= download-base
destination = ${buildout:directory}/theia_common.py
[theia-export]
<= download-base
destination = ${buildout:directory}/theia_export.py
[software-info]
python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
python = ${python:location}/bin/python
rsync = ${rsync:location}/bin/rsync
sqlite3 = ${sqlite3:location}/bin/sqlite3
bash = ${bash:location}/bin/bash
theia-export = ${theia-export:output}
import contextlib
import errno
import glob
import hashlib
import os
import re
import subprocess as sp
import sqlite3
import six
import zc.buildout.configparser
from slapos.util import bytes2str, str2bytes
RSYNC_FLAGS = ('-rlptgo', '--safe-links', '--stats', '--ignore-missing-args', '--delete', '--delete-excluded')
RSYNC_REGEX = '^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
EXCLUDE_PATTERNS = ('*.sock', '*.socket', '*.pid', '.installed*.cfg')
EXCLUDE_FLAGS = ['--exclude={}'.format(x) for x in sorted(EXCLUDE_PATTERNS)]
def makedirs(path):
try:
os.makedirs(path if os.path.isdir(path) else os.path.dirname(path))
except OSError as e:
if e.errno != errno.EEXIST:
raise
def copytree(rsyncbin, src, dst, exclude=[], extrargs=[], verbosity='-v'):
# Ensure there is a trailing slash in the source directory
# to avoid creating an additional directory level at the destination
src = os.path.join(src, '')
# Compute absolute path of destination
dst = os.path.abspath(dst)
# Create destination dir if it doesn't exist
makedirs(dst)
command = [rsyncbin]
command.extend(RSYNC_FLAGS)
# Exclude destination file from sources
command.append('--filter=-/ {}'.format(dst))
command.extend(EXCLUDE_FLAGS)
command.extend(('--filter=-/ {}'.format(x) for x in sorted(exclude)))
command.extend(extrargs)
command.append(verbosity)
command.append(src)
command.append(dst)
try:
return sp.check_output(command, universal_newlines=True)
except sp.CalledProcessError as e:
# Not all rsync errors are to be considered as errors
if e.returncode != 24 or re.search(RSYNC_REGEX, e.output, re.M) is None:
raise
return e.output
def copydb(sqlite3bin, src_db, dst_db):
makedirs(dst_db)
sp.check_output((sqlite3bin, src_db, '.backup ' + dst_db))
def remove(path):
try:
os.remove(path)
except OSError:
if os.path.exists(path):
raise
def parse_installed(partition):
paths = []
custom_script = os.path.join(partition, 'srv', '.backup_identity_script')
for cfg in glob.glob(os.path.join(partition, '.installed*.cfg')):
try:
with open(cfg) as f:
installed_cfg = zc.buildout.configparser.parse(f, cfg)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in six.itervalues(installed_cfg):
for p in section.get('__buildout_installed__', '').splitlines():
p = p.strip()
if p and p != custom_script:
paths.append(p)
return paths
def sha256sum(file_path, chunk_size=1024 * 1024):
sha256 = hashlib.sha256()
with open(file_path, 'rb') as f:
chunk = f.read(chunk_size)
while chunk:
sha256.update(chunk)
chunk = f.read(chunk_size)
return sha256.hexdigest()
def hashwalk(backup_dir, mirror_partitions):
scripts = {}
for p in mirror_partitions:
script_path = os.path.join(p, 'srv', '.backup_identity_script')
if os.path.exists(script_path):
scripts[os.path.abspath(p)] = script_path
for dirpath, dirnames, filenames in os.walk(backup_dir):
filenames.sort()
for f in filenames:
filepath = os.path.join(dirpath, f)
if os.path.isfile(filepath):
displaypath = os.path.relpath(filepath, start=backup_dir)
yield '%s %s' % (sha256sum(filepath), displaypath)
remaining_dirnames = []
for subdir in dirnames:
subdirpath = os.path.abspath(os.path.join(dirpath, subdir))
custom_hashscript = scripts.get(subdirpath)
if custom_hashscript:
print('Using custom signature script %s' % custom_hashscript)
for s in hashcustom(subdirpath, backup_dir, custom_hashscript):
yield s
else:
remaining_dirnames.append(subdir)
remaining_dirnames.sort()
dirnames[:] = remaining_dirnames
@contextlib.contextmanager
def cwd(path):
old_path = os.getcwd()
try:
os.chdir(path)
yield
finally:
os.chdir(old_path)
def hashcustom(mirrordir, backup_dir, custom_hashscript):
workingdir = os.path.join(mirrordir, os.pardir, os.pardir, os.pardir)
with cwd(os.path.abspath(workingdir)):
for dirpath, _, filenames in os.walk(mirrordir):
filepaths = []
for f in filenames:
path = os.path.join(dirpath, f)
if os.path.isfile(path):
filepaths.append('./' + os.path.relpath(path, start=workingdir))
if not filepaths:
continue
hashprocess = sp.Popen(
custom_hashscript, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE)
out, err = hashprocess.communicate(str2bytes('\0'.join(filepaths)))
if hashprocess.returncode != 0:
template = "Custom signature script %s failed on inputs:\n%s"
msg = template % (custom_hashscript, '\n'.join(filepaths))
msg += "\nwith stdout:\n%s" % bytes2str(out)
msg += "\nand stderr:\n%s" % bytes2str(err)
raise Exception(msg)
signatures = bytes2str(out).strip('\n').split('\n')
signatures.sort()
displaypath = os.path.relpath(dirpath, start=backup_dir)
for s in signatures:
yield '%s %s/ (custom)' % (s, displaypath)
import argparse
import glob
import itertools
import os
import sys
import time
import traceback
import six
from six.moves import configparser
sys.path.append(os.path.dirname(__file__))
from theia_common import copytree, copydb, hashwalk, parse_installed, remove
os.environ['LC_ALL'] = 'C'
os.umask(0o77)
BACKUP_WAIT = 10
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--rsync', required=True)
parser.add_argument('--sqlite3', required=True)
parser.add_argument('--root', required=True)
parser.add_argument('--backup', required=True)
parser.add_argument('--cfg', required=True)
parser.add_argument('--dirs', action='append')
parser.add_argument('--exitfile', required=True)
parser.add_argument('--errorfile', required=True)
args = parser.parse_args()
TheiaExport(args)()
class TheiaExport(object):
def __init__(self, args):
self.rsync_bin = args.rsync
self.sqlite3_bin = args.sqlite3
self.root_dir = args.root
self.backup_dir = args.backup
self.slapos_cfg = cfg = args.cfg
self.dirs = args.dirs
self.exit_file = args.exitfile
self.error_file = args.errorfile
configp = configparser.SafeConfigParser()
configp.read(cfg)
self.proxy_db = configp.get('slapproxy', 'database_uri')
self.instance_dir = configp.get('slapos', 'instance_root')
partitions = glob.glob(os.path.join(self.instance_dir, 'slappart*'))
self.partition_dirs = [p for p in partitions if os.path.isdir(p)]
self.copytree_partitions_args = {}
self.logs = []
def mirrorpath(self, src):
return os.path.abspath(os.path.join(
self.backup_dir, os.path.relpath(src, start=self.root_dir)))
def backuptree(self, src, exclude=[], extrargs=[], verbosity='-v'):
dst = self.mirrorpath(src)
return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
def backupdb(self):
copydb(self.sqlite3_bin, self.proxy_db, self.mirrorpath(self.proxy_db))
def backuppartition(self, partition):
installed = parse_installed(partition)
rules = os.path.join(partition, 'srv', 'exporter.exclude')
extrargs = ('--filter=.-/ ' + rules,) if os.path.exists(rules) else ()
self.backuptree(partition, exclude=installed, extrargs=extrargs)
self.copytree_partitions_args[partition] = (installed, extrargs)
def sign(self, signaturefile):
remove(signaturefile)
pardir = os.path.abspath(os.path.join(self.backup_dir, os.pardir))
tmpfile = os.path.join(pardir, 'backup.signature.tmp')
mirror_partitions = [self.mirrorpath(p) for p in self.partition_dirs]
with open(tmpfile, 'w') as f:
for s in hashwalk(self.backup_dir, mirror_partitions):
f.write(s + '\n')
os.rename(tmpfile, signaturefile)
def checkpartition(self, partition, pattern='/srv/backup/'):
installed, extrargs = self.copytree_partitions_args[partition]
output = self.backuptree(
partition,
exclude=installed,
extrargs=extrargs + ('--dry-run', '--update'),
verbosity='--out-format=%n',
)
return [path for path in output.splitlines() if pattern in path]
def loginfo(self, msg):
print(msg)
self.logs.append(msg)
def __call__(self):
remove(self.error_file)
exitcode = 0
try:
self.export()
except Exception:
exitcode = 1
exc = traceback.format_exc()
with open(self.error_file, 'w') as f:
f.write('\n ... OK\n\n'.join(self.logs))
f.write('\n ... ERROR !\n\n')
f.write(exc)
print('\n\nERROR\n\n' + exc)
finally:
with open(self.exit_file, 'w') as f:
f.write(str(exitcode))
sys.exit(exitcode)
def export(self):
export_start_date = int(time.time())
etc_dir = os.path.join(self.root_dir, 'etc')
with open(os.path.join(etc_dir, '.resilient_timestamp'), 'w') as f:
f.write(str(export_start_date))
self.loginfo('Backup directory ' + etc_dir)
self.backuptree(etc_dir, extrargs=('--filter=- */', '--filter=-! .*'))
for d in self.dirs:
self.loginfo('Backup directory ' + d)
self.backuptree(d)
self.loginfo('Backup slapproxy database')
self.backupdb()
self.loginfo('Backup partitions')
for p in self.partition_dirs:
self.backuppartition(p)
self.loginfo('Compute backup signature')
self.sign(os.path.join(self.backup_dir, 'backup.signature'))
time.sleep(10)
self.loginfo('Check partitions')
modified = list(itertools.chain.from_iterable(
self.checkpartition(p) for p in self.partition_dirs))
if modified:
msg = 'Some files have been modified since the backup started'
self.loginfo(msg + ':')
self.loginfo('\n'.join(modified))
self.loginfo("Let's wait %d minutes and try again" % BACKUP_WAIT)
time.sleep(BACKUP_WAIT * 60)
raise Exception(msg)
self.loginfo('Done')
if __name__ == '__main__':
main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment