Commit de334909 authored by Nicolas Wavrant's avatar Nicolas Wavrant

resilient: port the export script of the webrunner to python

Rewrite the [runner-exporter bash script](https://lab.nexedi.com/nexedi/slapos/blob/2a01586bfe17f46e2ef26ec6e45bef692435ed28/software/slaprunner/template/runner-export.sh.jinja2) in python.

Also parametarize the signature function to not use only sha256sum which can create issues on some systems. This should be a separated commit, but that shouldn't be a blocker.

Write unit tests for the main functions of this exporter script. As every trick in the original bash script were not obvious / understood / found by me, I may have missed some.

/reviewed-on !44
parent b72ec2ad
...@@ -55,6 +55,7 @@ setup(name=name, ...@@ -55,6 +55,7 @@ setup(name=name,
'dnspython', 'dnspython',
'requests', 'requests',
'jsonschema', 'jsonschema',
'zc.buildout',
] + additional_install_requires, ] + additional_install_requires,
extras_require = { extras_require = {
'lampconfigure': ["mysqlclient"], #needed for MySQL Database access 'lampconfigure': ["mysqlclient"], #needed for MySQL Database access
...@@ -110,6 +111,7 @@ setup(name=name, ...@@ -110,6 +111,7 @@ setup(name=name,
'pubsubserver = slapos.pubsub:main', 'pubsubserver = slapos.pubsub:main',
'qemu-qmp-client = slapos.qemuqmpclient:main', 'qemu-qmp-client = slapos.qemuqmpclient:main',
'rdiffbackup.genstatrss = slapos.resilient.rdiffBackupStat2RSS:main', 'rdiffbackup.genstatrss = slapos.resilient.rdiffBackupStat2RSS:main',
'runner-exporter = slapos.resilient.runner_exporter:runExport',
'securedelete = slapos.securedelete:main', 'securedelete = slapos.securedelete:main',
'slapos-kill = slapos.systool:kill', 'slapos-kill = slapos.systool:kill',
'slaprunnertest = slapos.runner.runnertest:main', 'slaprunnertest = slapos.runner.runnertest:main',
......
from __future__ import print_function
import argparse
import errno
import glob
import itertools
import os
import re
import shutil
import subprocess
import sys
import time
from contextlib import contextmanager
from datetime import datetime
from hashlib import sha256
from zc.buildout.configparser import parse
os.environ['LC_ALL'] = 'C'
os.umask(0o77)
def read_file_by_chunk(path, chunk_size=1024 * 1024):
with open(path, 'rb') as f:
chunk = f.read(chunk_size)
while chunk:
yield chunk
chunk = f.read(chunk_size)
@contextmanager
def CwdContextManager(path):
"""
Context Manager for executing code in a given directory.
There is no need to provide fallback or basic checks
in this code, as these checkes should exist in the code
invoking this Context Manager.
If someone needs to add checks here, I'm pretty sure
it means that they are trying to hide legitimate errors.
See tests to see examples of invokation
"""
old_path = os.getcwd()
try:
os.chdir(path)
yield
finally:
os.chdir(old_path)
def parseArgumentList():
parser = argparse.ArgumentParser()
parser.add_argument('--srv-path', required=True)
parser.add_argument('--backup-path', required=True)
parser.add_argument('--etc-path', required=True)
parser.add_argument('--rsync-binary', default='rsync')
parser.add_argument('--backup-wait-time', type=int, required=True)
parser.add_argument('-n', action='store_true', dest='dry', default=False)
return parser.parse_args()
def rsync(rsync_binary, source, destination, extra_args=None, dry=False):
arg_list = [
rsync_binary,
'-rlptgov',
'--stats',
'--safe-links',
'--ignore-missing-args',
'--delete',
'--delete-excluded'
]
if isinstance(extra_args, list):
arg_list.extend(extra_args)
if isinstance(source, list):
arg_list.extend(source)
else:
arg_list.append(source)
arg_list.append(destination)
if dry:
print('DEBUG:', arg_list)
return
try:
print(subprocess.check_output(arg_list))
except subprocess.CalledProcessError as e:
# All rsync errors are not to be considered as errors
allowed_error_message_list = \
'^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
if e.returncode != 24 or \
re.search(allowed_error_message_regex, e.output, re.M) is None:
raise
def getExcludePathList(path):
excluded_path_list = [
"*.sock",
"*.socket",
"*.pid",
".installed*.cfg",
]
def append_relative(path_list):
for p in path_list:
p = p.strip()
if p:
excluded_path_list.append(os.path.relpath(p, path))
for partition in glob.glob(os.path.join(path, "instance", "slappart*")):
if not os.path.isdir(partition):
continue
with CwdContextManager(partition):
try:
with open("srv/exporter.exclude") as f:
exclude = f.readlines()
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
append_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
append_relative(section.get(
'__buildout_installed__', '').splitlines())
return excluded_path_list
def getSha256Sum(file_path_list):
result_list = []
for file_path in file_path_list:
hash_sum = sha256()
for chunk in read_file_by_chunk(file_path):
hash_sum.update(chunk)
result_list.append("%s %s" % (hash_sum.hexdigest(), file_path))
return result_list
def synchroniseRunnerConfigurationDirectory(config, backup_path):
if not os.path.exists(backup_path):
os.makedirs(backup_path)
file_list = ['config.json']
for hidden_file in os.listdir('.'):
if hidden_file[0] == '.':
file_list.append(hidden_file)
rsync(config.rsync_binary, file_list, backup_path, dry=config.dry)
def synchroniseRunnerWorkingDirectory(config, backup_path):
file_list = []
exclude_list = []
if os.path.isdir('instance'):
file_list.append('instance')
exclude_list = getExcludePathList(os.getcwd())
# XXX: proxy.db should be properly dumped to leverage its
# atomic properties
for file in ('project', 'public', 'proxy.db'):
if os.path.exists(file):
file_list.append(file)
if file_list:
rsync(
config.rsync_binary, file_list, backup_path,
["--exclude={}".format(x) for x in exclude_list],
dry=config.dry
)
def getSlappartSignatureMethodDict():
slappart_signature_method_dict = {}
for partition in glob.glob("./instance/slappart*"):
if os.path.isdir(partition):
script_path = os.path.join(partition, 'srv', '.backup_identity_script')
if os.path.exists(script_path):
slappart_signature_method_dict[partition] = script_path
return slappart_signature_method_dict
def writeSignatureFile(slappart_signature_method_dict, runner_working_path, signature_file_path='backup.signature'):
special_slappart_list = slappart_signature_method_dict.keys()
signature_list = []
for dirpath, dirname_list, filename_list in os.walk('.'):
if dirpath == '.' or not filename_list:
continue
# Find if special signature function should be applied
for special_slappart in special_slappart_list:
backup_identity_script_path = os.path.join(
runner_working_path,
slappart_signature_method_dict[special_slappart]
)
if dirpath.startswith(special_slappart):
signature_process = subprocess.Popen(
backup_identity_script_path,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
(output, error_output) = signature_process.communicate(
'\0'.join([os.path.join(dirpath, filename) for filename in filename_list])
)
if signature_process.returncode != 0:
print(
"An issue occured when calculating the custom signature"
" with %s :\n%s" % (
backup_identity_script_path, error_output
)
)
sys.exit(1)
# We have to rstrip as most programs return an empty line
# at the end of their output
signature_list.extend(output.strip('\n').split('\n'))
else:
signature_list.extend(
getSha256Sum([
os.path.join(dirpath, filename)
for filename in filename_list
])
)
# Write the signatures in file
with open(signature_file_path, 'w+') as signature_file:
signature_file.write("\n".join(sorted(signature_list)))
def backupFilesWereModifiedDuringExport(export_start_date):
export_time = time.time() - export_start_date
return bool(
subprocess.check_output((
'find', '-cmin', str(export_time / 60.), '-type', 'f', '-path', '*/srv/backup/*'
))
)
def runExport():
export_start_date = int(time.time())
print(datetime.fromtimestamp(export_start_date).isoformat())
args = parseArgumentList()
def _rsync(*params):
return rsync(args.rsync_binary, *params, dry=args.dry)
runner_working_path = os.path.join(args.srv_path, 'runner')
backup_runner_path = os.path.join(args.backup_path, 'runner')
# Synchronise runner's etc directory
with CwdContextManager(args.etc_path):
with open('.resilient-timestamp', 'w') as f:
f.write(str(export_start_date))
# "+ '/'" is mandatory otherwise rsyncing the etc directory
# will create in the backup_etc_path only a file called etc
backup_etc_path = os.path.join(args.backup_path, 'etc') + '/'
synchroniseRunnerConfigurationDirectory(args, backup_etc_path)
# Synchronise runner's working directory
# and aggregate signature functions as we are here
with CwdContextManager(runner_working_path):
synchroniseRunnerWorkingDirectory(args, backup_runner_path)
slappart_signature_method_dict = getSlappartSignatureMethodDict()
# Calculate signature of synchronised files
with CwdContextManager(args.backup_path):
writeSignatureFile(slappart_signature_method_dict, runner_working_path)
# BBB: clean software folder if it was synchronized
# in an old instance
backup_software_path = os.path.join(backup_runner_path, 'software')
if os.path.isdir(backup_software_path):
shutil.rmtree(backup_software_path)
# Wait a little to increase the probability to detect an ongoing backup.
time.sleep(10)
# Check that export didn't happen during backup of instances
with CwdContextManager(backup_runner_path):
if backupFilesWereModifiedDuringExport(export_start_date):
print("ERROR: Some backups are not consistent, exporter should be re-run."
" Let's sleep %s minutes, to let the backup end..." % backup_wait_time)
time.sleep(backup_wait_time * 60)
system.exit(1)
\ No newline at end of file
import mock
import os
import shutil
import time
import unittest
from slapos.resilient import runner_exporter
from StringIO import StringIO
tested_instance_cfg = """[buildout]
installed_develop_eggs =
parts = folders hello-nicolas hello-rafael exclude
[folders]
__buildout_installed__ =
__buildout_signature__ = wcwidth-0.1.7-py2.7.egg contextlib2-0.5.5-py2.7.egg ...
etc = /some/prefix/slappart18/test/etc
home = /srv/slapgrid/slappart18/test
recipe = slapos.cookbook:mkdirectory
srv = /some/prefix/slappart18/test/srv
[hello-nicolas]
__buildout_installed__ = {cwd}/instance/slappart0/etc/nicolas.txt
__buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7.egg zc.buildout-2.12.2-py2.7.egg slapos.recipe.template-4.3-py2.7.egg setuptools-40.4.3-py2.7.egg
mode = 0644
name = Nicolas
output = /some/prefix/slappart18/test/etc/nicolas.txt
recipe = slapos.recipe.template
[hello-rafael]
__buildout_installed__ = {cwd}/instance/slappart0/etc//rafael.txt
__buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7.egg zc.buildout-2.12.2-py2.7.egg slapos.recipe.template-4.3-py2.7.egg setuptools-40.4.3-py2.7.egg
name = Rafael
output = /some/prefix/slappart18/test/etc/rafael.txt
recipe = slapos.recipe.template
[exclude]
__buildout_installed__ = {cwd}/instance/slappart0/srv/exporter.exclude
__buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7.egg zc.buildout-2.12.2-py2.7.egg slapos.recipe.template-4.3-py2.7.egg setuptools-40.4.3-py2.7.egg
recipe = slapos.recipe.template:jinja2
rendered = /some/prefix/slappart18/test/srv/exporter.exclude
template = inline:
srv/backup/**"""
class Config():
pass
class TestRunnerExporter(unittest.TestCase):
def setUp(self):
if not os.path.exists('test_folder'):
os.mkdir('test_folder')
os.chdir('test_folder')
def tearDown(self):
if os.path.basename(os.getcwd()) == 'test_folder':
os.chdir('..')
shutil.rmtree('test_folder')
elif 'test_folder' in os.listdir('.'):
shutil.rmtree('test_folder')
def _createFile(self, path, content=''):
with open(path, 'w') as f:
f.write(content)
def _createExecutableFile(self, path, content=''):
self._createFile(path, content)
os.chmod(path, 0700)
def _setUpFakeInstanceFolder(self):
self._createFile('proxy.db')
os.makedirs('project')
os.makedirs('public')
"""Create data mirroring tested_instance_cfg"""
os.makedirs('instance/slappart0/etc')
os.makedirs('instance/slappart0/srv/backup')
os.makedirs('instance/slappart1/etc')
os.makedirs('instance/slappart1/srv/backup')
self._createFile('instance/slappart0/.installed.cfg',
tested_instance_cfg.format(cwd=os.getcwd()))
self._createFile('instance/slappart0/srv/backup/data.dat',
'all my fortune lays on this secret !')
self._createFile('instance/slappart0/srv/exporter.exclude',
'srv/backup/**')
self._createFile('instance/slappart0/etc/config.json')
self._createFile('instance/slappart0/etc/.parameters.xml')
self._createFile('instance/slappart0/etc/.project',
'workspace/slapos-dev/software/erp5')
self._createExecutableFile(
'instance/slappart1/srv/.backup_identity_script',
"#!/bin/sh\nexec xargs -0 md5sum"
)
def test_CwdContextManager(self):
os.makedirs('a/b')
with runner_exporter.CwdContextManager('a'):
self.assertEqual(os.listdir('.'), ['b'])
os.mkdir('c')
self.assertEqual(os.listdir('.'), ['a'])
self.assertEqual(sorted(os.listdir('a')), ['b', 'c'])
def test_getExcludePathList(self):
self._setUpFakeInstanceFolder()
self.assertEqual(
sorted(runner_exporter.getExcludePathList(os.getcwd())),
[
'*.pid',
'*.sock',
'*.socket',
'.installed*.cfg',
'instance/slappart0/etc/nicolas.txt',
'instance/slappart0/etc/rafael.txt',
'instance/slappart0/srv/backup/**',
'instance/slappart0/srv/exporter.exclude',
]
)
@mock.patch('subprocess.check_output')
def test_synchroniseRunnerConfigurationDirectory(self, check_output_mock):
self._setUpFakeInstanceFolder()
config = Config()
config.rsync_binary = 'rsync'
config.dry = False
with runner_exporter.CwdContextManager('instance/slappart0/etc'):
runner_exporter.synchroniseRunnerConfigurationDirectory(
config, 'backup/runner/etc/'
)
self.assertEqual(check_output_mock.call_count, 1)
check_output_mock.assert_any_call(
['rsync', '-rlptgov', '--stats', '--safe-links', '--ignore-missing-args', '--delete', '--delete-excluded', 'config.json', '.parameters.xml', '.project', 'backup/runner/etc/']
)
@mock.patch('subprocess.check_output')
def test_synchroniseRunnerWorkingDirectory(self, check_output_mock):
self._setUpFakeInstanceFolder()
config = Config()
config.rsync_binary = 'rsync'
config.dry = False
with runner_exporter.CwdContextManager(os.getcwd()):
runner_exporter.synchroniseRunnerWorkingDirectory(
config, 'backup/runner/runner'
)
self.assertEqual(check_output_mock.call_count, 1)
check_output_mock.assert_any_call(
['rsync', '-rlptgov', '--stats', '--safe-links', '--ignore-missing-args', '--delete', '--delete-excluded', '--exclude=*.sock', '--exclude=*.socket', '--exclude=*.pid', '--exclude=.installed*.cfg', '--exclude=instance/slappart0/srv/backup/**', '--exclude=instance/slappart0/etc/nicolas.txt', '--exclude=instance/slappart0/etc/rafael.txt', '--exclude=instance/slappart0/srv/exporter.exclude', 'instance', 'project', 'public', 'proxy.db', 'backup/runner/runner']
)
def test_getSlappartSignatureMethodDict(self):
self._setUpFakeInstanceFolder()
slappart_signature_method_dict = runner_exporter.getSlappartSignatureMethodDict()
self.assertEqual(
slappart_signature_method_dict,
{
'./instance/slappart1': './instance/slappart1/srv/.backup_identity_script',
}
)
def test_writeSignatureFile(self):
self._setUpFakeInstanceFolder()
os.makedirs('backup/instance/etc')
os.makedirs('backup/instance/slappart0')
os.makedirs('backup/instance/slappart1')
self._createFile('backup/instance/etc/.project', 'workspace/slapos-dev/software/erp5')
self._createFile('backup/instance/slappart0/data', 'hello')
self._createFile('backup/instance/slappart1/data', 'world')
slappart_signature_method_dict = {
'./instance/slappart1': './instance/slappart1/srv/.backup_identity_script',
}
with runner_exporter.CwdContextManager('backup'):
runner_exporter.writeSignatureFile(slappart_signature_method_dict, '..', signature_file_path='backup.signature')
with open('backup.signature', 'r') as f:
signature_file_content = f.read()
# Slappart1 is using md5sum as signature, others are using sha256sum (default)
self.assertEqual(signature_file_content, """2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 ./instance/slappart0/data
49b74873d57ff0307b7c9364e2fe2a3876d8722fbe7ce3a6f1438d47647a86f4 ./instance/etc/.project
7d793037a0760186574b0282f2f435e7 ./instance/slappart1/data""")
def test_backupFilesWereModifiedDuringExport(self):
self._setUpFakeInstanceFolder()
with runner_exporter.CwdContextManager('instance'):
self.assertTrue(runner_exporter.backupFilesWereModifiedDuringExport(time.time() - 5))
time.sleep(2)
self.assertFalse(runner_exporter.backupFilesWereModifiedDuringExport(time.time() - 1))
self._createFile('slappart1/srv/backup/bakckup.data', 'my backup')
self.assertTrue(runner_exporter.backupFilesWereModifiedDuringExport(time.time() - 1))
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment