Commit a8d34062 authored by Łukasz Nowak's avatar Łukasz Nowak

resilient: Provide more information on error

It is important for DevOps to understand which files were modified during
synchronisation procedure in order to track issues fast.
parent e7572560
......@@ -94,13 +94,11 @@ def synchroniseRunnerWorkingDirectory(config, backup_path):
)
def backupFilesWereModifiedDuringExport(export_start_date):
def getBackupFilesModifiedDuringExportList(export_start_date):
export_time = time.time() - export_start_date
return bool(
subprocess.check_output((
return subprocess.check_output((
'find', '-cmin', str(export_time / 60.), '-type', 'f', '-path', '*/srv/backup/*'
))
)
)).split()
def runExport():
......@@ -147,8 +145,10 @@ def runExport():
# Check that export didn't happen during backup of instances
with CwdContextManager(os.path.join(runner_working_path, 'instance')):
if backupFilesWereModifiedDuringExport(export_start_date):
print("ERROR: Some backups are not consistent, exporter should be re-run."
" Let's sleep %s minutes, to let the backup end..." % args.backup_wait_time)
modified_file_list = getBackupFilesModifiedDuringExportList(export_start_date)
if len(modified_file_list):
print("ERROR: Files were modified since the backup started, exporter should be re-run."
" Let's sleep %s minutes, to let the backup end. Modified files:\n%s" % (
args.backup_wait_time, '\n'.join(modified_file_list)))
time.sleep(args.backup_wait_time * 60)
sys.exit(1)
......@@ -138,7 +138,7 @@ class TestRunnerExporter(unittest.TestCase):
)
self.assertEqual(check_output_mock.call_count, 1)
check_output_mock.assert_any_call(
['rsync', '-rlptgov', '--stats', '--safe-links', '--ignore-missing-args', '--delete', '--delete-excluded', 'config.json', '.parameters.xml', '.project', 'backup/runner/etc/']
['rsync', '-rlptgov', '--stats', '--safe-links', '--ignore-missing-args', '--delete', '--delete-excluded', 'config.json', '.project', '.parameters.xml', 'backup/runner/etc/']
)
......@@ -195,11 +195,20 @@ class TestRunnerExporter(unittest.TestCase):
49b74873d57ff0307b7c9364e2fe2a3876d8722fbe7ce3a6f1438d47647a86f4 ./etc/.project
7d793037a0760186574b0282f2f435e7 ./runner/instance/slappart1/data""")
def test_backupFilesWereModifiedDuringExport(self):
def test_getBackupFilesModifiedDuringExportList(self):
self._setUpFakeInstanceFolder()
with runner_exporter.CwdContextManager('instance'):
self.assertTrue(runner_exporter.backupFilesWereModifiedDuringExport(time.time() - 5))
self.assertEqual(
runner_exporter.getBackupFilesModifiedDuringExportList(time.time() - 5),
['./slappart0/srv/backup/data.dat']
)
time.sleep(2)
self.assertFalse(runner_exporter.backupFilesWereModifiedDuringExport(time.time() - 1))
self.assertEqual(
runner_exporter.getBackupFilesModifiedDuringExportList(time.time() - 1),
[]
)
self._createFile('slappart1/srv/backup/bakckup.data', 'my backup')
self.assertTrue(runner_exporter.backupFilesWereModifiedDuringExport(time.time() - 1))
self.assertEqual(
runner_exporter.getBackupFilesModifiedDuringExportList(time.time() - 1),
['./slappart1/srv/backup/bakckup.data']
)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment