Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.toolbox
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
5
Merge Requests
5
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
slapos.toolbox
Commits
c55e1100
Commit
c55e1100
authored
Apr 04, 2019
by
Thomas Gambier
🚴🏼
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[runner] don't check modified files that wouldn't be rsynced anyway
parent
c915b5d7
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
71 additions
and
29 deletions
+71
-29
slapos/resilient/runner_exporter.py
slapos/resilient/runner_exporter.py
+28
-9
slapos/test/test_runner_exporter.py
slapos/test/test_runner_exporter.py
+43
-20
No files found.
slapos/resilient/runner_exporter.py
View file @
c55e1100
...
...
@@ -8,6 +8,7 @@ import shutil
import
subprocess
import
sys
import
time
import
fnmatch
from
datetime
import
datetime
from
.runner_utils
import
*
...
...
@@ -95,13 +96,31 @@ def synchroniseRunnerWorkingDirectory(config, backup_path):
dry
=
config
.
dry
)
def
isFilepathMatching
(
filepath
,
pattern_list
):
for
pattern
in
pattern_list
:
if
fnmatch
.
fnmatch
(
filepath
,
pattern
):
return
True
return
False
def
getBackupFilesModifiedDuringExportList
(
export_start_date
):
def
getBackupFilesModifiedDuringExportList
(
config
,
export_start_date
):
export_time
=
time
.
time
()
-
export_start_date
return
subprocess
.
check_output
((
'find'
,
'-cmin'
,
str
(
export_time
/
60.
),
'-type'
,
'f'
,
'-path'
,
'*/srv/backup/*'
)).
split
()
# find all files that were modified during export
modified_files
=
subprocess
.
check_output
((
'find'
,
'instance'
,
'-cmin'
,
str
(
export_time
/
60.
),
'-type'
,
'f'
,
'-path'
,
'*/srv/backup/*'
)).
splitlines
()
if
not
modified_files
:
return
()
# filter those modified files through getExcludePathList.
# Indeed, some modified files may be listed in getExcludePathList and in this
# case, we won't copy them to PBS so it's not really important if they are
# modified.
exclude_file_list
=
getExcludePathList
(
os
.
getcwd
())
not_important_modified_file_list
=
filter
(
lambda
f
:
isFilepathMatching
(
f
,
exclude_file_list
),
modified_files
)
if
not_important_modified_file_set
:
print
(
"WARNING: The following files in srv/backup were modified since the exporter started (srv/backup should contain almost static files):"
,
*
sorted
(
not_important_modified_file_set
),
sep
=
'
\
n
'
)
return
set
(
modified_files
).
difference
(
not_important_modified_file_list
)
def
runExport
():
export_start_date
=
int
(
time
.
time
())
...
...
@@ -146,11 +165,11 @@ def runExport():
time
.
sleep
(
10
)
# Check that export didn't happen during backup of instances
with
CwdContextManager
(
os
.
path
.
join
(
runner_working_path
,
'instance'
)
):
modified_file_list
=
getBackupFilesModifiedDuringExportList
(
export_start_date
)
with
CwdContextManager
(
runner_working_path
):
modified_file_list
=
getBackupFilesModifiedDuringExportList
(
args
,
export_start_date
)
if
len
(
modified_file_list
):
print
(
"ERROR:
Files were modified since the backup started
, exporter should be re-run."
" Let's sleep %s minutes, to let the backup end.
Modified files:
\
n
%s"
%
(
print
(
"ERROR:
The following files in srv/backup were modified since the exporter started. Since they must be backup
, exporter should be re-run."
" Let's sleep %s minutes, to let the backup end.
\
n
%s"
%
(
args
.
backup_wait_time
,
'
\
n
'
.
join
(
modified_file_list
)))
time
.
sleep
(
args
.
backup_wait_time
*
60
)
sys
.
exit
(
1
)
slapos/test/test_runner_exporter.py
View file @
c55e1100
...
...
@@ -40,7 +40,15 @@ __buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7
recipe = slapos.recipe.template:jinja2
rendered = /some/prefix/slappart18/test/srv/exporter.exclude
template = inline:
srv/backup/**"""
srv/backup/*.log
[exclude1]
__buildout_installed__ = {cwd}/instance/slappart1/srv/exporter.exclude
__buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7.egg zc.buildout-2.12.2-py2.7.egg slapos.recipe.template-4.3-py2.7.egg setuptools-40.4.3-py2.7.egg
recipe = slapos.recipe.template:jinja2
rendered = /some/prefix/slappart18/test/srv/exporter.exclude
template = inline:
srv/backup/log/**"""
class
Config
():
...
...
@@ -79,21 +87,32 @@ class TestRunnerExporter(unittest.TestCase):
os
.
makedirs
(
'instance/slappart0/srv/backup'
)
os
.
makedirs
(
'instance/slappart1/etc'
)
os
.
makedirs
(
'instance/slappart1/srv/backup'
)
os
.
makedirs
(
'instance/slappart1/srv/backup
/log
'
)
self
.
_createFile
(
'instance/slappart0/.installed.cfg'
,
tested_instance_cfg
.
format
(
cwd
=
os
.
getcwd
()))
self
.
_createFile
(
'instance/slappart0/srv/backup/data.dat'
,
'all my fortune lays on this secret !'
)
self
.
_createFile
(
'instance/slappart0/srv/backup/data.log'
,
'this log is not so important !'
)
self
.
_createFile
(
'instance/slappart0/srv/exporter.exclude'
,
'srv/backup/*
*
'
)
'srv/backup/*
.log
'
)
self
.
_createFile
(
'instance/slappart0/etc/config.json'
)
self
.
_createFile
(
'instance/slappart0/etc/.parameters.xml'
)
self
.
_createFile
(
'instance/slappart0/etc/.project'
,
'workspace/slapos-dev/software/erp5'
)
self
.
_createFile
(
'instance/slappart1/srv/backup/data.dat'
,
'This is important data also !'
)
self
.
_createFile
(
'instance/slappart1/srv/backup/log/log1'
,
'First log'
)
self
.
_createFile
(
'instance/slappart1/srv/backup/log/log2'
,
'Second log'
)
self
.
_createFile
(
'instance/slappart1/srv/exporter.exclude'
,
'srv/backup/log/**'
)
self
.
_createExecutableFile
(
'instance/slappart1/srv/.backup_identity_script'
,
"#!/bin/sh
\
n
exec xargs -0 md5sum"
...
...
@@ -120,8 +139,10 @@ class TestRunnerExporter(unittest.TestCase):
'.installed*.cfg'
,
'instance/slappart0/etc/nicolas.txt'
,
'instance/slappart0/etc/rafael.txt'
,
'instance/slappart0/srv/backup/*
*
'
,
'instance/slappart0/srv/backup/*
.log
'
,
'instance/slappart0/srv/exporter.exclude'
,
'instance/slappart1/srv/backup/log/**'
,
'instance/slappart1/srv/exporter.exclude'
,
]
)
...
...
@@ -155,7 +176,7 @@ class TestRunnerExporter(unittest.TestCase):
self
.
assertEqual
(
check_output_mock
.
call_count
,
1
)
check_output_mock
.
assert_any_call
(
[
'rsync'
,
'-rlptgov'
,
'--stats'
,
'--safe-links'
,
'--ignore-missing-args'
,
'--delete'
,
'--delete-excluded'
,
'--exclude=*.pid'
,
'--exclude=*.sock'
,
'--exclude=*.socket'
,
'--exclude=.installed*.cfg'
,
'--exclude=instance/slappart0/etc/nicolas.txt'
,
'--exclude=instance/slappart0/etc/rafael.txt'
,
'--exclude=instance/slappart0/srv/backup/*
*'
,
'--exclude=instance/slappart0
/srv/exporter.exclude'
,
'instance'
,
'project'
,
'proxy.db'
,
'public'
,
'backup/runner/runner'
]
[
'rsync'
,
'-rlptgov'
,
'--stats'
,
'--safe-links'
,
'--ignore-missing-args'
,
'--delete'
,
'--delete-excluded'
,
'--exclude=*.pid'
,
'--exclude=*.sock'
,
'--exclude=*.socket'
,
'--exclude=.installed*.cfg'
,
'--exclude=instance/slappart0/etc/nicolas.txt'
,
'--exclude=instance/slappart0/etc/rafael.txt'
,
'--exclude=instance/slappart0/srv/backup/*
.log'
,
'--exclude=instance/slappart0/srv/exporter.exclude'
,
'--exclude=instance/slappart1/srv/backup/log/**'
,
'--exclude=instance/slappart1
/srv/exporter.exclude'
,
'instance'
,
'project'
,
'proxy.db'
,
'public'
,
'backup/runner/runner'
]
)
def
test_getSlappartSignatureMethodDict
(
self
):
...
...
@@ -204,18 +225,20 @@ class TestRunnerExporter(unittest.TestCase):
def
test_getBackupFilesModifiedDuringExportList
(
self
):
self
.
_setUpFakeInstanceFolder
()
with
runner_exporter
.
CwdContextManager
(
'instance'
):
self
.
assertEqual
(
runner_exporter
.
getBackupFilesModifiedDuringExportList
(
time
.
time
()
-
5
),
[
'./slappart0/srv/backup/data.dat'
]
)
time
.
sleep
(
2
)
self
.
assertEqual
(
runner_exporter
.
getBackupFilesModifiedDuringExportList
(
time
.
time
()
-
1
),
[]
)
self
.
_createFile
(
'slappart1/srv/backup/bakckup.data'
,
'my backup'
)
self
.
assertEqual
(
runner_exporter
.
getBackupFilesModifiedDuringExportList
(
time
.
time
()
-
1
),
[
'./slappart1/srv/backup/bakckup.data'
]
)
config
=
Config
()
config
.
rsync_binary
=
'rsync'
self
.
assertEqual
(
runner_exporter
.
getBackupFilesModifiedDuringExportList
(
config
,
time
.
time
()
-
5
),
[
'instance/slappart0/srv/backup/data.dat'
,
'instance/slappart1/srv/backup/data.dat'
]
)
time
.
sleep
(
2
)
self
.
assertFalse
(
runner_exporter
.
getBackupFilesModifiedDuringExportList
(
config
,
time
.
time
()
-
1
)
)
self
.
_createFile
(
'instance/slappart1/srv/backup/bakckup.data'
,
'my backup'
)
self
.
assertEqual
(
runner_exporter
.
getBackupFilesModifiedDuringExportList
(
config
,
time
.
time
()
-
1
),
[
'instance/slappart1/srv/backup/bakckup.data'
]
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment