Commit 4ee761cd authored by Vincent Pelletier's avatar Vincent Pelletier Committed by Rafael Monnerat

software/slaprunner: Rework rsync workhorse.

Do not apply exclusions to unrelated paths ("project", "proxy.db" and
"etc").
Simplify global exclusion file generation.
Add support for absolute and partition-relative exclusion paths
(previously, only partition-relative were supported).
Pipe exclusion file to rsync, to avoid an intermediate disk write.
Remove some directory prefix duplication by using PWD-relative paths
(except in one place, for runner-import.sh.jinja2 comparison).
parent af99eb04
......@@ -15,7 +15,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 02755403c9f0b52d717160b0b2d0cfb7
md5sum = afd426c01891d06e95f17b622ecd172f
[template-runner]
filename = instance-runner.cfg
......@@ -31,11 +31,11 @@ md5sum = 9db9957f452bda370cb2d5cc2e833e85
[template-runner-export-script]
filename = template/runner-export.sh.jinja2
md5sum = 7ba7ede9bf850551a9add58812e95d6e
md5sum = a8cb62a948300e6641af935e81ac4aef
[instance-runner-export]
filename = instance-runner-export.cfg.in
md5sum = 852a2ed99af566d27e5e4403334a3376
md5sum = e9aa653417a85ed2a7a7ccfb64668ace
[template-resilient]
filename = instance-resilient.cfg.jinja2
......
......@@ -2,6 +2,7 @@
extends =
buildout.hash.cfg
../../component/bash/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/busybox/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
......
......@@ -62,6 +62,9 @@ ip = ${slaprunner:ipv4}
[supervisord]
port = ${supervisord-free-port:port}
[exporter-configuration]
coreutils-location = {{ dumps(parameter_dict['coreutils-location']) }}
[exporter]
recipe = slapos.recipe.template:jinja2
template = {{ exporter_script_path }}
......@@ -71,6 +74,7 @@ wrapper = ${:rendered}
mode = 700
context =
section directory directory
section parameter_dict exporter-configuration
raw output_log_file ${directory:log}/resilient.log
raw shell_binary {{ bash_executable_location }}
raw rsync_binary {{ rsync_executable_location }}
......
......@@ -47,6 +47,9 @@ import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
mode = 0644
[template-runner-export-configuration]
coreutils-location = ${coreutils:location}
[template-runner-export]
recipe = slapos.recipe.template:jinja2
template = ${instance-runner-export:target}
......@@ -55,6 +58,7 @@ mode = 640
context =
key pbsready_export_template_path template-pbsready-export:rendered
key template_runner_path instance-base-runner:rendered
section parameter_dict template-runner-export-configuration
raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
raw buildout_executable_location ${buildout:executable}
......
......@@ -18,50 +18,57 @@ backup_directory='{{ directory["backup"] }}'
etc_directory='{{ directory["etc"] }}'
tmp_directory='{{ directory["tmp"] }}'
sync_element () {
path=$1
backup_path=$2
shift 2
element_list=$*
rsync () {
set -x
'{{ rsync_binary }}' -rlptgov --stats --safe-links --delete --delete-excluded "$@"
set +x
}
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
exclude_content="instance/supervisord.socket"
for partition in $srv_directory/runner/instance/slappart*
do
exclude_file="$partition/srv/exporter.exclude"
if [ -e "$exclude_file" ]; then
partition_exclude_content_relative=$(cat "$exclude_file")
# For every line of the local exclude file, add the absolute path
for line in "$partition_exclude_content_relative"
do
relativise () {
while IFS= read -r line; do
if [ ! -z "$line" ]; then
exclude_content="$exclude_content\ninstance/$(basename $partition)/$line"
'{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
fi
done
}
(
path=$srv_directory/runner
backup_path=$backup_directory/runner/
cd "$path"
if [ -d instance ]; then
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
(
echo "*.sock"
echo "*.socket"
echo "*.pid"
echo ".installed*.cfg"
for partition in "$path"/instance/slappart*; do
# So "relativise" can handle relative paths (which are expected to be relative to partition).
cd "$partition"
exclude_file=srv/exporter.exclude
if [ -r "$exclude_file" ]; then
relativise "$path" < "$exclude_file"
fi
done
echo -e "$exclude_content" > $srv_directory/exporter.exclude
for element in $element_list
do
echo "Changing current directory to $path."
cd $path;
if [ -f $element ] || [ -d $element ]; then
command="{{ rsync_binary }} -rlptgov --stats --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element $backup_path"
echo "Running : $command"
$command
) | rsync --exclude-from=- instance "$backup_path"
fi
done
}
sync_element $srv_directory/runner {{ directory['backup'] }}/runner/ instance project proxy.db
test -d project && rsync project "$backup_path"
test -f proxy.db && rsync proxy.db "$backup_path"
)
# We sync .* appart
date +%s -u > {{ directory['etc'] }}/.resilient-timestamp
sync_element {{ directory['etc'] }} {{ directory['backup'] }}/etc/ config.json
# Hidden files are related to the webrunner's internals
cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/
if [ -d {{ directory['backup'] }}/runner/software ]; then
rm {{ directory['backup'] }}/runner/software/*
(
cd "$etc_directory"
date +%s -u > .resilient-timestamp
rsync config.json "$backup_directory"/etc/
# Hidden files are related to the webrunner's internals
cp -r .??* "$backup_directory/etc/"
)
if [ -d "$backup_directory"/runner/software ]; then
rm "$backup_directory"/runner/software/*
fi
(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment