Commit 50d5e186 authored by Julien Muchembled's avatar Julien Muchembled

slaprunner: partial rewrite of import/export scrips in Python

Further rewrite should be done but at least there's no awk code anymore.
parent f92aefb0
[buildout]
parts = gawk
[gawk]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/gawk/gawk-4.1.4.tar.gz
md5sum = f20c94ca51b6ebfc9bffb90f95c8ffbb
......@@ -15,7 +15,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = e89d25c1bb7bf507bd96973d444934a6
md5sum = 78e4c796b15a9c5b9b41c79a3c995ee7
[template-runner]
filename = instance-runner.cfg
......@@ -23,19 +23,19 @@ md5sum = c76c66c331f1f0d9f416c56ead53e804
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
md5sum = feee1dc29d160e0d796dab0ba767745f
md5sum = 40e839e76d03fabefb1859ea72383529
[instance-runner-import]
filename = instance-runner-import.cfg.in
md5sum = 70931f214b3f7c3992e29c150a30d3cf
md5sum = 5d49f4fe6de7bd02db0067d0aeac6de4
[template-runner-export-script]
filename = template/runner-export.sh.jinja2
md5sum = b921435ab5a8c408026e74d9a40eea44
md5sum = 02023f89b622c49aee42130f839846a7
[instance-runner-export]
filename = instance-runner-export.cfg.in
md5sum = 546b368baaed84bd2d3be7d6b68a5030
md5sum = 22a5dfe9bfc8533f8eab851e0996a6d7
[template-resilient]
filename = instance-resilient.cfg.jinja2
......
......@@ -2,12 +2,10 @@
extends =
buildout.hash.cfg
../../component/bash/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/busybox/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/gawk/buildout.cfg
../../component/git/buildout.cfg
../../component/tig/buildout.cfg
../../component/logrotate/buildout.cfg
......
......@@ -62,11 +62,6 @@ ip = ${slaprunner:ipv4}
[supervisord]
port = ${supervisord-free-port:port}
[exporter-configuration]
coreutils-location = {{ dumps(parameter_dict['coreutils-location']) }}
gawk-location = {{ dumps(parameter_dict['gawk-location']) }}
backup_wait_time = {{ dumps(parameter_dict['backup_wait_time']) }}
[exporter]
recipe = slapos.recipe.template:jinja2
template = {{ exporter_script_path }}
......@@ -75,8 +70,10 @@ rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
wrapper = ${:rendered}
mode = 700
context =
import sys sys
import easy_install zc.buildout.easy_install
section directory directory
section parameter_dict exporter-configuration
raw backup_wait_time {{ backup_wait_time }}
raw output_log_file ${directory:log}/resilient.log
raw shell_binary {{ bash_executable_location }}
raw rsync_binary {{ rsync_executable_location }}
......
......@@ -72,10 +72,6 @@ maximum = 22241
[cron-entry-prepare-software]
recipe =
[importer-configuration]
coreutils-location = ${coreutils:location}
gawk-location = ${gawk:location}
[importer]
recipe = slapos.recipe.template:jinja2
template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
......@@ -89,12 +85,13 @@ restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-ba
restore-error-message-file-basename = importer-error-message-file
resilient-log-basename = resilient.log
context =
import sys sys
import easy_install zc.buildout.easy_install
key backend_url slaprunner:access-url
key ipv4 slaprunner:ipv4
key ipv6 slaprunner:ipv6
key proxy_port slaprunner:proxy_port
key instance_folder slaprunner:instance_root
section parameter_dict importer-configuration
section directory directory
section supervisord supervisord
raw output_log_file $${directory:log}/$${:resilient-log-basename}
......
......@@ -47,11 +47,6 @@ import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
mode = 0644
[template-runner-export-configuration]
coreutils-location = ${coreutils:location}
gawk-location = ${gawk:location}
backup_wait_time = ${exporter-default-configuration:backup_wait_time}
[template-runner-export]
recipe = slapos.recipe.template:jinja2
template = ${instance-runner-export:target}
......@@ -60,7 +55,7 @@ mode = 640
context =
key pbsready_export_template_path template-pbsready-export:rendered
key template_runner_path instance-base-runner:rendered
section parameter_dict template-runner-export-configuration
raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
raw buildout_executable_location ${buildout:executable}
......
......@@ -24,15 +24,8 @@ rsync () {
set +x
}
relativise () {
while IFS= read -r line; do
if [ ! -z "$line" ]; then
'{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
fi
done
}
(
# XXX: code duplication with runner-import.sh.jinja2
path=$srv_directory/runner
backup_path=$backup_directory/runner/
cd "$path"
......@@ -41,31 +34,46 @@ relativise () {
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
# Also, ignore all buildout-managed files.
(
echo "*.sock"
echo "*.socket"
echo "*.pid"
echo ".installed*.cfg"
for partition in "$path"/instance/slappart*; do
# So "relativise" can handle relative paths (which are expected to be relative to partition).
cd "$partition"
exclude_file=srv/exporter.exclude
if [ -r "$exclude_file" ]; then
relativise "$path" < "$exclude_file"
fi
for installed in .installed*.cfg; do
if [ -r "$installed" ]; then
# Print every line from each __buildout_installed__ found.
'{{ parameter_dict["gawk-location"] }}/bin/gawk' '
BEGIN { do_print = 0 }
match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next }
/^\S/ { do_print = 0; next }
match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] }
' "$installed" | relativise "$path"
fi
done
done
) | rsync --exclude-from=- instance "$backup_path"
exclude=$({{ sys.executable }} - "$path" <<EOF
if 1:
import glob, errno, os, sys
sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
from zc.buildout.configparser import parse
path = sys.argv[1]
def print_relative(path_list):
for p in path_list:
p = p.strip()
if p:
print(os.path.relpath(p, path))
print("*.sock")
print("*.socket")
print("*.pid")
print(".installed*.cfg")
for partition in glob.glob(path + "/instance/slappart*"):
os.chdir(partition)
try:
with open("srv/exporter.exclude") as f:
exclude = f.readlines()
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "$exclude" |rsync --exclude-from=- instance "$backup_path"
fi
test -d project && rsync project "$backup_path"
......@@ -116,6 +124,6 @@ if diff "$tmp_backup_sum" "$tmp_filtered_signature"; then
exit 0
fi
echo "ERROR: Some backups are not consistent, exporter should be re-run."
echo "Let's sleep {{ parameter_dict['backup_wait_time'] }} minutes, to let the backup end..."
sleep {{ parameter_dict["backup_wait_time"] }}m
echo "Let's sleep {{ backup_wait_time }} minutes, to let the backup end..."
sleep {{ backup_wait_time }}m
exit 1
......@@ -41,16 +41,9 @@ rsync () {
set +x
}
relativise () {
while IFS= read -r line; do
if [ ! -z "$line" ]; then
'{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
fi
done
}
log_message "Restoring WebRunner content..."
(
# XXX: code duplication with runner-export.sh.jinja2
path=$srv_directory/runner
backup_path=$backup_directory/runner/
cd "$backup_path"
......@@ -59,31 +52,46 @@ log_message "Restoring WebRunner content..."
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
# Also, ignore all buildout-managed files.
(
echo "*.sock"
echo "*.socket"
echo "*.pid"
echo ".installed*.cfg"
for partition in "$path"/instance/slappart*; do
# So "relativise" can handle relative paths (which are expected to be relative to partition).
cd "$partition"
exclude_file=srv/exporter.exclude
if [ -r "$exclude_file" ]; then
relativise "$path" < "$exclude_file"
fi
for installed in .installed*.cfg; do
if [ -r "$installed" ]; then
# Print every line from each __buildout_installed__ found.
'{{ parameter_dict["gawk-location"] }}/bin/gawk' '
BEGIN { do_print = 0 }
match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next }
/^\S/ { do_print = 0; next }
match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] }
' "$installed" | relativise "$path"
fi
done
done
) | rsync --exclude-from=- instance "$path"
exclude=$({{ sys.executable }} - "$path" <<EOF
if 1:
import glob, errno, os, sys
sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
from zc.buildout.configparser import parse
path = sys.argv[1]
def print_relative(path_list):
for p in path_list:
p = p.strip()
if p:
print(os.path.relpath(p, path))
print("*.sock")
print("*.socket")
print("*.pid")
print(".installed*.cfg")
for partition in glob.glob(path + "/instance/slappart*"):
os.chdir(partition)
try:
with open("srv/exporter.exclude") as f:
exclude = f.readlines()
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "$exclude" |rsync --exclude-from=- instance "$path"
fi
test -d project && rsync project "$path"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment