Commit 50d5e186 authored by Julien Muchembled's avatar Julien Muchembled

slaprunner: partial rewrite of import/export scrips in Python

Further rewrite should be done but at least there's no awk code anymore.
parent f92aefb0
[buildout]
parts = gawk
[gawk]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/gawk/gawk-4.1.4.tar.gz
md5sum = f20c94ca51b6ebfc9bffb90f95c8ffbb
......@@ -15,7 +15,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = e89d25c1bb7bf507bd96973d444934a6
md5sum = 78e4c796b15a9c5b9b41c79a3c995ee7
[template-runner]
filename = instance-runner.cfg
......@@ -23,19 +23,19 @@ md5sum = c76c66c331f1f0d9f416c56ead53e804
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
md5sum = feee1dc29d160e0d796dab0ba767745f
md5sum = 40e839e76d03fabefb1859ea72383529
[instance-runner-import]
filename = instance-runner-import.cfg.in
md5sum = 70931f214b3f7c3992e29c150a30d3cf
md5sum = 5d49f4fe6de7bd02db0067d0aeac6de4
[template-runner-export-script]
filename = template/runner-export.sh.jinja2
md5sum = b921435ab5a8c408026e74d9a40eea44
md5sum = 02023f89b622c49aee42130f839846a7
[instance-runner-export]
filename = instance-runner-export.cfg.in
md5sum = 546b368baaed84bd2d3be7d6b68a5030
md5sum = 22a5dfe9bfc8533f8eab851e0996a6d7
[template-resilient]
filename = instance-resilient.cfg.jinja2
......
......@@ -2,12 +2,10 @@
extends =
buildout.hash.cfg
../../component/bash/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/busybox/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/gawk/buildout.cfg
../../component/git/buildout.cfg
../../component/tig/buildout.cfg
../../component/logrotate/buildout.cfg
......
......@@ -62,11 +62,6 @@ ip = ${slaprunner:ipv4}
[supervisord]
port = ${supervisord-free-port:port}
[exporter-configuration]
coreutils-location = {{ dumps(parameter_dict['coreutils-location']) }}
gawk-location = {{ dumps(parameter_dict['gawk-location']) }}
backup_wait_time = {{ dumps(parameter_dict['backup_wait_time']) }}
[exporter]
recipe = slapos.recipe.template:jinja2
template = {{ exporter_script_path }}
......@@ -75,8 +70,10 @@ rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
wrapper = ${:rendered}
mode = 700
context =
import sys sys
import easy_install zc.buildout.easy_install
section directory directory
section parameter_dict exporter-configuration
raw backup_wait_time {{ backup_wait_time }}
  • Bad raw usage, allows buildout syntax injection. Use a separate section and dumps(), as was done before.

    [EDIT]: Actually, the injection is likely not at buildout syntax level (depending on expansion effect on parsing), but if not then it is at jinja2 template context declaraion level. Ex: backup_wait_time = 'dummy\n import some_value some_module'.

    Edited by Vincent Pelletier
  • "syntax injection" > why should we protect against it ?

    Working on instantiation is a task that I find complicated enough like that, for example with all those values that have to be propagated (even in places, where I'd have preferred it was implicit). The previous form looked too much for me.

Please register or sign in to reply
raw output_log_file ${directory:log}/resilient.log
raw shell_binary {{ bash_executable_location }}
raw rsync_binary {{ rsync_executable_location }}
......
......@@ -72,10 +72,6 @@ maximum = 22241
[cron-entry-prepare-software]
recipe =
[importer-configuration]
coreutils-location = ${coreutils:location}
gawk-location = ${gawk:location}
[importer]
recipe = slapos.recipe.template:jinja2
template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
......@@ -89,12 +85,13 @@ restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-ba
restore-error-message-file-basename = importer-error-message-file
resilient-log-basename = resilient.log
context =
import sys sys
import easy_install zc.buildout.easy_install
key backend_url slaprunner:access-url
key ipv4 slaprunner:ipv4
key ipv6 slaprunner:ipv6
key proxy_port slaprunner:proxy_port
key instance_folder slaprunner:instance_root
section parameter_dict importer-configuration
section directory directory
section supervisord supervisord
raw output_log_file $${directory:log}/$${:resilient-log-basename}
......
......@@ -47,11 +47,6 @@ import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
mode = 0644
[template-runner-export-configuration]
coreutils-location = ${coreutils:location}
gawk-location = ${gawk:location}
backup_wait_time = ${exporter-default-configuration:backup_wait_time}
[template-runner-export]
recipe = slapos.recipe.template:jinja2
template = ${instance-runner-export:target}
......@@ -60,7 +55,7 @@ mode = 640
context =
key pbsready_export_template_path template-pbsready-export:rendered
key template_runner_path instance-base-runner:rendered
section parameter_dict template-runner-export-configuration
raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
Please register or sign in to reply
raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
raw buildout_executable_location ${buildout:executable}
......
......@@ -24,15 +24,8 @@ rsync () {
set +x
}
relativise () {
while IFS= read -r line; do
if [ ! -z "$line" ]; then
'{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
fi
done
}
(
# XXX: code duplication with runner-import.sh.jinja2
path=$srv_directory/runner
backup_path=$backup_directory/runner/
cd "$path"
......@@ -41,31 +34,46 @@ relativise () {
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
# Also, ignore all buildout-managed files.
(
echo "*.sock"
echo "*.socket"
echo "*.pid"
echo ".installed*.cfg"
for partition in "$path"/instance/slappart*; do
# So "relativise" can handle relative paths (which are expected to be relative to partition).
cd "$partition"
exclude_file=srv/exporter.exclude
if [ -r "$exclude_file" ]; then
relativise "$path" < "$exclude_file"
fi
for installed in .installed*.cfg; do
if [ -r "$installed" ]; then
# Print every line from each __buildout_installed__ found.
'{{ parameter_dict["gawk-location"] }}/bin/gawk' '
BEGIN { do_print = 0 }
match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next }
/^\S/ { do_print = 0; next }
match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] }
' "$installed" | relativise "$path"
fi
done
done
) | rsync --exclude-from=- instance "$backup_path"
exclude=$({{ sys.executable }} - "$path" <<EOF
if 1:
import glob, errno, os, sys
sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
from zc.buildout.configparser import parse
path = sys.argv[1]
def print_relative(path_list):
for p in path_list:
p = p.strip()
if p:
print(os.path.relpath(p, path))
print("*.sock")
print("*.socket")
print("*.pid")
print(".installed*.cfg")
for partition in glob.glob(path + "/instance/slappart*"):
os.chdir(partition)
try:
with open("srv/exporter.exclude") as f:
exclude = f.readlines()
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "$exclude" |rsync --exclude-from=- instance "$backup_path"
fi
test -d project && rsync project "$backup_path"
......@@ -116,6 +124,6 @@ if diff "$tmp_backup_sum" "$tmp_filtered_signature"; then
exit 0
fi
echo "ERROR: Some backups are not consistent, exporter should be re-run."
echo "Let's sleep {{ parameter_dict['backup_wait_time'] }} minutes, to let the backup end..."
sleep {{ parameter_dict["backup_wait_time"] }}m
echo "Let's sleep {{ backup_wait_time }} minutes, to let the backup end..."
sleep {{ backup_wait_time }}m
exit 1
......@@ -41,16 +41,9 @@ rsync () {
set +x
}
relativise () {
while IFS= read -r line; do
if [ ! -z "$line" ]; then
'{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
fi
done
}
log_message "Restoring WebRunner content..."
(
# XXX: code duplication with runner-export.sh.jinja2
path=$srv_directory/runner
backup_path=$backup_directory/runner/
cd "$backup_path"
......@@ -59,31 +52,46 @@ log_message "Restoring WebRunner content..."
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
# Also, ignore all buildout-managed files.
(
echo "*.sock"
echo "*.socket"
echo "*.pid"
echo ".installed*.cfg"
for partition in "$path"/instance/slappart*; do
# So "relativise" can handle relative paths (which are expected to be relative to partition).
cd "$partition"
exclude_file=srv/exporter.exclude
if [ -r "$exclude_file" ]; then
relativise "$path" < "$exclude_file"
fi
for installed in .installed*.cfg; do
if [ -r "$installed" ]; then
# Print every line from each __buildout_installed__ found.
'{{ parameter_dict["gawk-location"] }}/bin/gawk' '
BEGIN { do_print = 0 }
match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next }
/^\S/ { do_print = 0; next }
match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] }
' "$installed" | relativise "$path"
fi
done
done
) | rsync --exclude-from=- instance "$path"
exclude=$({{ sys.executable }} - "$path" <<EOF
if 1:
import glob, errno, os, sys
sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
from zc.buildout.configparser import parse
path = sys.argv[1]
def print_relative(path_list):
for p in path_list:
p = p.strip()
if p:
print(os.path.relpath(p, path))
print("*.sock")
print("*.socket")
print("*.pid")
print(".installed*.cfg")
for partition in glob.glob(path + "/instance/slappart*"):
os.chdir(partition)
try:
with open("srv/exporter.exclude") as f:
exclude = f.readlines()
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "$exclude" |rsync --exclude-from=- instance "$path"
fi
test -d project && rsync project "$path"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment