Commit 87c805d6 authored by Łukasz Nowak's avatar Łukasz Nowak Committed by Matevz Golob

caddy-frontend: Improve validation with decoupled configuration state calculation

Validation happens on each configuration change, but for sure it is checked
each 2 hours.

State of configuration is calculated in separate script.
parent 99e59aef
......@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
[template-apache-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 77b06f3a7bf67c990a363f8d5155596c
md5sum = ade2eb66b2123d7f8ff5df5a37720dbe
[template-apache-replicate]
filename = instance-apache-replicate.cfg.in
......@@ -94,11 +94,15 @@ md5sum = b9f73f6323f9fceea054c46c854d2862
[template-graceful-script]
filename = templates/graceful-script.sh.in
md5sum = 959f49437cffb096368f0b6434f06d3d
md5sum = 061cc244558fd3af2b6bacf17cae5555
[template-validate-script]
filename = templates/validate-script.sh.in
md5sum = 3174bf29f439d14e292f8719bba19a7a
md5sum = 89839a3f4ab71cab459afd1c27d00326
[template-configuration-state-script]
filename = templates/configuration-state-script.sh.in
md5sum = 00dfdc488548d1b14e89ba5f2f08fcb8
[template-rotate-script]
filename = templates/rotate-script.sh.in
......
......@@ -122,6 +122,7 @@ template_caddy_frontend_configuration = ${template-caddy-frontend-configuration:
template_graceful_script = ${template-graceful-script:target}
template_validate_script = ${template-validate-script:target}
template_rotate_script = ${template-rotate-script:target}
template_configuration_state_script = ${template-configuration-state-script:target}
template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
template_empty = ${template-empty:target}
......@@ -256,6 +257,10 @@ filename = graceful-script.sh.in
<=download-template
filename = validate-script.sh.in
[template-configuration-state-script]
<=download-template
filename = configuration-state-script.sh.in
# Migrated from KVM recipe
[http-proxy]
# https://github.com/nodejitsu/node-http-proxy
......
......@@ -9,7 +9,6 @@ extends =
parts =
directory
configtest
logrotate-entry-caddy
caddy-frontend
switch-caddy-softwaretype
......@@ -371,8 +370,7 @@ frontend-configuration = ${directory:etc}/Caddyfile
access-log = ${directory:log}/frontend-access.log
error-log = ${directory:log}/frontend-error.log
pid-file = ${directory:run}/httpd.pid
frontend-configuration-verification = ${frontend-caddy-validate:rendered}
frontend-graceful-command = ${:frontend-configuration-verification} && kill -USR1 $(cat ${:pid-file})
frontend-graceful-command = ${frontend-caddy-validate:rendered} && kill -USR1 $(cat ${:pid-file})
not-found-file = ${caddy-directory:document-root}/notfound.html
master-certificate = ${caddy-directory:master-autocert-dir}/master.pem
# Communication with ATS
......@@ -380,11 +378,6 @@ cache-port = ${trafficserver-variable:input-port}
cache-through-port = 26011
ssl-cache-through-port = 26012
[configtest]
recipe = slapos.cookbook:wrapper
command-line = ${frontend-caddy-validate:rendered}
wrapper-path = ${directory:bin}/caddy-configtest
# BBB: SlapOS Master non-zero knowledge BEGIN
[get-self-signed-fallback-access]
recipe = collective.recipe.shelloutput
......@@ -569,33 +562,48 @@ command = ${trafficserver-rotate-script:rendered}
### End of ATS sections
### Caddy Graceful and promises
[frontend-caddy-graceful]
[frontend-caddy-configuration-state]
< = jinja2-template-base
template = {{ parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/frontend-caddy-safe-graceful
template = {{ parameter_dict['template_configuration_state_script'] }}
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
path_list = ${caddy-configuration:frontend-configuration} ${frontend-configuration:log-access-configuration} ${caddy-directory:slave-configuration}/*.conf ${caddy-directory:slave-with-cache-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*.pem ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.crt
sha256sum = {{ parameter_dict['sha256sum'] }}
signature_file = ${directory:run}/caddy_graceful_signature
extra-context =
key graceful_reload_command caddy-configuration:frontend-graceful-command
key path_list :path_list
key sha256sum :sha256sum
key signature_file :signature_file
[frontend-caddy-configuration-state-graceful]
< = frontend-caddy-configuration-state
signature_file = ${directory:run}/graceful_configuration_state_signature
[frontend-caddy-configuration-state-validate]
< = frontend-caddy-configuration-state
signature_file = ${directory:run}/validate_configuration_state_signature
[frontend-caddy-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/frontend-caddy-safe-graceful
mode = 0700
extra-context =
key graceful_reload_command caddy-configuration:frontend-graceful-command
key caddy_configuration_state frontend-caddy-configuration-state-graceful:rendered
[frontend-caddy-validate]
< = jinja2-template-base
template = {{ parameter_dict['template_validate_script'] }}
rendered = ${directory:bin}/frontend-caddy-validate
mode = 0700
sha256sum = {{ parameter_dict['sha256sum'] }}
signature_file = ${directory:run}/caddy_validate_signature
last_state_file = ${directory:run}/caddy_configuration_last_state
extra-context =
key wrapper caddy-wrapper:wrapper-path
key path_list frontend-caddy-graceful:path_list
key sha256sum :sha256sum
key signature_file :signature_file
key caddy_configuration_state frontend-caddy-configuration-state-validate:rendered
key last_state_file :last_state_file
[frontend-caddy-lazy-graceful]
< = jinja2-template-base
......
#!/bin/sh
set -e
SIGNATURE_FILE={{ signature_file }}
NSIGNATURE_FILE={{ signature_file }}.tmp
touch $SIGNATURE_FILE
{{ sha256sum }} {{ path_list }} 2> /dev/null | sort -k 66 > $NSIGNATURE_FILE
if diff "$SIGNATURE_FILE" "$NSIGNATURE_FILE" > /dev/null ; then
rm -f ${NSIGNATURE_FILE}
exit 1
else
mv "$NSIGNATURE_FILE" "$SIGNATURE_FILE"
exit 0
fi
#!/bin/sh
SIGNATURE_FILE={{ signature_file }}
NSIGNATURE_FILE={{ signature_file }}.tmp
set -e
touch $SIGNATURE_FILE
{{ sha256sum }} {{ path_list }} 2> /dev/null | sort -k 66 > $NSIGNATURE_FILE
# If no diff, no restart for now
if diff "$SIGNATURE_FILE" "$NSIGNATURE_FILE" > /dev/null ; then
if {{ caddy_configuration_state }} ; then
echo "Reloading.."
{{ graceful_reload_command }}
else
echo "Nothing changed, so nothing to reload"
exit 0
fi
echo "Reloading.."
{{ graceful_reload_command }}
mv "$NSIGNATURE_FILE" "$SIGNATURE_FILE"
......@@ -2,21 +2,16 @@
set -e
SIGNATURE_FILE={{ signature_file }}
NSIGNATURE_FILE={{ signature_file }}.tmp
SIGNATURE_STATUS={{ signature_file }}.status
LAST_STATE_FILE={{ last_state_file }}
touch $SIGNATURE_FILE
{{ sha256sum }} {{ path_list }} 2> /dev/null | sort -k 66 > $NSIGNATURE_FILE
if diff "$SIGNATURE_FILE" "$NSIGNATURE_FILE" > /dev/null; then
rm -f "$NSIGNATURE_FILE"
else
mv "$NSIGNATURE_FILE" "$SIGNATURE_FILE"
# force validation each 2 hours
[ -f $LAST_STATE_FILE ] && find $LAST_STATE_FILE -mmin +120 -delete
configuration_state=$({{ caddy_configuration_state }})
if [ ! -f $LAST_STATE_FILE ] || $configuration_state ; then
# do not catch errors during validation
set +e
{{ wrapper }} -validate
echo $? > $SIGNATURE_STATUS
echo $? > $LAST_STATE_FILE
set -e
fi
exit `cat $SIGNATURE_STATUS`
exit `cat $LAST_STATE_FILE`
......@@ -343,8 +343,6 @@ class TestDataMixin(object):
self._test_file_list(['var', 'run'], [
# run by cron from time to time
'monitor/monitor-collect.pid',
# may appear or not
'caddy_graceful_signature.tmp',
])
def test_file_list_etc_cron_d(self):
......@@ -392,6 +390,12 @@ class TestDataMixin(object):
plugin = plugin_path[strip:]
if plugin in ignored_plugin_list:
continue
# reset frontend-caddy-configuration-promise.py state
if plugin == 'frontend-caddy-configuration-promise.py':
validate_path = os.path.join(
partition_path, 'bin', 'frontend-caddy-validate')
if os.path.exists(validate_path):
subprocess_status_output(validate_path)
plugin_status, plugin_result = subprocess_status_output([
runpromise_bin,
'-c', monitor_conf,
......
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,8 +2,8 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,8 +2,8 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
......@@ -2,9 +2,9 @@ T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment