Commit 9d866327 authored by Łukasz Nowak's avatar Łukasz Nowak Committed by Łukasz Nowak

caddy-frontend: Modernize profiles

Features:

 * jinja2 is used to generate instance templates
 * downloads are done the same way for all resources
 * create with shared content for all instance profiles
 * fill in instance-common with shared sections
 * render templates late in order to ease its extenension and development
 * drop not needd duplicated section
 * drop slap-parameter in frontend and replicate template
 * simplify monitor configuration
 * move instance-parameter to instance file
   Thanks to this only one and topmost profile is reponsible for parsing and
   passing through the information which comes from the network
parent 8d39cd73
No related merge requests found
......@@ -26,7 +26,7 @@ Generally things to be done with ``caddy-frontend``:
* ``apache-ca-certificate``
* ``apache-certificate`` and ``apache-key``
* change ``switch-softwaretype`` to way how ``software/erp5`` does, which will help with dropping jinja2 template for ``caddy-wrapper``, which is workaround for current situation, cf `note_62678 <https://lab.nexedi.com/nexedi/slapos/merge_requests/312#note_62678>`_
* drop jinja2 template for ``caddy-wrapper``
* use `slapos!326 <https://lab.nexedi.com/nexedi/slapos/merge_requests/326>`_, and especially `note about complex restart scenarios <https://lab.nexedi.com/nexedi/slapos/merge_requests/326#note_60198>`_, instead of self-developed graceful restart scripts
* move out ``test/utils.py`` and use it from shared python distribution
* provide various tricks for older browsers::
......@@ -61,7 +61,7 @@ Generally things to be done with ``caddy-frontend``:
<FilesMatch "\.(cgi|shtml|phtml|php)$">
SSLOptions +StdEnvVars
</FilesMatch>
* reduce the time of configuration validation (in ``instance-apache-frontend.cfg`` sections ``[configtest]``, ``[caddy-configuration]``, ``[nginx-configuration]``), as it is not scalable on frontend with 2000+ slaves (takes few minutes instead of few, < 5, seconds), issue posted `upstream <https://github.com/mholt/caddy/issues/2220>`_
* reduce the time of configuration validation (in ``instance-apache-frontend.cfg.in`` sections ``[configtest]``, ``[caddy-configuration]``, ``[nginx-configuration]``), as it is not scalable on frontend with 2000+ slaves (takes few minutes instead of few, < 5, seconds), issue posted `upstream <https://github.com/mholt/caddy/issues/2220>`_
* drop ``6tunnel`` and use ``bind`` in Caddy configuration, as soon as multiple binds will be possible, tracked in upstream `bind: support multiple values <https://github.com/mholt/caddy/pull/2128>`_ and `ipv6: does not bind on ipv4 and ipv6 for sites that resolve to both <https://github.com/mholt/caddy/issues/864>`_
* use caddy-frontend in `standalone style playbooks <https://lab.nexedi.com/nexedi/slapos.package/tree/master/playbook/roles/standalone-shared>`_
* ensure `QUIC <https://en.wikipedia.org/wiki/QUIC>`_ is used by caddy
......
......@@ -13,20 +13,24 @@
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg
md5sum = b73505ae80d6325a244f5094f8edc0ae
filename = instance.cfg.in
md5sum = 345b3a9b12241448913ec3ec8f929d51
[template-common]
filename = instance-common.cfg.in
md5sum = c801b7f9f11f0965677c22e6bbe9281b
[template-apache-frontend]
filename = instance-apache-frontend.cfg
md5sum = b170d0987563b481eb71cf705c3658ab
filename = instance-apache-frontend.cfg.in
md5sum = da1f58f77e81e11264e5e2131794dc19
[template-apache-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 27e98547061bd81e5f84cb7dd21b683b
md5sum = 8d34141a9cd1e51462aba845c7bea85b
[template-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = fb6c93f42f232e381174a5951c3fc222
md5sum = bdce0cf67a8f533891eabdec0b7eee87
[template-slave-configuration]
filename = templates/custom-virtualhost.conf.in
......@@ -34,7 +38,7 @@ md5sum = 54ae95597a126ae552c3a913ddf29e5e
[template-replicate-publish-slave-information]
filename = templates/replicate-publish-slave-information.cfg.in
md5sum = 8d318af17da5631d4242c0d6d1531066
md5sum = 6a308c29b54d53cfd82ae23ba77a35dd
[template-caddy-frontend-configuration]
filename = templates/Caddyfile.in
......@@ -42,7 +46,7 @@ md5sum = 6689d96fc18d9aad78d77fe87770d4da
[template-custom-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = fb6c93f42f232e381174a5951c3fc222
md5sum = bdce0cf67a8f533891eabdec0b7eee87
[caddy-backend-url-validator]
filename = templates/caddy-backend-url-validator.in
......
......@@ -34,16 +34,72 @@ eggs +=
websockify
erp5.util
[template-common]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-common.cfg.in
rendered = ${buildout:directory}/instance-common.cfg
mode = 0644
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
[template-frontend-parameter-section]
common_profile = ${template-common:rendered}
bin_directory = ${buildout:bin-directory}
sixtunnel = ${6tunnel:location}
caddy = ${caddy:output}
caddy_location = ${caddy:location}
curl = ${curl:location}
dash = ${dash:location}
dcron = ${dcron:location}
gzip = ${gzip:location}
logrotate = ${logrotate:location}
openssl = ${openssl:location}
trafficserver = ${trafficserver:location}
monitor_template = ${monitor-template:output}
template_cached_slave_virtualhost = ${template-cached-slave-virtualhost:target}
template_caddy_frontend_configuration = ${template-caddy-frontend-configuration:target}
template_caddy_graceful_script = ${template-caddy-graceful-script:target}
template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
template_caddy_wrapper = ${template-caddy-wrapper:output}
template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
template_empty = ${template-empty:target}
template_log_access = ${template-log-access:target}
template_nging_configuration = ${template-nginx-configuration:output}
template_nginx_eventsource_slave_virtualhost = ${template-nginx-eventsource-slave-virtualhost:target}
template_nginx_notebook_slave_virtualhost = ${template-nginx-notebook-slave-virtualhost:target}
template_not_found_html = ${template-not-found-html:target}
template_slave_configuration = ${template-slave-configuration:target}
template_slave_list = ${template-slave-list:target}
template_trafficserver_records_config = ${template-trafficserver-records-config:location}
template_trafficserver_records_config_filename = ${template-trafficserver-records-config:filename}
template_trafficserver_records_config_location = ${template-trafficserver-records-config:location}
template_trafficserver_storage_config_filename = ${template-trafficserver-storage-config:filename}
template_trafficserver_storage_config_location = ${template-trafficserver-storage-config:location}
template_wrapper = ${template-wrapper:output}
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in
rendered = ${buildout:directory}/template.cfg
mode = 0644
context =
key common_profile template-common:rendered
key monitor2_template monitor2-template:rendered
key template_caddy_frontend template-caddy-frontend:target
key template_caddy_replicate template-caddy-replicate:target
key template_replicate_publish_slave_information template-replicate-publish-slave-information:target
key caddy_backend_url_validator caddy-backend-url-validator:output
section template_frontend_parameter_dict template-frontend-parameter-section
[template-caddy-frontend]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-frontend.cfg
output = ${buildout:directory}/template-caddy-frontend.cfg
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
mode = 0644
[caddy-backend-url-validator]
......@@ -116,22 +172,14 @@ output = ${buildout:directory}/template-wrapper.cfg
mode = 0644
[template-trafficserver-records-config]
recipe = hexagonit.recipe.download
ignore-existing = true
<=download-template
url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = records.config.jinja2
download-only = true
mode = 0644
[template-trafficserver-storage-config]
recipe = hexagonit.recipe.download
ignore-existing = true
<=download-template
url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = storage.config.jinja2
download-only = true
mode = 0644
# NGINX Configuration
[template-nginx-configuration]
......
......@@ -6,10 +6,7 @@ rendered = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key slap_software_type slap-parameter:slap_software_type
key slave_instance_list slap-parameter:slave_instance_list
raw common_profile {{ common_profile }}
${:extra-context}
{% set part_list = [] %}
......@@ -173,7 +170,9 @@ monitor-url-list +=
{% endfor %}
[buildout]
extends = {{ template_monitor }}
extends =
{{ common_profile }}
{{ template_monitor }}
parts =
monitor-base
publish-slave-information
......@@ -182,23 +181,4 @@ parts =
{{ ' %s' % part }}
{% endfor %}
# publish-information
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
[slap-parameter]
slave_instance_list =
-frontend-quantity = 1
-frontend-type = single-default
{% endif %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
\ No newline at end of file
[buildout]
parts =
dynamic-template-caddy-replicate
switch-softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[slap-parameters]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/$${:filename}
extra-context =
context =
import json_module json
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key slap_software_type slap-parameters:slap-software-type
key slapparameter_dict slap-parameters:configuration
key slave_instance_list slap-parameters:slave-instance-list
$${:extra-context}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${dynamic-template-caddy-replicate:rendered}
RootSoftwareInstance = $${dynamic-template-caddy-replicate:rendered}
custom-personal = $${dynamic-template-caddy-replicate:rendered}
single-default = ${template-caddy-frontend:output}
single-custom-personal = ${template-caddy-frontend:output}
replicate = $${dynamic-template-caddy-replicate:rendered}
[dynamic-template-caddy-replicate]
< = jinja2-template-base
template = ${template-caddy-replicate:target}
filename = instance-caddy-replicate.cfg
extensions = jinja2.ext.do
extra-context =
import subprocess_module subprocess
raw caddy_backend_url_validator ${caddy-backend-url-validator:output}
raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
raw template_monitor ${monitor2-template:rendered}
[buildout]
extends = {{ common_profile }}
parts =
dynamic-template-caddy-replicate
switch-softwaretype
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
key slap_software_type instance-parameter:slap-software-type
key slapparameter_dict instance-parameter:configuration
key slave_instance_list instance-parameter:slave-instance-list
section instance_parameter instance-parameter
${:extra-context}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${dynamic-template-caddy-replicate:rendered}
RootSoftwareInstance = ${dynamic-template-caddy-replicate:rendered}
custom-personal = ${dynamic-template-caddy-replicate:rendered}
single-default = ${dynamic-template-caddy-frontend:rendered}
single-custom-personal = ${dynamic-template-caddy-frontend:rendered}
replicate = ${dynamic-template-caddy-replicate:rendered}
[dynamic-template-caddy-frontend-parameters]
{% for key,value in template_frontend_parameter_dict.iteritems() %}
{{ key }} = {{ dumps(value) }}
{% endfor -%}
[dynamic-template-caddy-frontend]
< = jinja2-template-base
template = {{ template_caddy_frontend }}
filename = instance-caddy-frontend.cfg
extensions = jinja2.ext.do
extra-context =
section parameter_dict dynamic-template-caddy-frontend-parameters
[dynamic-template-caddy-replicate]
< = jinja2-template-base
template = {{ template_caddy_replicate }}
filename = instance-caddy-replicate.cfg
extensions = jinja2.ext.do
extra-context =
import subprocess_module subprocess
raw caddy_backend_url_validator {{ caddy_backend_url_validator }}
raw template_publish_slave_information {{ template_replicate_publish_slave_information }}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
raw template_monitor {{ monitor2_template }}
raw common_profile {{ common_profile }}
[instance-parameter]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
# Define default parameter(s) that will be used later, in case user didn't
# specify it
# All parameters are available through the configuration.XX syntax.
# All possible parameters should have a default.
configuration.domain = example.org
configuration.public-ipv4 =
configuration.port = 4443
configuration.plain_http_port = 8080
configuration.plain_nginx_port = 8081
configuration.nginx_port = 9443
configuration.server-admin = admin@example.com
# BBB: apache_custom_https and apache_custom_http
configuration.apache_custom_https = ""
configuration.apache_custom_http = ""
configuration.caddy_custom_https = ""
configuration.caddy_custom_http = ""
configuration.apache-key =
configuration.apache-certificate =
configuration.apache-ca-certificate =
configuration.open-port = 80 443
configuration.extra_slave_instance_list =
configuration.disk-cache-size = 8G
configuration.ram-cache-size = 1G
configuration.trafficserver-autoconf-port = 8083
configuration.trafficserver-mgmt-port = 8084
configuration.re6st-verification-url = http://[2001:67c:1254:4::1]/index.html
configuration.enable-http2-by-default = true
configuration.mpm-graceful-shutdown-timeout = 5
configuration.monitor-httpd-port = 8072
configuration.frontend-name =
\ No newline at end of file
......@@ -20,8 +20,7 @@ recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
extra-context =
context =
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
raw common_profile {{ common_profile }}
${:extra-context}
{% do logrotate_dict.pop('recipe') %}
......@@ -395,6 +394,7 @@ slave-instance-information-list = {{ json_module.dumps(slave_instance_informatio
monitor-base-url = {{ monitor_base_url }}
[buildout]
extends = {{ common_profile }}
parts +=
slave-log-directories
{% for part in part_list %}
......@@ -409,9 +409,6 @@ parts +=
tunnel-6to4-base-nginx_http_port
tunnel-6to4-base-nginx_https_port
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
cache-access = {{ cache_access }}
{% endif %}
......@@ -41,11 +41,8 @@ log-access-url = {{ json_module.dumps(slave_information.pop('log-access-urls', 1
{% endfor %}
[buildout]
extends = {{ common_profile }}
parts =
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
\ No newline at end of file
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment