caddy-frontend: Modernize profiles

Features:

 * jinja2 is used to generate instance templates
 * downloads are done the same way for all resources
 * create with shared content for all instance profiles
 * fill in instance-common with shared sections
 * render templates late in order to ease its extenension and development
 * drop not needd duplicated section
 * drop slap-parameter in frontend and replicate template
 * simplify monitor configuration
 * move instance-parameter to instance file
   Thanks to this only one and topmost profile is reponsible for parsing and
   passing through the information which comes from the network

software/caddy-frontend/TODO.rst

@@ -26,7 +26,7 @@ Generally things to be done with ``caddy-frontend``:
   * ``apache-ca-certificate``
   * ``apache-certificate`` and ``apache-key``
 
- * change ``switch-softwaretype`` to way how ``software/erp5`` does, which will help with dropping jinja2 template for ``caddy-wrapper``, which is workaround for current situation, cf `note_62678 <https://lab.nexedi.com/nexedi/slapos/merge_requests/312#note_62678>`_
+ * drop jinja2 template for ``caddy-wrapper``
  * use `slapos!326 <https://lab.nexedi.com/nexedi/slapos/merge_requests/326>`_, and especially `note about complex restart scenarios <https://lab.nexedi.com/nexedi/slapos/merge_requests/326#note_60198>`_, instead of self-developed graceful restart scripts
  * move out ``test/utils.py`` and use it from shared python distribution
  * provide various tricks for older browsers::
@@ -61,7 +61,7 @@ Generally things to be done with ``caddy-frontend``:
     <FilesMatch "\.(cgi|shtml|phtml|php)$">
           SSLOptions +StdEnvVars
     </FilesMatch>
- * reduce the time of configuration validation (in ``instance-apache-frontend.cfg`` sections ``[configtest]``, ``[caddy-configuration]``, ``[nginx-configuration]``), as it is not scalable on frontend with 2000+ slaves (takes few minutes instead of few, < 5, seconds), issue posted `upstream <https://github.com/mholt/caddy/issues/2220>`_
+ * reduce the time of configuration validation (in ``instance-apache-frontend.cfg.in`` sections ``[configtest]``, ``[caddy-configuration]``, ``[nginx-configuration]``), as it is not scalable on frontend with 2000+ slaves (takes few minutes instead of few, < 5, seconds), issue posted `upstream <https://github.com/mholt/caddy/issues/2220>`_
  * drop ``6tunnel`` and use ``bind`` in Caddy configuration, as soon as multiple binds will be possible, tracked in upstream `bind: support multiple values <https://github.com/mholt/caddy/pull/2128>`_ and `ipv6: does not bind on ipv4 and ipv6 for sites that resolve to both <https://github.com/mholt/caddy/issues/864>`_
  * use caddy-frontend in `standalone style playbooks <https://lab.nexedi.com/nexedi/slapos.package/tree/master/playbook/roles/standalone-shared>`_
  * ensure `QUIC <https://en.wikipedia.org/wiki/QUIC>`_ is used by caddy

software/caddy-frontend/buildout.hash.cfg

@@ -13,20 +13,24 @@
 # section inheritance (< = ...) are NOT supported (but you should really
 # not need these here).
 [template]
-filename = instance.cfg
-md5sum = b73505ae80d6325a244f5094f8edc0ae
+filename = instance.cfg.in
+md5sum = 345b3a9b12241448913ec3ec8f929d51
+
+[template-common]
+filename = instance-common.cfg.in
+md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
-filename = instance-apache-frontend.cfg
-md5sum = b170d0987563b481eb71cf705c3658ab
+filename = instance-apache-frontend.cfg.in
+md5sum = da1f58f77e81e11264e5e2131794dc19
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 27e98547061bd81e5f84cb7dd21b683b
+md5sum = 8d34141a9cd1e51462aba845c7bea85b
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = fb6c93f42f232e381174a5951c3fc222
+md5sum = bdce0cf67a8f533891eabdec0b7eee87
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -34,7 +38,7 @@ md5sum = 54ae95597a126ae552c3a913ddf29e5e
 
 [template-replicate-publish-slave-information]
 filename = templates/replicate-publish-slave-information.cfg.in
-md5sum = 8d318af17da5631d4242c0d6d1531066
+md5sum = 6a308c29b54d53cfd82ae23ba77a35dd
 
 [template-caddy-frontend-configuration]
 filename = templates/Caddyfile.in
@@ -42,7 +46,7 @@ md5sum = 6689d96fc18d9aad78d77fe87770d4da
 
 [template-custom-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = fb6c93f42f232e381174a5951c3fc222
+md5sum = bdce0cf67a8f533891eabdec0b7eee87
 
 [caddy-backend-url-validator]
 filename = templates/caddy-backend-url-validator.in

software/caddy-frontend/common.cfg

@@ -34,16 +34,72 @@ eggs +=
   websockify
   erp5.util
 
+[template-common]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-common.cfg.in
+rendered = ${buildout:directory}/instance-common.cfg
+mode = 0644
+context =
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+
+[template-frontend-parameter-section]
+common_profile = ${template-common:rendered}
+
+bin_directory = ${buildout:bin-directory}
+
+sixtunnel = ${6tunnel:location}
+caddy = ${caddy:output}
+caddy_location = ${caddy:location}
+curl = ${curl:location}
+dash = ${dash:location}
+dcron = ${dcron:location}
+gzip = ${gzip:location}
+logrotate = ${logrotate:location}
+openssl = ${openssl:location}
+trafficserver = ${trafficserver:location}
+
+monitor_template = ${monitor-template:output}
+template_cached_slave_virtualhost = ${template-cached-slave-virtualhost:target}
+template_caddy_frontend_configuration = ${template-caddy-frontend-configuration:target}
+template_caddy_graceful_script = ${template-caddy-graceful-script:target}
+template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
+template_caddy_wrapper = ${template-caddy-wrapper:output}
+template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
+template_empty = ${template-empty:target}
+template_log_access = ${template-log-access:target}
+template_nging_configuration = ${template-nginx-configuration:output}
+template_nginx_eventsource_slave_virtualhost = ${template-nginx-eventsource-slave-virtualhost:target}
+template_nginx_notebook_slave_virtualhost = ${template-nginx-notebook-slave-virtualhost:target}
+template_not_found_html = ${template-not-found-html:target}
+template_slave_configuration = ${template-slave-configuration:target}
+template_slave_list = ${template-slave-list:target}
+template_trafficserver_records_config = ${template-trafficserver-records-config:location}
+template_trafficserver_records_config_filename = ${template-trafficserver-records-config:filename}
+template_trafficserver_records_config_location = ${template-trafficserver-records-config:location}
+template_trafficserver_storage_config_filename = ${template-trafficserver-storage-config:filename}
+template_trafficserver_storage_config_location = ${template-trafficserver-storage-config:location}
+template_wrapper = ${template-wrapper:output}
+
 [template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-output = ${buildout:directory}/template.cfg
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance.cfg.in
+rendered = ${buildout:directory}/template.cfg
 mode = 0644
+context =
+  key common_profile template-common:rendered
+
+  key monitor2_template monitor2-template:rendered
+  key template_caddy_frontend template-caddy-frontend:target
+  key template_caddy_replicate template-caddy-replicate:target
+  key template_replicate_publish_slave_information template-replicate-publish-slave-information:target
+  key caddy_backend_url_validator caddy-backend-url-validator:output
+
+  section template_frontend_parameter_dict template-frontend-parameter-section
 
 [template-caddy-frontend]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-apache-frontend.cfg
-output = ${buildout:directory}/template-caddy-frontend.cfg
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
 mode = 0644
 
 [caddy-backend-url-validator]
@@ -116,22 +172,14 @@ output = ${buildout:directory}/template-wrapper.cfg
 mode = 0644
 
 [template-trafficserver-records-config]
-recipe = hexagonit.recipe.download
-ignore-existing = true
+<=download-template
 url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = records.config.jinja2
-download-only = true
-mode = 0644
 
 [template-trafficserver-storage-config]
-recipe = hexagonit.recipe.download
-ignore-existing = true
+<=download-template
 url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = storage.config.jinja2
-download-only = true
-mode = 0644
 
 # NGINX Configuration
 [template-nginx-configuration]

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -6,10 +6,7 @@ rendered = ${buildout:directory}/${:filename}
 extra-context =
 context =
     import json_module json
-    key eggs_directory buildout:eggs-directory
-    key develop_eggs_directory buildout:develop-eggs-directory
-    key slap_software_type slap-parameter:slap_software_type
-    key slave_instance_list slap-parameter:slave_instance_list
+    raw common_profile {{ common_profile }}
     ${:extra-context}
 
 {% set part_list = [] %}
@@ -173,7 +170,9 @@ monitor-url-list +=
 {% endfor %}
 
 [buildout]
-extends = {{ template_monitor }}
+extends =
+  {{ common_profile }}
+  {{ template_monitor }}
 parts =
   monitor-base
   publish-slave-information
@@ -182,23 +181,4 @@ parts =
 {{ '  %s' % part }}
 {% endfor %}
 #      publish-information
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
-
-
-[slap_connection]
-# Kept for backward compatibility
-computer_id = ${slap-connection:computer-id}
-partition_id = ${slap-connection:partition-id}
-server_url = ${slap-connection:server-url}
-software_release_url = ${slap-connection:software-release-url}
-key_file = ${slap-connection:key-file}
-cert_file = ${slap-connection:cert-file}
-
-[slap-parameter]
-slave_instance_list =
--frontend-quantity = 1
--frontend-type = single-default
 {% endif %}

software/caddy-frontend/instance-common.cfg.in

+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slap_connection]
+# Kept for backward compatibility
+computer_id = ${slap-connection:computer-id}
+partition_id = ${slap-connection:partition-id}
+server_url = ${slap-connection:server-url}
+software_release_url = ${slap-connection:software-release-url}
+key_file = ${slap-connection:key-file}
+cert_file = ${slap-connection:cert-file}
\ No newline at end of file

software/caddy-frontend/instance.cfg

-[buildout]
-parts =
-  dynamic-template-caddy-replicate
-  switch-softwaretype
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
-[slap-parameters]
-recipe = slapos.cookbook:slapconfiguration
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
-
-[jinja2-template-base]
-recipe = slapos.recipe.template:jinja2
-rendered = $${buildout:directory}/$${:filename}
-extra-context =
-context =
-    import json_module json
-    key eggs_directory buildout:eggs-directory
-    key develop_eggs_directory buildout:develop-eggs-directory
-    key slap_software_type slap-parameters:slap-software-type
-    key slapparameter_dict slap-parameters:configuration
-    key slave_instance_list slap-parameters:slave-instance-list
-    $${:extra-context}
-
-[switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = $${dynamic-template-caddy-replicate:rendered}
-RootSoftwareInstance = $${dynamic-template-caddy-replicate:rendered}
-custom-personal = $${dynamic-template-caddy-replicate:rendered}
-single-default = ${template-caddy-frontend:output}
-single-custom-personal = ${template-caddy-frontend:output}
-replicate = $${dynamic-template-caddy-replicate:rendered}
-
-[dynamic-template-caddy-replicate]
-< = jinja2-template-base
-template = ${template-caddy-replicate:target}
-filename = instance-caddy-replicate.cfg
-extensions = jinja2.ext.do
-extra-context =
-    import subprocess_module subprocess
-    raw caddy_backend_url_validator ${caddy-backend-url-validator:output}
-    raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
-# Must match the key id in [switch-softwaretype] which uses this section.
-    raw software_type RootSoftwareInstance-default-custom-personal-replicate
-    raw template_monitor ${monitor2-template:rendered}
-

software/caddy-frontend/instance.cfg.in

+[buildout]
+extends = {{ common_profile }}
+
+parts =
+  dynamic-template-caddy-replicate
+  switch-softwaretype
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:filename}
+extra-context =
+context =
+    import json_module json
+    key slap_software_type instance-parameter:slap-software-type
+    key slapparameter_dict instance-parameter:configuration
+    key slave_instance_list instance-parameter:slave-instance-list
+    section instance_parameter instance-parameter
+    ${:extra-context}
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${dynamic-template-caddy-replicate:rendered}
+RootSoftwareInstance = ${dynamic-template-caddy-replicate:rendered}
+custom-personal = ${dynamic-template-caddy-replicate:rendered}
+single-default = ${dynamic-template-caddy-frontend:rendered}
+single-custom-personal = ${dynamic-template-caddy-frontend:rendered}
+replicate = ${dynamic-template-caddy-replicate:rendered}
+
+[dynamic-template-caddy-frontend-parameters]
+{% for key,value in template_frontend_parameter_dict.iteritems() %}
+{{ key }} = {{ dumps(value) }}
+{% endfor -%}
+
+[dynamic-template-caddy-frontend]
+< = jinja2-template-base
+template = {{ template_caddy_frontend }}
+filename = instance-caddy-frontend.cfg
+extensions = jinja2.ext.do
+extra-context =
+  section parameter_dict dynamic-template-caddy-frontend-parameters
+
+[dynamic-template-caddy-replicate]
+< = jinja2-template-base
+template = {{ template_caddy_replicate }}
+filename = instance-caddy-replicate.cfg
+extensions = jinja2.ext.do
+extra-context =
+    import subprocess_module subprocess
+    raw caddy_backend_url_validator {{ caddy_backend_url_validator }}
+    raw template_publish_slave_information {{ template_replicate_publish_slave_information }}
+# Must match the key id in [switch-softwaretype] which uses this section.
+    raw software_type RootSoftwareInstance-default-custom-personal-replicate
+    raw template_monitor {{ monitor2_template }}
+    raw common_profile {{ common_profile }}
+
+[instance-parameter]
+# Fetches parameters defined in SlapOS Master for this instance.
+# Always the same.
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+# Define default parameter(s) that will be used later, in case user didn't
+# specify it
+# All parameters are available through the configuration.XX syntax.
+# All possible parameters should have a default.
+configuration.domain = example.org
+configuration.public-ipv4 =
+configuration.port = 4443
+configuration.plain_http_port = 8080
+configuration.plain_nginx_port = 8081
+configuration.nginx_port = 9443
+configuration.server-admin = admin@example.com
+# BBB: apache_custom_https and apache_custom_http
+configuration.apache_custom_https = ""
+configuration.apache_custom_http = ""
+configuration.caddy_custom_https = ""
+configuration.caddy_custom_http = ""
+configuration.apache-key =
+configuration.apache-certificate =
+configuration.apache-ca-certificate =
+configuration.open-port = 80 443
+configuration.extra_slave_instance_list =
+configuration.disk-cache-size = 8G
+configuration.ram-cache-size = 1G
+configuration.trafficserver-autoconf-port = 8083
+configuration.trafficserver-mgmt-port = 8084
+configuration.re6st-verification-url = http://[2001:67c:1254:4::1]/index.html
+configuration.enable-http2-by-default = true
+configuration.mpm-graceful-shutdown-timeout = 5
+configuration.monitor-httpd-port = 8072
+configuration.frontend-name =
\ No newline at end of file

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -20,8 +20,7 @@ recipe = slapos.recipe.template:jinja2
 extensions = jinja2.ext.do
 extra-context =
 context =
-    key eggs_directory buildout:eggs-directory
-    key develop_eggs_directory buildout:develop-eggs-directory
+    raw common_profile {{ common_profile }}
     ${:extra-context}
 
 {% do logrotate_dict.pop('recipe') %}
@@ -395,6 +394,7 @@ slave-instance-information-list = {{ json_module.dumps(slave_instance_informatio
 monitor-base-url = {{ monitor_base_url }}
 
 [buildout]
+extends = {{ common_profile }}
 parts +=
     slave-log-directories
 {% for part in part_list %}
@@ -409,9 +409,6 @@ parts +=
     tunnel-6to4-base-nginx_http_port
     tunnel-6to4-base-nginx_https_port
 
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
 cache-access = {{ cache_access }}
 
 {% endif %}

software/caddy-frontend/templates/replicate-publish-slave-information.cfg.in

@@ -41,11 +41,8 @@ log-access-url = {{ json_module.dumps(slave_information.pop('log-access-urls', 1
 {% endfor %}
 
 [buildout]
+extends = {{ common_profile }}
 parts =
 {% for part in part_list %}
 {{ '  %s' % part }}
-{% endfor %}
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
\ No newline at end of file
+{% endfor %}
\ No newline at end of file