caddy-frontend: Improve generated files

Features:

 * amend configuration with comments
 * drop obsolete comments from Apache copy
 * remove not needed whitespaces
 * use indentation for conditionals in Jinja2

software/caddy-frontend/TODO.rst

@@ -15,7 +15,6 @@ Generally things to be done with ``caddy-frontend``:
  * ``ssl_proxy_ca_crt`` for ``ssl_proxy_verify``, this is related to bug https://github.com/mholt/caddy/issues/1550, proposed solution `just adding your CA to the system's trust store`
  * ``check-error-on-caddy-log`` like ``check-error-on-apache-log``
  * cover test suite like resilient tests for KVM and prove it works the same way as Caddy
- * make beautiful (eg. with whitespaces and nice comments) generated files (mostly Jinja2)
  * have ``caddy-frontend`` specific parameters, with backward compatibility to ``apache-frontend`` ones (like ``apache_custom_http`` --> ``caddy_custom_http``)
  * change ``switch-softwaretype`` to way how ``software/erp5`` does, which will help with dropping jinja2 template for ``caddy-wrapper``, which is workaround for current situation https://lab.nexedi.com/nexedi/slapos/merge_requests/312#note_62678
  * use `slapos!326 <https://lab.nexedi.com/nexedi/slapos/merge_requests/326>`_ instead of self-developed graceful restart scripts

software/caddy-frontend/buildout.hash.cfg

@@ -38,7 +38,7 @@ md5sum = 8d318af17da5631d4242c0d6d1531066
 
 [template-caddy-frontend-configuration]
 filename = templates/Caddyfile.in
-md5sum = 924d3bb528f590916552534934c604a2
+md5sum = 9404959e500a868aab1a217503117047
 
 [template-custom-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
@@ -50,11 +50,11 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = b524304177e7854232aa43bed98ddbfd
+md5sum = fa7dc8481f0c3066045c1dd5a8a3191a
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in
-md5sum = 5aab4c15189a39837f56d4f442b233c6
+md5sum = bfcc2bcfe9151b9d3f25c4616e2c4f4f
 
 [template-log-access]
 filename = templates/template-log-access.conf.in
@@ -82,7 +82,7 @@ md5sum = 117238225b3fc3c5b5be381815f44c67
 
 [template-nginx-configuration]
 filename = templates/nginx.cfg.in
-md5sum = b1d6bac767db77ad1662edd06aabdf49
+md5sum = fadb2fcaf0f2b4fe735617fac222f7ed
 
 [template-nginx-eventsource-slave-virtualhost]
 filename = templates/nginx-eventsource-slave.conf.in
@@ -90,7 +90,7 @@ md5sum = 69d65e461cd7cd5ef5b1ccd0098b50c8
 
 [template-nginx-notebook-slave-virtualhost]
 filename = templates/nginx-notebook-slave.conf.in
-md5sum = 753e87647d1ed4655432393bba062d3f
+md5sum = b97ec5b84d5e0d3a76871c15b5bcce2e
 
 [template-apache-lazy-script-call]
 filename = templates/apache-lazy-script-call.sh.in

software/caddy-frontend/templates/Caddyfile.in

-# Automatically generated
+# Main caddy configuration file
 
 import {{frontend_configuration.get('log-access-configuration')}}
 import {{ slave_configuration_directory }}/*.conf
 import {{ slave_with_cache_configuration_directory }}/*.conf
 
+# Catch-all and 404 for not configured instances
 :{{ https_port }} {
   tls {{ login_certificate }} {{ login_key }}
   bind {{ local_ipv4 }}
@@ -14,6 +15,16 @@ import {{ slave_with_cache_configuration_directory }}/*.conf
   }
 }
 
+:{{ http_port }} {
+  bind {{ local_ipv4 }}
+  status 404 /
+  log / {{ access_log }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
+  errors {{ error_log }} {
+    * {{ not_found_file }}
+  }
+}
+
+# Access to server-status Caddy-style
 https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv4 }}:{{ https_port }}/server-status {
   tls {{ login_certificate }} {{ login_key }}
   bind {{ local_ipv4 }}
@@ -28,12 +39,3 @@ https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv
     * {{ not_found_file }}
   }
 }
-
-:{{ http_port }} {
-  bind {{ local_ipv4 }}
-  status 404 /
-  log / {{ access_log }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
-  errors {{ error_log }} {
-    * {{ not_found_file }}
-  }
-}

software/caddy-frontend/templates/cached-virtualhost.conf.in

-{% set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
-{% set server_alias_list =  slave_parameter.get('server-alias', '').split() %}
-{% set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES %}
-{% set host_list = [slave_parameter.get('custom_domain')] + server_alias_list %}
-{% set http_backend_host_list = [] %}
-{% set https_backend_host_list = [] %}
-{% for host in host_list %}
-{%   do http_backend_host_list.append('http://%s:%s' % (host, cached_port)) %}
-{%   do https_backend_host_list.append('http://%s:%s' % (host, ssl_cached_port)) %}
-{% endfor %}
+{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
+{%- set server_alias_list =  slave_parameter.get('server-alias', '').split() %}
+{%- set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES %}
+{%- set host_list = [slave_parameter.get('custom_domain')] + server_alias_list %}
+{%- set http_backend_host_list = [] %}
+{%- set https_backend_host_list = [] %}
+{%- for host in host_list %}
+{%-   do http_backend_host_list.append('http://%s:%s' % (host, cached_port)) %}
+{%-   do https_backend_host_list.append('http://%s:%s' % (host, ssl_cached_port)) %}
+{%- endfor %}
 
-# Only accept generic (i.e not Zope) backends on http
+# SSL-disabled backends
 {{ http_backend_host_list|join(', ') }} {
   bind {{ local_ipv4 }}
-{% if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
+{%- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
     status 501 /
-{% endif %}
+{%- endif %}
 # Rewrite part
   proxy / {{ slave_parameter.get('backend_url', '') }} {
     # As backend is trusting REMOTE_USER header unset it always
@@ -22,30 +22,31 @@
 
     transparent
     timeout 600s
-{% if ssl_proxy_verify %}
-{%   if 'ssl_proxy_ca_crt' in slave_parameter %}
-{%   endif %}
-{% else %}
+{%- if ssl_proxy_verify %}
+{%-   if 'ssl_proxy_ca_crt' in slave_parameter %}
+{%-   endif %}
+{%- else %}
     insecure_skip_verify
-{% endif %}
+{%- endif %}
   }
 }
 
+# SSL-enabled backends
 {{ https_backend_host_list|join(', ') }} {
   bind {{ local_ipv4 }}
-{% if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
+{%- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
     status 501 /
-{% endif %}
+{%- endif %}
   proxy / {{ slave_parameter.get('https_backend_url', '') }} {
     # As backend is trusting REMOTE_USER header unset it always
     header_upstream -REMOTE_USER
     transparent
     timeout 600s
-{% if ssl_proxy_verify %}
-{%   if 'ssl_proxy_ca_crt' in slave_parameter %}
-{%   endif %}
-{% else %}
+{%- if ssl_proxy_verify %}
+{%-   if 'ssl_proxy_ca_crt' in slave_parameter %}
+{%-   endif %}
+{%- else %}
     insecure_skip_verify
-{% endif %}
+{%- endif %}
   }
 }

software/caddy-frontend/templates/nginx-notebook-slave.conf.in

-{% set url = slave_parameter.get('url') %}
-{% set https_url = slave_parameter.get('https-url', url) %}
-{% if url.startswith("http://") or url.startswith("https://") %}
-{%   set upstream = url.split("/")[2] %}
-{%   set https_upstream = https_url.split("/")[2] %}
+{%- set url = slave_parameter.get('url') %}
+{%- set https_url = slave_parameter.get('https-url', url) %}
+{%- if url.startswith("http://") or url.startswith("https://") %}
+{%-   set upstream = url.split("/")[2] %}
+{%-   set https_upstream = https_url.split("/")[2] %}
 
+# SSL-enabled
 https://{{ slave_parameter.get('custom_domain') }}:{{ nginx_https_port }} {
   bind {{ local_ipv4 }}
   log / {{ slave_parameter.get('access_log') }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
   errors {{ slave_parameter.get('error_log') }}
 
   tls {{ slave_parameter.get('path_to_ssl_crt', slave_parameter.get('login_certificate')) }} {{ slave_parameter.get('path_to_ssl_key', slave_parameter.get('login_key')) }} {
-{% if slave_parameter.get('path_to_ssl_ca_crt') %}
+{%-   if slave_parameter.get('path_to_ssl_ca_crt') %}
     clients {{ slave_parameter.get('path_to_ssl_ca_crt') }}
-{% endif %}
+{%-   endif %}
     alpn http/1.1
   }
 
@@ -33,6 +34,7 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ nginx_https_port }} {
   }
 }
 
+# SSL-disabled
 http://{{ slave_parameter.get('custom_domain') }}:{{ nginx_http_port }} {
   bind {{ local_ipv4 }}
   log / {{ slave_parameter.get('access_log') }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
@@ -54,4 +56,4 @@ http://{{ slave_parameter.get('custom_domain') }}:{{ nginx_http_port }} {
     insecure_skip_verify
   }
 }
-{% endif %}
+{%- endif %}

software/caddy-frontend/templates/nginx.cfg.in

@@ -57,6 +57,7 @@
 
 import {{ slave_configuration_directory }}/*.conf
 
+# Catch-all and 404 for not configured instances
 :{{ port }} {
   tls {{ ssl_certificate }} {{ ssl_key }}
   bind {{ local_ip }}

[Jérome Perrin]

@luke I saw that this URL is http:// but appended to https_backend_host_list . I have not checked, but it looks like a typo

[Łukasz Nowak]

It is not a typo :)

I already was shocked by this and fixed it with commit 0b606475, but adding note here explaining, that it is always http access can answer such question.

Please note that this approach comes from apache-frontend and I am still unsure what was high level idea behind it.