XXX caddy-frontend: Switch from Caddy to Haproxy

XXX: checkpoint reached, the frontend-haproxy is able to start, serves nothing

software/caddy-frontend/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [profile-caddy-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = a6a626fd1579fd1d4b80ea67433ca16a
+md5sum = 2618f19ebedb1e17ec7f430a382dacab
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,15 +30,15 @@ md5sum = 1248911409cbeea980a838b04ee451d2
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 8ce1d5bf09662d941f940be7e6493918
+md5sum = a176d12fd0978b0d90b5c39196cef2d5
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
 md5sum = df304a8aee87b6f2425241016a48f7a5
 
-[profile-caddy-frontend-configuration]
-_update_hash_filename_ = templates/Caddyfile.in
-md5sum = fdf46b1dee6ea6b91b9aa9e322a0530d
+[template-frontend-haproxy-configuration]
+_update_hash_filename_ = templates/frontend-haproxy.cfg.in
+md5sum = 66a197ba35871f6ea2c0607a3ca873e8
 
 [template-not-found-html]
 _update_hash_filename_ = templates/notfound.html
@@ -108,6 +108,10 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
 filename = instance-kedifa.cfg.in
 md5sum = 3daebc4b37088fa01183a853920d4143
 
+[template-frontend-haproxy-rsyslogd-conf]
+_update_hash_filename_ = templates/frontend-haproxy-rsyslogd.conf.in
+md5sum = b851c81ae40b230ed8073e9c47468299
+
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
 md5sum = 3ec9e088817f6a0e3b3b71919590e6b3

software/caddy-frontend/software.cfg

@@ -3,7 +3,6 @@ extends =
   buildout.hash.cfg
   ../../stack/slapos.cfg
   ../../component/dash/buildout.cfg
-  ../../component/caddy/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../component/logrotate/buildout.cfg
   ../../component/trafficserver/buildout.cfg
@@ -97,7 +96,8 @@ profile_slave_list = ${profile-slave-list:target}
 # templates
 template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
 template_backend_haproxy_rsyslogd_conf = ${template-backend-haproxy-rsyslogd-conf:target}
-template_caddy_frontend_configuration = ${profile-caddy-frontend-configuration:target}
+template_frontend_haproxy_configuration = ${template-frontend-haproxy-configuration:target}
+template_frontend_haproxy_rsyslogd_conf = ${template-frontend-haproxy-rsyslogd-conf:target}
 template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
 template_configuration_state_script = ${template-configuration-state-script:target}
 template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
@@ -119,7 +119,6 @@ bin_directory = ${buildout:bin-directory}
 sixtunnel = ${6tunnel:location}
 nginx = ${nginx-output:nginx}
 nginx_mime = ${nginx-output:mime}
-caddy = ${caddy:output}
 haproxy_executable = ${haproxy:location}/sbin/haproxy
 rsyslogd_executable = ${rsyslogd:location}/sbin/rsyslogd
 curl = ${curl:location}
@@ -170,7 +169,7 @@ mode = 640
 [profile-replicate-publish-slave-information]
 <=download-template
 
-[profile-caddy-frontend-configuration]
+[template-frontend-haproxy-configuration]
 <=download-template
 
 [template-not-found-html]
@@ -221,6 +220,9 @@ mode = 0644
 [template-backend-haproxy-rsyslogd-conf]
 <=download-template
 
+[template-frontend-haproxy-rsyslogd-conf]
+<=download-template
+
 [versions]
 # Modern KeDiFa requires zc.lockfile
 zc.lockfile = 1.4

software/caddy-frontend/templates/Caddyfile.in

-# Main caddy configuration file
-
-import {{ slave_configuration_directory }}/*.conf
-
-:{{ https_port }} {
-  tls {{ master_certificate }} {{ master_certificate }}
-  bind {{ local_ipv4 }}
-  status 404 /
-  log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
-    rotate_size 10000000
-  }
-  errors {{ error_log }} {
-    rotate_size 10000000
-    * {{ not_found_file }}
-  }
-}
-
-:{{ http_port }} {
-  bind {{ local_ipv4 }}
-  status 404 /
-  log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
-    rotate_size 10000000
-  }
-  errors {{ error_log }} {
-    rotate_size 10000000
-    * {{ not_found_file }}
-  }
-}
-
-# Access to server-status Caddy-style
-https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv4 }}:{{ https_port }}/server-status {
-  tls {{ frontend_configuration['ip-access-certificate'] }} {{ frontend_configuration['ip-access-certificate'] }}
-  bind {{ local_ipv4 }}
-  basicauth "{{ username }}" {{ password | trim }} {
-    "Server Status"
-    /
-  }
-  expvar
-  pprof
-  log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
-    rotate_size 10000000
-  }
-  errors {{ error_log }} {
-    rotate_size 10000000
-    * {{ not_found_file }}
-  }
-}

software/caddy-frontend/templates/frontend-haproxy-rsyslogd.conf.in

+module(
+  load="imuxsock"
+  SysSock.Name="{{ configuration['log-socket'] }}")
+
+# Just simply output the raw line without any additional information, as
+# haproxy emits enough information by itself
+# Also cut out first empty space in msg, which is related to rsyslogd
+# internal and end up cutting on 8k, as it's default of $MaxMessageSize
+template(name="rawoutput" type="string" string="%msg:2:8192%\n")
+$ActionFileDefaultTemplate rawoutput
+
+$FileCreateMode 0600
+$DirCreateMode 0700
+$Umask 0022
+
+$WorkDirectory {{ configuration['spool-directory'] }}
+
+# Setup logging per slave, by extracting the slave name from the log stream
+{%- set regex = ".*-frontend (.*)-http.{0,1}/" %}
+template(name="extract_slave_name" type="string" string="%msg:R,ERE,1,FIELD:{{ regex }}--end%")
+set $!slave_name = exec_template("extract_slave_name");
+template(name="slave_output" type="string" string="{{ configuration['slave-log-directory'] }}/%$!slave_name%_frontend_log")
+if (re_match($msg, "{{ regex }}")) then {
+ action(type="omfile" dynaFile="slave_output")
+ stop
+}
+
+{#- emit all not catched messages to full log file #}
+*.* {{ configuration['log-file'] }}

software/caddy-frontend/templates/frontend-haproxy.cfg.in

+global
+  pidfile {{ configuration['pid-file'] }}
+  # master-worker is compatible with foreground with process management
+  master-worker
+
+log {{ configuration['log-socket'] }} local0
+defaults
+  log global
+  mode http
+  option httplog
+{#-  timeout queue XXX TODO #}
+{#-  timeout server XXX TODO #}
+{#-  timeout client XXX TODO #}
+{#-  timeout connect XXX TODO #}
+{#-  retries XXX TODO #}
+
+{%- set SCHEME_PREFIX_MAPPING = { 'http': 'http_backend', 'https': 'https_backend'} %}
+{%- macro frontend_entry(slave_instance, scheme, wildcard) %}
+{#-   wildcard switch allows to put dangerous entries in the end, as haproxy parses with first match #}
+{%-   if slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['hostname'] and slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['port'] %}
+{%-     set host_list = (slave_instance.get('server-alias') or  '').split() %}
+{%-     if slave_instance.get('custom_domain') not in host_list %}
+{%-       do host_list.append(slave_instance.get('custom_domain')) %}
+{%-     endif %}
+{%-     set matched = {'count': 0} %}
+{%-     for host in host_list %}
+{#-       Match up to the end or optional port (starting with ':') #}
+{#-       Please note that this matching is quite sensitive to changes and hard to test, so avoid needless changes #}
+{%-       if wildcard and host.startswith('*.') %}
+{%-         do matched.__setitem__('count', matched['count'] + 1) %}
+# match wildcard {{ host }}
+  acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i {{ host[2:] }}($|:.*)
+{%-       elif not wildcard and not host.startswith('*.') %}
+{%-         do matched.__setitem__('count', matched['count'] + 1) %}
+  acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i ^{{ host }}($|:.*)
+{%-       endif %}
+{%-     endfor %}
+{%-     if matched['count'] > 0 %}
+  use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}
+{%-     endif %}
+{%-   endif %}
+{%- endmacro %}
+
+frontend http-frontend
+  bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
+# Here use Host header
+{%- for slave_instance in frontend_slave_list -%}
+{{ frontend_entry(slave_instance, 'http', False) }}
+{%- endfor %}
+{%- for slave_instance in frontend_slave_list -%}
+{{ frontend_entry(slave_instance, 'http', True) }}
+{%- endfor %}
+
+frontend https-frontend
+  bind {{ configuration['local-ipv4'] }}:{{ configuration['https-port'] }}
+# Here use ssl_fc_sni and fallback to Host header
+{%- for slave_instance in frontend_slave_list -%}
+{{ frontend_entry(slave_instance, 'https', False) }}
+{%- endfor %}
+{%- for slave_instance in frontend_slave_list -%}
+{{ frontend_entry(slave_instance, 'https', True) }}
+{%- endfor %}
+
+{%- for slave_instance in frontend_slave_list %}
+{%-   for (scheme, prefix) in SCHEME_PREFIX_MAPPING.items() %}
+backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
+{%-   endfor %}
+{%- endfor %}