caddy-frontend: Implement failover backend

By adding failover url the user is able to configure special backend to use
in case if the real backend is down.

Original PoC was done by Kazuhiko SHIOZAKI <kazuhiko@nexedi.com>.

software/caddy-frontend/CHANGES.caddy_frontend.rst

@@ -21,6 +21,7 @@ Here are listed the most important changes, which might affect upgrades.
  * fix: use kedifa with with for file with multiple CAs
  * feature: support query string (the characters after ? in the url) in url and https-url
  * fix: by having unique acl names fix rare bug of directing traffic to https-url instead of url or otherwise
+ * feature: failover backend
 
 1.0.164 (2020-09-24)
 --------------------

software/caddy-frontend/README.caddy_frontend.rst

@@ -236,6 +236,8 @@ This set of parameters is used to control the way how the backend checks will be
 
 Please be aware that the `health-check-timeout` is really short by default, so in case if `/` of the backend is slow to reply configure proper path with `health-check-http-path` to not mark such backend down too fast, before increasing the check timeout.
 
+Thanks to using health-check it's possible to configure failover system. By providing `health-check-failover-url` or `health-check-failover-https-url` some special backend can be used to reply in case if original backend replies with error (codes like `5xx`). As a note one can setup this failover URL like `https://failover.example.com/?p=` so that the path from the incoming request will be passed as parameter. Additionally authentication to failover URL is supported with `health-check-authenticate-to-failover-backend` and SSL Proxy verification with `health-check-failover-ssl-proxy-verify` and `health-check-failover-ssl-proxy-ca-crt`.
+
 Examples
 ========
 

software/caddy-frontend/buildout.hash.cfg

@@ -26,11 +26,11 @@ md5sum = a6a626fd1579fd1d4b80ea67433ca16a
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 1ab3fc07bb186601b54c584a3ccaf1c3
+md5sum = 1248911409cbeea980a838b04ee451d2
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 9eb14b83ee6fc8a5afa8267d9bcf4772
+md5sum = 8ce1d5bf09662d941f940be7e6493918
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -50,7 +50,7 @@ md5sum = a0ae858a3db8825c22d33d323392f588
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = 8c4e2548a12c8fd7dba74f940201745a
+md5sum = 17f9582671327d8e4321a7fd1cdcb0fe
 
 [template-empty]
 _update_hash_filename_ = templates/empty.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -200,7 +200,7 @@ context =
 {%     endfor %}
 {%     do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %}
 {%   endif %}
-{%   for url_key in ['url', 'https-url'] %}
+{%   for url_key in ['url', 'https-url', 'health-check-failover-url', 'health-check-failover-https-url'] %}
 {%     if url_key in slave %}
 {%       set url = (slave[url_key] or '').strip() %}
 {%       if not validators.url(url) %}
@@ -210,14 +210,16 @@ context =
 {%       endif %}
 {%     endif %}
 {%   endfor %}
-{%   if 'ssl_proxy_ca_crt' in slave %}
-{%     set ssl_proxy_ca_crt = slave.get('ssl_proxy_ca_crt', '') %}
+{%   for k in ['ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt'] %}
+{%     if k in slave %}
+{%       set crt = slave.get(k, '') %}
 {%       set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %}
-{%     do check_popen.communicate(ssl_proxy_ca_crt) %}
+{%       do check_popen.communicate(crt) %}
 {%       if check_popen.returncode != 0 %}
-{%       do slave_error_list.append('ssl_proxy_ca_crt is invalid') %}
+{%         do slave_error_list.append('%s is invalid' % (k,)) %}
 {%       endif %}
 {%     endif %}
+{%   endfor %}
 {# BBB: SlapOS Master non-zero knowledge BEGIN #}
 {%   for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %}
 {%     if key in slave %}

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -286,6 +286,44 @@
       "default": "1",
       "type": "integer"
     },
+    "health-check-failover-url": {
+      "description": "URL of the failover backend",
+      "pattern": "^(http|https|ftp)://",
+      "title": "Failover backend URL",
+      "type": "string"
+    },
+    "health-check-failover-https-url": {
+      "description": "HTTPS URL of the failover backend if it is different from health-check-failover-url parameter. Note: It requires https-url to be configured, as otherwise the differentiation does not make sense..",
+      "pattern": "^(http|https|ftp)://",
+      "title": "Failover HTTPS Backend URL",
+      "type": "string"
+    },
+    "health-check-authenticate-to-failover-backend": {
+      "description": "If set to true the frontend certificate will be used as authentication certificate to the failover backend. Note: failover backend might have to know the frontend CA, available with 'backend-client-caucase-url'.",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "title": "Authenticate to failover backend",
+      "type": "string"
+    },
+    "health-check-failover-ssl-proxy-verify": {
+      "default": "false",
+      "description": "If set to true, failover backend SSL Certificates will be checked and frontend will refuse to proxy if certificate is invalid",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "title": "Verify failover backend certificates",
+      "type": "string"
+    },
+    "health-check-failover-ssl-proxy-ca-crt": {
+      "default": "",
+      "description": "Content of the SSL Certificate Authority file of the failover backend (to be used with health-check-failover-ssl-proxy-verify)",
+      "textarea": true,
+      "title": "SSL failover backend Authority's Certificate",
+      "type": "string"
+    },
     "strict-transport-security": {
       "title": "Strict Transport Security",
       "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. Setting the value enables HSTS and sets the value of max-age. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -58,6 +58,18 @@ context =
 {%-     do slave_instance.__setitem__(prefix, info_dict) %}
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
+{%-   for key, prefix in [('health-check-failover-url', 'http_backend'), ('health-check-failover-https-url', 'https_backend')] %}
+{%-     set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %}
+{%-     set info_dict = slave_instance[prefix] %}
+{%-     do info_dict.__setitem__('health-check-failover-scheme', parsed.scheme) %}
+{%-     do info_dict.__setitem__('health-check-failover-hostname', parsed.hostname) %}
+{%-     do info_dict.__setitem__('health-check-failover-port', parsed.port or DEFAULT_PORT[parsed.scheme]) %}
+{%-     do info_dict.__setitem__('health-check-failover-path', parsed.path) %}
+{%-     do info_dict.__setitem__('health-check-failover-query', parsed.query) %}
+{%-     do info_dict.__setitem__('health-check-failover-fragment', parsed.fragment) %}
+{%-     do slave_instance.__setitem__(prefix, info_dict) %}
+{%-   endfor %}
+{%-   do slave_instance.__setitem__('health-check-failover-ssl-proxy-verify', ('' ~ slave_instance.get('health-check-failover-ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
 {%-   do slave_instance.__setitem__('enable-http2', ('' ~ slave_instance.get('enable-http2', configuration['enable-http2-by-default'])).lower() in TRUE_VALUES) %}
 {%-   for key in ['https-only', 'websocket-transparent'] %}
 {%-     do slave_instance.__setitem__(key, ('' ~ slave_instance.get(key, 'true')).lower() in TRUE_VALUES) %}
@@ -135,6 +147,7 @@ context =
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('strict-transport-security', int(slave_instance['strict-transport-security'])) %}
 {%-   do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %}
+{%-   do slave_instance.__setitem__('health-check-authenticate-to-failover-backend', ('' ~ slave_instance.get('health-check-authenticate-to-failover-backend', '')).lower() in TRUE_VALUES) %}
 {#-   Setup active check #}
 {%-   do slave_instance.__setitem__('health-check',  ('' ~ slave_instance.get('health-check', '')).lower() in TRUE_VALUES) %}
 {%-   if slave_instance['health-check']  %}
@@ -266,7 +279,7 @@ command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} {{ slave_refere
 {%-   set certificate = '%s/%s' % (autocert, cert_name) %}
 {%-   do slave_parameter_dict.__setitem__('certificate', certificate )%}
 {#-   Set ssl certificates for each slave #}
-{%-     for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%}
+{%-     for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt')%}
 {%-       set cert_file_key = 'path_to_' + cert_name %}
 {%-       if cert_name in slave_instance %}
 {%-         set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) %}

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -32,9 +32,15 @@ defaults
 {%-       endif %}
 {%-     endfor %}
 {%-     if matched['count'] > 0 %}
+{%-       if slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['health-check-failover-hostname'] %}
+  acl is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }} nbsrv({{ slave_instance['slave_reference'] }}-{{ scheme }}) eq 0
+  use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}_{{ scheme }} ! is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }}
+  use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover if is_{{ slave_instance['slave_reference'] }}_{{ scheme }} is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }}
+{%-       else %}
   use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}_{{ scheme }}
 {%-       endif %}
 {%-     endif %}
+{%-   endif %}
 {%- endmacro %}
 
 # statistic
@@ -123,5 +129,44 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
 {%-         endif %}
 {%-       endif %}
 {%-     endif %}
+{%-     if info_dict['health-check-failover-hostname'] and info_dict['health-check-failover-port'] %}
+{%-       set ssl_list = [] %}
+{%-       if info_dict['health-check-failover-scheme'] == 'https' %}
+{%-         if slave_instance['health-check-authenticate-to-failover-backend'] %}
+{%-           do ssl_list.append('crt %s' % (configuration['certificate'],)) %}
+{%-         endif %}
+{%-         do ssl_list.append('ssl verify') %}
+{%-         if slave_instance['health-check-failover-ssl-proxy-verify'] %}
+{%-           if slave_instance['path_to_health-check-failover-ssl-proxy-ca-crt']  %}
+{%-             do ssl_list.append('required ca-file %s' % (slave_instance['path_to_health-check-failover-ssl-proxy-ca-crt'],)) %}
+{%-           else %}
+{#-           Backend SSL shall be verified, but not CA provided, disallow connection #}
+{#-           Simply dropping hostname from the dict will result with ignoring it... #}
+{%-           do info_dict.__setitem__('health-check-failover-hostname', '') %}
+{%-           endif %}
+{%-         else %}
+{%-           do ssl_list.append('none') %}
+{%-         endif %}
+{%-       endif %}
+
+backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover
+{%-       set hostname = info_dict['health-check-failover-hostname'] %}
+{%-       set port = info_dict['health-check-failover-port'] %}
+{%-       set path_list = [info_dict['health-check-failover-path'].rstrip('/')] %}
+{%-       set query = info_dict['health-check-failover-query'] %}
+{%-       if query %}
+{%-         do path_list.append(query) %}
+{%-       endif %}
+{%-       set path = '?'.join(path_list) %}
+{%-       if hostname and port %}
+  timeout server {{ slave_instance['request-timeout'] }}s
+  timeout connect {{ slave_instance['backend-connect-timeout'] }}s
+  retries {{ slave_instance['backend-connect-retries'] }}
+  server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
+{%-         if path %}
+  http-request set-path {{ path }}%[path]
+{%-         endif %}
+{%-       endif %}
+{%-     endif %}
 {%-   endfor %}
 {%- endfor %}

software/caddy-frontend/test/test_data/test.TestSlaveHealthCheck.test_file_list_log-CADDY.txt

@@ -21,6 +21,21 @@ T-2/var/log/httpd/_health-check-default_error_log
 T-2/var/log/httpd/_health-check-disabled_access_log
 T-2/var/log/httpd/_health-check-disabled_backend_log
 T-2/var/log/httpd/_health-check-disabled_error_log
+T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_access_log
+T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_backend_log
+T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_error_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_access_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_backend_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_error_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_access_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_backend_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_error_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_access_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_backend_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_error_log
+T-2/var/log/httpd/_health-check-failover-url_access_log
+T-2/var/log/httpd/_health-check-failover-url_backend_log
+T-2/var/log/httpd/_health-check-failover-url_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log