caddy-frontend: Implement failover backend

By adding failover url the user is able to configure special backend to use
in case if the real backend is down.

Original PoC was done by Kazuhiko SHIOZAKI <kazuhiko@nexedi.com>.

software/caddy-frontend/CHANGES.caddy_frontend.rst

@@ -21,6 +21,7 @@ Here are listed the most important changes, which might affect upgrades.
  * fix: use kedifa with with for file with multiple CAs
  * feature: support query string (the characters after ? in the url) in url and https-url
  * fix: by having unique acl names fix rare bug of directing traffic to https-url instead of url or otherwise
+ * feature: failover backend
 
 1.0.164 (2020-09-24)
 --------------------

software/caddy-frontend/README.caddy_frontend.rst

@@ -236,6 +236,8 @@ This set of parameters is used to control the way how the backend checks will be
 
 Please be aware that the `health-check-timeout` is really short by default, so in case if `/` of the backend is slow to reply configure proper path with `health-check-http-path` to not mark such backend down too fast, before increasing the check timeout.
 
+Thanks to using health-check it's possible to configure failover system. By providing `health-check-failover-url` or `health-check-failover-https-url` some special backend can be used to reply in case if original backend replies with error (codes like `5xx`). As a note one can setup this failover URL like `https://failover.example.com/?p=` so that the path from the incoming request will be passed as parameter. Additionally authentication to failover URL is supported with `health-check-authenticate-to-failover-backend` and SSL Proxy verification with `health-check-failover-ssl-proxy-verify` and `health-check-failover-ssl-proxy-ca-crt`.
+
 Examples
 ========
 

software/caddy-frontend/buildout.hash.cfg

@@ -26,11 +26,11 @@ md5sum = a6a626fd1579fd1d4b80ea67433ca16a
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 1ab3fc07bb186601b54c584a3ccaf1c3
+md5sum = 1248911409cbeea980a838b04ee451d2
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 9eb14b83ee6fc8a5afa8267d9bcf4772
+md5sum = 8ce1d5bf09662d941f940be7e6493918
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -50,7 +50,7 @@ md5sum = a0ae858a3db8825c22d33d323392f588
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = 8c4e2548a12c8fd7dba74f940201745a
+md5sum = 17f9582671327d8e4321a7fd1cdcb0fe
 
 [template-empty]
 _update_hash_filename_ = templates/empty.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -200,7 +200,7 @@ context =
 {%     endfor %}
 {%     do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %}
 {%   endif %}
-{%   for url_key in ['url', 'https-url'] %}
+{%   for url_key in ['url', 'https-url', 'health-check-failover-url', 'health-check-failover-https-url'] %}
 {%     if url_key in slave %}
 {%       set url = (slave[url_key] or '').strip() %}
 {%       if not validators.url(url) %}
@@ -210,14 +210,16 @@ context =
 {%       endif %}
 {%     endif %}
 {%   endfor %}
-{%   if 'ssl_proxy_ca_crt' in slave %}
-{%     set ssl_proxy_ca_crt = slave.get('ssl_proxy_ca_crt', '') %}
-{%     set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %}
-{%     do check_popen.communicate(ssl_proxy_ca_crt) %}
-{%     if check_popen.returncode != 0 %}
-{%       do slave_error_list.append('ssl_proxy_ca_crt is invalid') %}
+{%   for k in ['ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt'] %}
+{%     if k in slave %}
+{%       set crt = slave.get(k, '') %}
+{%       set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %}
+{%       do check_popen.communicate(crt) %}
+{%       if check_popen.returncode != 0 %}
+{%         do slave_error_list.append('%s is invalid' % (k,)) %}
+{%       endif %}
 {%     endif %}
-{%   endif %}
+{%   endfor %}
 {# BBB: SlapOS Master non-zero knowledge BEGIN #}
 {%   for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %}
 {%     if key in slave %}

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -286,6 +286,44 @@
       "default": "1",
       "type": "integer"
     },
+    "health-check-failover-url": {
+      "description": "URL of the failover backend",
+      "pattern": "^(http|https|ftp)://",
+      "title": "Failover backend URL",
+      "type": "string"
+    },
+    "health-check-failover-https-url": {
+      "description": "HTTPS URL of the failover backend if it is different from health-check-failover-url parameter. Note: It requires https-url to be configured, as otherwise the differentiation does not make sense..",
+      "pattern": "^(http|https|ftp)://",
+      "title": "Failover HTTPS Backend URL",
+      "type": "string"
+    },
+    "health-check-authenticate-to-failover-backend": {
+      "description": "If set to true the frontend certificate will be used as authentication certificate to the failover backend. Note: failover backend might have to know the frontend CA, available with 'backend-client-caucase-url'.",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "title": "Authenticate to failover backend",
+      "type": "string"
+    },
+    "health-check-failover-ssl-proxy-verify": {
+      "default": "false",
+      "description": "If set to true, failover backend SSL Certificates will be checked and frontend will refuse to proxy if certificate is invalid",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "title": "Verify failover backend certificates",
+      "type": "string"
+    },
+    "health-check-failover-ssl-proxy-ca-crt": {
+      "default": "",
+      "description": "Content of the SSL Certificate Authority file of the failover backend (to be used with health-check-failover-ssl-proxy-verify)",
+      "textarea": true,
+      "title": "SSL failover backend Authority's Certificate",
+      "type": "string"
+    },
     "strict-transport-security": {
       "title": "Strict Transport Security",
       "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. Setting the value enables HSTS and sets the value of max-age. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -58,6 +58,18 @@ context =
 {%-     do slave_instance.__setitem__(prefix, info_dict) %}
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
+{%-   for key, prefix in [('health-check-failover-url', 'http_backend'), ('health-check-failover-https-url', 'https_backend')] %}
+{%-     set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %}
+{%-     set info_dict = slave_instance[prefix] %}
+{%-     do info_dict.__setitem__('health-check-failover-scheme', parsed.scheme) %}
+{%-     do info_dict.__setitem__('health-check-failover-hostname', parsed.hostname) %}
+{%-     do info_dict.__setitem__('health-check-failover-port', parsed.port or DEFAULT_PORT[parsed.scheme]) %}
+{%-     do info_dict.__setitem__('health-check-failover-path', parsed.path) %}
+{%-     do info_dict.__setitem__('health-check-failover-query', parsed.query) %}
+{%-     do info_dict.__setitem__('health-check-failover-fragment', parsed.fragment) %}
+{%-     do slave_instance.__setitem__(prefix, info_dict) %}
+{%-   endfor %}
+{%-   do slave_instance.__setitem__('health-check-failover-ssl-proxy-verify', ('' ~ slave_instance.get('health-check-failover-ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
 {%-   do slave_instance.__setitem__('enable-http2', ('' ~ slave_instance.get('enable-http2', configuration['enable-http2-by-default'])).lower() in TRUE_VALUES) %}
 {%-   for key in ['https-only', 'websocket-transparent'] %}
 {%-     do slave_instance.__setitem__(key, ('' ~ slave_instance.get(key, 'true')).lower() in TRUE_VALUES) %}
@@ -135,6 +147,7 @@ context =
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('strict-transport-security', int(slave_instance['strict-transport-security'])) %}
 {%-   do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %}
+{%-   do slave_instance.__setitem__('health-check-authenticate-to-failover-backend', ('' ~ slave_instance.get('health-check-authenticate-to-failover-backend', '')).lower() in TRUE_VALUES) %}
 {#-   Setup active check #}
 {%-   do slave_instance.__setitem__('health-check',  ('' ~ slave_instance.get('health-check', '')).lower() in TRUE_VALUES) %}
 {%-   if slave_instance['health-check']  %}
@@ -266,7 +279,7 @@ command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} {{ slave_refere
 {%-   set certificate = '%s/%s' % (autocert, cert_name) %}
 {%-   do slave_parameter_dict.__setitem__('certificate', certificate )%}
 {#-   Set ssl certificates for each slave #}
-{%-     for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%}
+{%-     for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt')%}
 {%-       set cert_file_key = 'path_to_' + cert_name %}
 {%-       if cert_name in slave_instance %}
 {%-         set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) %}

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -32,7 +32,13 @@ defaults
 {%-       endif %}
 {%-     endfor %}
 {%-     if matched['count'] > 0 %}
+{%-       if slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['health-check-failover-hostname'] %}
+  acl is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }} nbsrv({{ slave_instance['slave_reference'] }}-{{ scheme }}) eq 0
+  use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}_{{ scheme }} ! is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }}
+  use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover if is_{{ slave_instance['slave_reference'] }}_{{ scheme }} is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }}
+{%-       else %}
   use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}_{{ scheme }}
+{%-       endif %}
 {%-     endif %}
 {%-   endif %}
 {%- endmacro %}
@@ -123,5 +129,44 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
 {%-         endif %}
 {%-       endif %}
 {%-     endif %}
+{%-     if info_dict['health-check-failover-hostname'] and info_dict['health-check-failover-port'] %}
+{%-       set ssl_list = [] %}
+{%-       if info_dict['health-check-failover-scheme'] == 'https' %}
+{%-         if slave_instance['health-check-authenticate-to-failover-backend'] %}
+{%-           do ssl_list.append('crt %s' % (configuration['certificate'],)) %}
+{%-         endif %}
+{%-         do ssl_list.append('ssl verify') %}
+{%-         if slave_instance['health-check-failover-ssl-proxy-verify'] %}
+{%-           if slave_instance['path_to_health-check-failover-ssl-proxy-ca-crt']  %}
+{%-             do ssl_list.append('required ca-file %s' % (slave_instance['path_to_health-check-failover-ssl-proxy-ca-crt'],)) %}
+{%-           else %}
+{#-           Backend SSL shall be verified, but not CA provided, disallow connection #}
+{#-           Simply dropping hostname from the dict will result with ignoring it... #}
+{%-           do info_dict.__setitem__('health-check-failover-hostname', '') %}
+{%-           endif %}
+{%-         else %}
+{%-           do ssl_list.append('none') %}
+{%-         endif %}
+{%-       endif %}
+
+backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover
+{%-       set hostname = info_dict['health-check-failover-hostname'] %}
+{%-       set port = info_dict['health-check-failover-port'] %}
+{%-       set path_list = [info_dict['health-check-failover-path'].rstrip('/')] %}
+{%-       set query = info_dict['health-check-failover-query'] %}
+{%-       if query %}
+{%-         do path_list.append(query) %}
+{%-       endif %}
+{%-       set path = '?'.join(path_list) %}
+{%-       if hostname and port %}
+  timeout server {{ slave_instance['request-timeout'] }}s
+  timeout connect {{ slave_instance['backend-connect-timeout'] }}s
+  retries {{ slave_instance['backend-connect-retries'] }}
+  server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
+{%-         if path %}
+  http-request set-path {{ path }}%[path]
+{%-         endif %}
+{%-       endif %}
+{%-     endif %}
 {%-   endfor %}
 {%- endfor %}

software/caddy-frontend/test/test.py

@@ -6146,6 +6146,16 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
         'ssl-proxy-verify': True,
         'ssl_proxy_ca_crt': '',
       },
+      'health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_DAMAGED': {
+        'url': cls.backend_https_url,
+        'health-check-failover-ssl-proxy-verify': True,
+        'health-check-failover-ssl-proxy-ca-crt': 'damaged',
+      },
+      'health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_EMPTY': {
+        'url': cls.backend_https_url,
+        'health-check-failover-ssl-proxy-verify': True,
+        'health-check-failover-ssl-proxy-ca-crt': '',
+      },
       'BAD-BACKEND': {
         'url': 'http://1:2:3:4',
         'https-url': 'http://host.domain:badport',
@@ -6263,8 +6273,8 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
       'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
       'domain': 'example.com',
       'accepted-slave-amount': '5',
-      'rejected-slave-amount': '26',
-      'slave-amount': '31',
+      'rejected-slave-amount': '28',
+      'slave-amount': '33',
       'rejected-slave-dict': {
         '_HTTPS-URL': ['slave https-url "https://[fd46::c2ae]:!py!u\'123123\'"'
                        ' invalid'],
@@ -6303,6 +6313,12 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
         '_EMPTY-BACKEND': [
           "slave https-url '' invalid",
           "slave url '' invalid"],
+        '_health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_DAMAGED': [
+          'health-check-failover-ssl-proxy-ca-crt is invalid'
+        ],
+        '_health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_EMPTY': [
+          'health-check-failover-ssl-proxy-ca-crt is invalid'
+        ],
         '_health-check-fall': [
           'Wrong health-check-fall WRONG'],
         '_health-check-fall-negative': [
@@ -6373,6 +6389,28 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
       parameter_dict
     )
 
+  def test_health_check_failover_ssl_proxy_ca_crt_damaged(self):
+    parameter_dict = self.parseSlaveParameterDict(
+      'health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_DAMAGED')
+    self.assertEqual(
+      {
+        'request-error-list': [
+          "health-check-failover-ssl-proxy-ca-crt is invalid"]
+      },
+      parameter_dict
+    )
+
+  def test_health_check_failover_ssl_proxy_ca_crt_empty(self):
+    parameter_dict = self.parseSlaveParameterDict(
+      'health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_EMPTY')
+    self.assertEqual(
+      {
+        'request-error-list': [
+          "health-check-failover-ssl-proxy-ca-crt is invalid"]
+      },
+      parameter_dict
+    )
+
   def test_server_alias_same(self):
     parameter_dict = self.parseSlaveParameterDict('SERVER-ALIAS-SAME')
     self.assertLogAccessUrlWithPop(parameter_dict)
@@ -7059,6 +7097,67 @@ class TestSlaveHealthCheck(SlaveHttpFrontendTestCase, TestDataMixin):
         'health-check-rise': '3',
         'health-check-fall': '7',
       },
+      'health-check-failover-url': {
+        'https-only': False,  # http and https access to check
+        'health-check-timeout': 1,  # fail fast for test
+        'health-check-interval': 1,  # fail fast for test
+        'url': cls.backend_url + 'url',
+        'https-url': cls.backend_url + 'https-url',
+        'health-check': True,
+        'health-check-http-path': '/health-check-failover-url',
+        'health-check-failover-url': cls.backend_url + 'failover-url?a=b&c=',
+        'health-check-failover-https-url':
+        cls.backend_url + 'failover-https-url?a=b&c=',
+      },
+      'health-check-failover-url-auth-to-backend': {
+        'https-only': False,  # http and https access to check
+        'health-check-timeout': 1,  # fail fast for test
+        'health-check-interval': 1,  # fail fast for test
+        'url': cls.backend_url + 'url',
+        'https-url': cls.backend_url + 'https-url',
+        'health-check': True,
+        'health-check-http-path': '/health-check-failover-url-auth-to-backend',
+        'health-check-authenticate-to-failover-backend': True,
+        'health-check-failover-url': 'https://%s:%s/failover-url?a=b&c=' % (
+          cls._ipv4_address, cls._server_https_auth_port),
+        'health-check-failover-https-url':
+        'https://%s:%s/failover-https-url?a=b&c=' % (
+          cls._ipv4_address, cls._server_https_auth_port),
+      },
+      'health-check-failover-url-ssl-proxy-verified': {
+        'url': cls.backend_url,
+        'health-check-timeout': 1,  # fail fast for test
+        'health-check-interval': 1,  # fail fast for test
+        'health-check': True,
+        'health-check-http-path': '/health-check-failover-url-ssl-proxy'
+        '-verified',
+        'health-check-failover-url': cls.backend_https_url,
+        'health-check-failover-ssl-proxy-verify': True,
+        'health-check-failover-ssl-proxy-ca-crt':
+        cls.test_server_ca.certificate_pem,
+      },
+      'health-check-failover-url-ssl-proxy-verify-unverified': {
+        'url': cls.backend_url,
+        'health-check-timeout': 1,  # fail fast for test
+        'health-check-interval': 1,  # fail fast for test
+        'health-check': True,
+        'health-check-http-path': '/health-check-failover-url-ssl-proxy-verify'
+        '-unverified',
+        'health-check-failover-url': cls.backend_https_url,
+        'health-check-failover-ssl-proxy-verify': True,
+        'health-check-failover-ssl-proxy-ca-crt':
+        cls.another_server_ca.certificate_pem,
+      },
+      'health-check-failover-url-ssl-proxy-verify-missing': {
+        'url': cls.backend_url,
+        'health-check-timeout': 1,  # fail fast for test
+        'health-check-interval': 1,  # fail fast for test
+        'health-check': True,
+        'health-check-http-path': '/health-check-failover-url-ssl-proxy-verify'
+        '-missing',
+        'health-check-failover-url': cls.backend_https_url,
+        'health-check-failover-ssl-proxy-verify': True,
+      },
     }
 
   @classmethod
@@ -7137,6 +7236,173 @@ backend _health-check-default-http
   def test_health_check_custom(self):
     self._test('health-check-custom')
 
+  def test_health_check_failover_url(self):
+    parameter_dict = self.assertSlaveBase('health-check-failover-url')
+    slave_parameter_dict = self.getSlaveParameterDictDict()[
+      'health-check-failover-url']
+
+    # check normal access
+    result = fakeHTTPResult(parameter_dict['domain'], '/path')
+    self.assertEqualResultJson(result, 'Path', '/url/path')
+    result = fakeHTTPSResult(parameter_dict['domain'], '/path')
+    self.assertEqual(self.certificate_pem, der2pem(result.peercert))
+    self.assertEqualResultJson(result, 'Path', '/https-url/path')
+
+    # start replying with bad status code
+    result = requests.put(
+      self.backend_url + slave_parameter_dict[
+        'health-check-http-path'].strip('/'),
+      headers={'X-Reply-Status-Code': '502'})
+    self.assertEqual(result.status_code, httplib.CREATED)
+
+    time.sleep(3)  # > health-check-timeout + health-check-interval
+
+    result = fakeHTTPSResult(parameter_dict['domain'], '/failoverpath')
+    self.assertEqual(self.certificate_pem, der2pem(result.peercert))
+    self.assertEqualResultJson(
+      result, 'Path', '/failover-https-url?a=b&c=/failoverpath')
+
+    result = fakeHTTPResult(parameter_dict['domain'], '/failoverpath')
+    self.assertEqualResultJson(
+      result, 'Path', '/failover-url?a=b&c=/failoverpath')
+
+  def test_health_check_failover_url_auth_to_backend(self):
+    parameter_dict = self.assertSlaveBase(
+      'health-check-failover-url-auth-to-backend')
+    slave_parameter_dict = self.getSlaveParameterDictDict()[
+      'health-check-failover-url-auth-to-backend']
+
+    self.startAuthenticatedServerProcess()
+    self.addCleanup(self.stopAuthenticatedServerProcess)
+    # assert that you can't fetch nothing without key
+    try:
+      requests.get(self.backend_https_auth_url, verify=False)
+    except Exception:
+      pass
+    else:
+      self.fail(
+        'Access to %r shall be not possible without certificate' % (
+          self.backend_https_auth_url,))
+    # check normal access
+    result = fakeHTTPResult(parameter_dict['domain'], '/path')
+    self.assertEqualResultJson(result, 'Path', '/url/path')
+    self.assertNotIn('X-Backend-Identification', result.headers)
+    result = fakeHTTPSResult(parameter_dict['domain'], '/path')
+    self.assertEqual(self.certificate_pem, der2pem(result.peercert))
+    self.assertEqualResultJson(result, 'Path', '/https-url/path')
+    self.assertNotIn('X-Backend-Identification', result.headers)
+
+    # start replying with bad status code
+    result = requests.put(
+      self.backend_url + slave_parameter_dict[
+        'health-check-http-path'].strip('/'),
+      headers={'X-Reply-Status-Code': '502'})
+    self.assertEqual(result.status_code, httplib.CREATED)
+
+    time.sleep(3)  # > health-check-timeout + health-check-interval
+
+    result = fakeHTTPSResult(parameter_dict['domain'], '/failoverpath')
+    self.assertEqual(self.certificate_pem, der2pem(result.peercert))
+    self.assertEqualResultJson(
+      result, 'Path', '/failover-https-url?a=b&c=/failoverpath')
+    self.assertEqual(
+      'Auth Backend', result.headers['X-Backend-Identification'])
+
+    result = fakeHTTPResult(parameter_dict['domain'], '/failoverpath')
+    self.assertEqualResultJson(
+      result, 'Path', '/failover-url?a=b&c=/failoverpath')
+    self.assertEqual(
+      'Auth Backend', result.headers['X-Backend-Identification'])
+
+  def test_health_check_failover_url_ssl_proxy_verified(self):
+    parameter_dict = self.assertSlaveBase(
+      'health-check-failover-url-ssl-proxy-verified')
+    slave_parameter_dict = self.getSlaveParameterDictDict()[
+      'health-check-failover-url-ssl-proxy-verified']
+
+    # check normal access
+    result = fakeHTTPSResult(parameter_dict['domain'], '/path')
+    self.assertEqual(self.certificate_pem, der2pem(result.peercert))
+    self.assertEqualResultJson(result, 'Path', '/path')
+
+    # start replying with bad status code
+    result = requests.put(
+      self.backend_url + slave_parameter_dict[
+        'health-check-http-path'].strip('/'),
+      headers={'X-Reply-Status-Code': '502'})
+    self.assertEqual(result.status_code, httplib.CREATED)
+
+    time.sleep(3)  # > health-check-timeout + health-check-interval
+
+    result = fakeHTTPSResult(
+      parameter_dict['domain'], '/test-path')
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+  def test_health_check_failover_url_ssl_proxy_unverified(self):
+    parameter_dict = self.assertSlaveBase(
+      'health-check-failover-url-ssl-proxy-verify-unverified')
+    slave_parameter_dict = self.getSlaveParameterDictDict()[
+      'health-check-failover-url-ssl-proxy-verify-unverified']
+
+    # check normal access
+    result = fakeHTTPSResult(parameter_dict['domain'], '/path')
+    self.assertEqual(self.certificate_pem, der2pem(result.peercert))
+    self.assertEqualResultJson(result, 'Path', '/path')
+
+    # start replying with bad status code
+    result = requests.put(
+      self.backend_url + slave_parameter_dict[
+        'health-check-http-path'].strip('/'),
+      headers={'X-Reply-Status-Code': '502'})
+    self.assertEqual(result.status_code, httplib.CREATED)
+
+    time.sleep(3)  # > health-check-timeout + health-check-interval
+
+    result = fakeHTTPSResult(
+      parameter_dict['domain'], '/test-path')
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    # as ssl proxy verification failed, service is unavailable
+    self.assertEqual(result.status_code, httplib.SERVICE_UNAVAILABLE)
+
+  def test_health_check_failover_url_ssl_proxy_missing(self):
+    parameter_dict = self.assertSlaveBase(
+      'health-check-failover-url-ssl-proxy-verify-missing')
+    slave_parameter_dict = self.getSlaveParameterDictDict()[
+      'health-check-failover-url-ssl-proxy-verify-missing']
+
+    # check normal access
+    result = fakeHTTPSResult(parameter_dict['domain'], '/path')
+    self.assertEqual(self.certificate_pem, der2pem(result.peercert))
+    self.assertEqualResultJson(result, 'Path', '/path')
+
+    # start replying with bad status code
+    result = requests.put(
+      self.backend_url + slave_parameter_dict[
+        'health-check-http-path'].strip('/'),
+      headers={'X-Reply-Status-Code': '502'})
+    self.assertEqual(result.status_code, httplib.CREATED)
+
+    time.sleep(3)  # > health-check-timeout + health-check-interval
+
+    result = fakeHTTPSResult(
+      parameter_dict['domain'], '/test-path')
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    # as ssl proxy verification failed, service is unavailable
+    self.assertEqual(result.status_code, httplib.SERVICE_UNAVAILABLE)
+
 
 if __name__ == '__main__':
   class HTTP6Server(ThreadedHTTPServer):

software/caddy-frontend/test/test_data/test.TestSlaveHealthCheck.test_file_list_log-CADDY.txt

@@ -21,6 +21,21 @@ T-2/var/log/httpd/_health-check-default_error_log
 T-2/var/log/httpd/_health-check-disabled_access_log
 T-2/var/log/httpd/_health-check-disabled_backend_log
 T-2/var/log/httpd/_health-check-disabled_error_log
+T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_access_log
+T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_backend_log
+T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_error_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_access_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_backend_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_error_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_access_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_backend_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_error_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_access_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_backend_log
+T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_error_log
+T-2/var/log/httpd/_health-check-failover-url_access_log
+T-2/var/log/httpd/_health-check-failover-url_backend_log
+T-2/var/log/httpd/_health-check-failover-url_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log