Feature Caddy Frontend Haproxy Active Check

See merge request nexedi/slapos!837

software/caddy-frontend/README.caddy_frontend.rst

@@ -245,6 +245,15 @@ Necessary to activate cache.
 
 ``enable_cache`` is an optional parameter.
 
+backend-active-check-*
+~~~~~~~~~~~~~~~~~~~~~~
+
+This set of parameters is used to control the way how the backend checks will be done. Such active checks can be really useful for `stale-if-error` caching technique and especially in case if backend is very slow to reply or to connect to.
+
+`backend-active-check-http-method` can be used to configure the HTTP method used to check the backend. Special method `CONNECT` can be used to check only for connection attempt.
+
+Please be aware that the `backend-active-check-timeout` is really short by default, so in case if `/` of the backend is slow to reply configure proper path with `backend-active-check-http-path` to not mark such backend down too fast, before increasing the check timeout.
+
 Examples
 ========
 

software/caddy-frontend/buildout.hash.cfg

@@ -26,11 +26,11 @@ md5sum = e7d7e1448b6420657e953026573311ca
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 2d421ce6def12f7796cfa28f59eef0df
+md5sum = c713aa837c5d0af8b59ed65e17c36c3a
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = ab143bfa2e20725aa35940c9033fa0ee
+md5sum = 8f187ebb6807732c9e40bdf03e6a8113
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -50,7 +50,7 @@ md5sum = 266f175dbdfc588af7a86b0b1884fe73
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = bf40f8d0a049a8dd924ccc731956c87e
+md5sum = 7c171979aa1d5c88faafe4993f087edd
 
 [template-log-access]
 _update_hash_filename_ = templates/template-log-access.conf.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -124,6 +124,35 @@ context =
 {%   elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
 {%     do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
 {%   endif %}
+{#   Check backend-active-check-* #}
+{%   set backend_active_check = (str(slave.get('backend-active-check', False)) or 'false').lower() %}
+{%   if backend_active_check in TRUE_VALUES %}
+{%     set backend_active_check_http_method = slave.get('backend-active-check-http-method') or 'GET' %}
+{%     if backend_active_check_http_method not in ['GET', 'OPTIONS', 'CONNECT', 'POST'] %}
+{%       do slave_error_list.append('Wrong backend-active-check-http-method %s' % (backend_active_check_http_method,)) %}
+{%     endif %}
+{%     set backend_active_check_http_path = slave.get('backend-active-check-http-path') or '/' %}
+{%     set backend_active_check_http_version = slave.get('backend-active-check-http-version') or 'HTTP/1.1' %}
+{%     if backend_active_check_http_version not in ['HTTP/1.1', 'HTTP/1.0'] %}
+{%       do slave_error_list.append('Wrong backend-active-check-http-version %s' % (backend_active_check_http_version,)) %}
+{%     endif %}
+{%     set backend_active_check_timeout = (slave.get('backend-active-check-timeout') or '2') | int(False) %}
+{%     if backend_active_check_timeout in [False] or backend_active_check_timeout <= 0 %}
+{%       do slave_error_list.append('Wrong backend-active-check-timeout %s' % (slave.get('backend-active-check-timeout'),)) %}
+{%     endif %}
+{%     set backend_active_check_interval = (slave.get('backend-active-check-interval') or '5') | int(False) %}
+{%     if backend_active_check_interval in [False] or backend_active_check_interval <= 0 %}
+{%       do slave_error_list.append('Wrong backend-active-check-interval %s' % (slave.get('backend-active-check-interval'),)) %}
+{%     endif %}
+{%     set backend_active_check_rise = (slave.get('backend-active-check-rise') or '1') | int(False) %}
+{%     if backend_active_check_rise in [False] or backend_active_check_rise <= 0 %}
+{%       do slave_error_list.append('Wrong backend-active-check-rise %s' % (slave.get('backend-active-check-rise'),)) %}
+{%     endif %}
+{%     set backend_active_check_fall = (slave.get('backend-active-check-fall') or '1') | int(False) %}
+{%     if backend_active_check_fall in [False] or backend_active_check_fall <= 0 %}
+{%       do slave_error_list.append('Wrong backend-active-check-fall %s' % (slave.get('backend-active-check-fall'),)) %}
+{%     endif %}
+{%   endif %}
 {#   Check ciphers #}
 {%   set slave_cipher_list = slave.get('ciphers', '').strip().split() %}
 {%   if slave_cipher_list %}

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -223,6 +223,68 @@
       ],
       "title": "Authenticate to backend",
       "type": "string"
+    },
+    "backend-active-check": {
+      "title": "Backend Active Check",
+      "description": "Enables active checks of the backend. For HTTP level checks the HTTP code shall be 2xx or 3xx, otherwise backend will be considered down.",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "default": "false",
+      "type": "string"
+    },
+    "backend-active-check-http-method": {
+      "title": "Backend Active Check HTTP Metod",
+      "description": "Selects method to do the active check. CONNECT means that connection will be enough for the check, otherwise it's HTTP method.",
+      "enum": [
+        "GET",
+        "OPTIONS",
+        "POST",
+        "CONNECT"
+      ],
+      "default": "GET",
+      "type": "string"
+    },
+    "backend-active-check-http-path": {
+      "title": "Backend Active Check HTTP Path",
+      "description": "A path on which do the active check, unused in case of CONNECT.",
+      "default": "/",
+      "type": "string"
+    },
+    "backend-active-check-http-version": {
+      "title": "Backend Active Check HTTP Version",
+      "description": "A HTTP version to use to check the backend, unused in case of CONNECT.",
+      "enum": [
+        "HTTP/1.1",
+        "HTTP/1.0"
+      ],
+      "default": "HTTP/1.1",
+      "type": "string"
+    },
+    "backend-active-check-timeout": {
+      "title": "Backend Active Check Timeout (seconds)",
+      "description": "A timeout to for the request to be fulfilled, after connection happen.",
+      "default": "2",
+      "type": "integer"
+    },
+    "backend-active-check-interval": {
+      "title": "Backend Active Check Interval (seconds)",
+      "description": "An interval of backend active check.",
+      "default": "5",
+      "type": "integer"
+    },
+    "backend-active-check-rise": {
+      "title": "Backend Active Check Rise",
+      "description": "Amount of correct responses from the backend to consider it up.",
+      "default": "1",
+      "type": "integer"
+    },
+    "backend-active-check-fall": {
+      "title": "Backend Active Check Fall",
+      "description": "Amount of bad responses from the backend to consider it down.",
+      "default": "1",
+      "type": "integer"
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -113,6 +113,33 @@ context =
 {%-     endif %}
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %}
+{#-   Setup active check #}
+{%-   do slave_instance.__setitem__('backend-active-check',  ('' ~ slave_instance.get('backend-active-check', '')).lower() in TRUE_VALUES) %}
+{%-   if slave_instance['backend-active-check']  %}
+{%-     if 'backend-active-check-http-method' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-http-method', 'GET') %}
+{%-     endif %}
+{%-     if 'backend-active-check-http-version' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-http-version', 'HTTP/1.1') %}
+{%-     endif %}
+{%-     if 'backend-active-check-interval' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-interval', '5') %}
+{%-     endif %}
+{%-     if 'backend-active-check-rise' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-rise', '1') %}
+{%-     endif %}
+{%-     if 'backend-active-check-fall' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-fall', '2') %}
+{%-     endif %}
+{%-     if 'backend-active-check-timeout' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-timeout', '2') %}
+{%-     endif %}
+{%-     do slave_instance.__setitem__('backend-active-check-http-path', slave_instance.get('backend-active-check-http-path') or '/') %}
+{%-   else %}
+{%-     do slave_instance.__setitem__('backend-active-check-http-method', '') %}
+{%-     do slave_instance.__setitem__('backend-active-check-http-version', '') %}
+{%-     do slave_instance.__setitem__('backend-active-check-http-path', '') %}
+{%-   endif %} {# if backend_active_check #}
 {#-   Set Up log files #}
 {%-   do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %}
 {%-   do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -100,7 +100,22 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
   timeout server {{ slave_instance['request-timeout'] }}s
   timeout connect {{ slave_instance['backend-connect-timeout'] }}s
   retries {{ slave_instance['backend-connect-retries'] }}
-  server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
+{%-         set active_check_list = [] %}
+{%-         set active_check_option_list = [] %}
+{%-         if slave_instance['backend-active-check'] %}
+{%-           do active_check_list.append('check') %}
+{%-           do active_check_list.append('inter %ss' % (slave_instance['backend-active-check-interval'])) %}
+{%-           do active_check_list.append('rise %s' % (slave_instance['backend-active-check-rise'])) %}
+{%-           do active_check_list.append('fall %s' % (slave_instance['backend-active-check-fall'])) %}
+{%-           if slave_instance['backend-active-check-http-method'] != 'CONNECT' %}
+{%-             do active_check_option_list.append('option httpchk %s %s %s' % (slave_instance['backend-active-check-http-method'], slave_instance['backend-active-check-http-path'] | urlencode, slave_instance['backend-active-check-http-version'])) %}
+{%-           endif %}
+{%-           do active_check_option_list.append('timeout check %ss' % (slave_instance['backend-active-check-timeout'])) %}
+{%-         endif %}
+  server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }} {{ ' ' + ' '.join(active_check_list)}}
+{%-         for active_check_option in active_check_option_list %}
+  {{ active_check_option }}
+{%-         endfor %}
 {%-         if path %}
   http-request set-path {{ path }}%[path]
 {%-         endif %}

software/caddy-frontend/test/test.py

@@ -48,6 +48,7 @@ from slapos.recipe.librecipe import generateHashFromFiles
 import xml.etree.ElementTree as ET
 import urlparse
 import socket
+import sys
 
 
 try:
@@ -430,6 +431,9 @@ def fakeHTTPResult(domain, real_ip, path, port=HTTP_PORT,
 class TestHandler(BaseHTTPRequestHandler):
   identification = None
 
+  def do_POST(self):
+    return self.do_GET()
+
   def do_GET(self):
     timeout = int(self.headers.dict.get('timeout', '0'))
     compress = int(self.headers.dict.get('compress', '0'))
@@ -6290,6 +6294,46 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
         'ssl_key': '${section:option}ssl_keyunsafe\nunsafe',
         'ssl_crt': '${section:option}ssl_crtunsafe\nunsafe',
       },
+      'backend-active-check-http-method': {
+        'backend-active-check': True,
+        'backend-active-check-http-method': 'WRONG',
+      },
+      'backend-active-check-http-version': {
+        'backend-active-check': True,
+        'backend-active-check-http-version': 'WRONG/1.1',
+      },
+      'backend-active-check-timeout': {
+        'backend-active-check': True,
+        'backend-active-check-timeout': 'WRONG',
+      },
+      'backend-active-check-timeout-negative': {
+        'backend-active-check': True,
+        'backend-active-check-timeout': '-2',
+      },
+      'backend-active-check-interval': {
+        'backend-active-check': True,
+        'backend-active-check-interval': 'WRONG',
+      },
+      'backend-active-check-interval-negative': {
+        'backend-active-check': True,
+        'backend-active-check-interval': '-2',
+      },
+      'backend-active-check-rise': {
+        'backend-active-check': True,
+        'backend-active-check-rise': 'WRONG',
+      },
+      'backend-active-check-rise-negative': {
+        'backend-active-check': True,
+        'backend-active-check-rise': '-2',
+      },
+      'backend-active-check-fall': {
+        'backend-active-check': True,
+        'backend-active-check-fall': 'WRONG',
+      },
+      'backend-active-check-fall-negative': {
+        'backend-active-check': True,
+        'backend-active-check-fall': '-2',
+      }
     }
 
   def test_master_partition_state(self):
@@ -6304,8 +6348,8 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
       'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
       'domain': 'example.com',
       'accepted-slave-amount': '7',
-      'rejected-slave-amount': '14',
-      'slave-amount': '21',
+      'rejected-slave-amount': '24',
+      'slave-amount': '31',
       'rejected-slave-dict': {
         '_HTTPS-URL': ['slave https-url "https://[fd46::c2ae]:!py!u\'123123\'"'
                        ' invalid'],
@@ -6340,6 +6384,26 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
         '_EMPTY-BACKEND': [
           "slave https-url '' invalid",
           "slave url '' invalid"],
+        '_backend-active-check-fall': [
+          'Wrong backend-active-check-fall WRONG'],
+        '_backend-active-check-fall-negative': [
+          'Wrong backend-active-check-fall -2'],
+        '_backend-active-check-http-method': [
+          'Wrong backend-active-check-http-method WRONG'],
+        '_backend-active-check-http-version': [
+          'Wrong backend-active-check-http-version WRONG/1.1'],
+        '_backend-active-check-interval': [
+          'Wrong backend-active-check-interval WRONG'],
+        '_backend-active-check-interval-negative': [
+          'Wrong backend-active-check-interval -2'],
+        '_backend-active-check-rise': [
+          'Wrong backend-active-check-rise WRONG'],
+        '_backend-active-check-rise-negative': [
+          'Wrong backend-active-check-rise -2'],
+        '_backend-active-check-timeout': [
+          'Wrong backend-active-check-timeout WRONG'],
+        '_backend-active-check-timeout-negative': [
+          'Wrong backend-active-check-timeout -2'],
       },
       'warning-slave-dict': {
         '_SSL_CA_CRT_ONLY': [
@@ -7080,3 +7144,139 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
       expected_partition_parameter_dict_dict,
       partition_parameter_dict_dict
     )
+
+
+class TestSlaveBackendActiveCheck(SlaveHttpFrontendTestCase, TestDataMixin):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'domain': 'example.com',
+      'public-ipv4': cls._ipv4_address,
+      'port': HTTPS_PORT,
+      'plain_http_port': HTTP_PORT,
+      'kedifa_port': KEDIFA_PORT,
+      'caucase_port': CAUCASE_PORT,
+      'mpm-graceful-shutdown-timeout': 2,
+      'request-timeout': '12',
+    }
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    cls.setUpAssertionDict()
+    return {
+      'backend-active-check-disabled': {
+        'url': cls.backend_url,
+      },
+      'backend-active-check-default': {
+        'url': cls.backend_url,
+        'backend-active-check': True,
+      },
+      'backend-active-check-connect': {
+        'url': cls.backend_url,
+        'backend-active-check': True,
+        'backend-active-check-http-method': 'CONNECT',
+      },
+      'backend-active-check-custom': {
+        'url': cls.backend_url,
+        'backend-active-check': True,
+        'backend-active-check-http-method': 'POST',
+        'backend-active-check-http-path': '/POST-path to be encoded',
+        'backend-active-check-http-version': 'HTTP/1.0',
+        'backend-active-check-timeout': '7',
+        'backend-active-check-interval': '15',
+        'backend-active-check-rise': '3',
+        'backend-active-check-fall': '7',
+      },
+    }
+
+  @classmethod
+  def setUpAssertionDict(cls):
+    backend = urlparse.urlparse(cls.backend_url).netloc
+    cls.assertion_dict = {
+      'backend-active-check-disabled': """\
+backend _backend-active-check-disabled-http
+  timeout server 12s
+  timeout connect 5s
+  retries 3
+  server _backend-active-check-disabled-backend %s""" % (backend,),
+      'backend-active-check-connect': """\
+backend _backend-active-check-connect-http
+  timeout server 12s
+  timeout connect 5s
+  retries 3
+  server _backend-active-check-connect-backend %s   check inter 5s"""
+      """ rise 1 fall 2
+  timeout check 2s""" % (backend,),
+      'backend-active-check-custom': """\
+backend _backend-active-check-custom-http
+  timeout server 12s
+  timeout connect 5s
+  retries 3
+  server _backend-active-check-custom-backend %s   check inter 15s"""
+      """ rise 3 fall 7
+  option httpchk POST /POST-path%%20to%%20be%%20encoded HTTP/1.0
+  timeout check 7s""" % (backend,),
+      'backend-active-check-default': """\
+backend _backend-active-check-default-http
+  timeout server 12s
+  timeout connect 5s
+  retries 3
+  server _backend-active-check-default-backend %s   check inter 5s"""
+      """ rise 1 fall 2
+  option httpchk GET / HTTP/1.1
+  timeout check 2s""" % (backend, )
+    }
+
+  def _get_backend_haproxy_configuration(self):
+    backend_configuration_file = glob.glob(os.path.join(
+      self.instance_path, '*', 'etc', 'backend-haproxy.cfg'))[0]
+    with open(backend_configuration_file) as fh:
+      return fh.read()
+
+  def _test(self, key):
+    parameter_dict = self.assertSlaveBase(key)
+    self.assertIn(
+      self.assertion_dict[key],
+      self._get_backend_haproxy_configuration()
+    )
+    result = fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'test-path/deep/.././deeper',
+      headers={
+        'Timeout': '10',  # more than default backend-connect-timeout == 5
+        'Accept-Encoding': 'gzip',
+      }
+    )
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
+
+  def test_backend_active_check_disabled(self):
+    self._test('backend-active-check-disabled')
+
+  def test_backend_active_check_default(self):
+    self._test('backend-active-check-default')
+
+  def test_backend_active_check_connect(self):
+    self._test('backend-active-check-connect')
+
+  def test_backend_active_check_custom(self):
+    self._test('backend-active-check-custom')
+
+
+if __name__ == '__main__':
+  class HTTP6Server(HTTPServer):
+    address_family = socket.AF_INET6
+  ip, port = sys.argv[1], int(sys.argv[2])
+  if ':' in ip:
+    klass = HTTP6Server
+    url_template = 'http://[%s]:%s/'
+  else:
+    klass = HTTPServer
+    url_template = 'http://%s:%s/'
+
+  server = klass((ip, port), TestHandler)
+  print url_template % server.server_address[:2]
+  server.serve_forever()

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_file_list_etc_cron_d-CADDY.txt

+T-0/etc/cron.d/logrotate
+T-0/etc/cron.d/monitor-configurator
+T-0/etc/cron.d/monitor-globalstate
+T-0/etc/cron.d/monitor_collect
+T-1/etc/cron.d/logrotate
+T-1/etc/cron.d/monitor-configurator
+T-1/etc/cron.d/monitor-globalstate
+T-1/etc/cron.d/monitor_collect
+T-2/etc/cron.d/logrotate
+T-2/etc/cron.d/monitor-configurator
+T-2/etc/cron.d/monitor-globalstate
+T-2/etc/cron.d/monitor_collect
+T-2/etc/cron.d/trafficserver-logrotate

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_file_list_log-CADDY.txt

+T-0/var/log/monitor-httpd-access.log
+T-0/var/log/monitor-httpd-error.log
+T-0/var/log/slapgrid-T-0-error.log
+T-1/var/log/expose-csr_id.log
+T-1/var/log/kedifa.log
+T-1/var/log/monitor-httpd-access.log
+T-1/var/log/monitor-httpd-error.log
+T-2/var/log/backend-haproxy.log
+T-2/var/log/frontend-access.log
+T-2/var/log/frontend-error.log
+T-2/var/log/httpd-csr_id/expose-csr_id.log
+T-2/var/log/httpd/_backend-active-check-connect_access_log
+T-2/var/log/httpd/_backend-active-check-connect_backend_log
+T-2/var/log/httpd/_backend-active-check-connect_error_log
+T-2/var/log/httpd/_backend-active-check-custom_access_log
+T-2/var/log/httpd/_backend-active-check-custom_backend_log
+T-2/var/log/httpd/_backend-active-check-custom_error_log
+T-2/var/log/httpd/_backend-active-check-default_access_log
+T-2/var/log/httpd/_backend-active-check-default_backend_log
+T-2/var/log/httpd/_backend-active-check-default_error_log
+T-2/var/log/httpd/_backend-active-check-disabled_access_log
+T-2/var/log/httpd/_backend-active-check-disabled_backend_log
+T-2/var/log/httpd/_backend-active-check-disabled_error_log
+T-2/var/log/monitor-httpd-access.log
+T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
+T-2/var/log/trafficserver/manager.log

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_file_list_plugin-CADDY.txt

+T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
+T-0/etc/plugin/aikc-user-caucase-updater.py
+T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
+T-0/etc/plugin/check-free-disk-space.py
+T-0/etc/plugin/monitor-bootstrap-status.py
+T-0/etc/plugin/monitor-http-frontend.py
+T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
+T-0/etc/plugin/rejected-slave.py
+T-1/etc/plugin/__init__.py
+T-1/etc/plugin/buildout-T-1-status.py
+T-1/etc/plugin/caucased.py
+T-1/etc/plugin/check-free-disk-space.py
+T-1/etc/plugin/expose-csr_id-ip-port-listening.py
+T-1/etc/plugin/kedifa-http-reply.py
+T-1/etc/plugin/monitor-bootstrap-status.py
+T-1/etc/plugin/monitor-http-frontend.py
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
+T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend_haproxy_http.py
+T-2/etc/plugin/backend_haproxy_https.py
+T-2/etc/plugin/buildout-T-2-status.py
+T-2/etc/plugin/caddy_frontend_ipv4_http.py
+T-2/etc/plugin/caddy_frontend_ipv4_https.py
+T-2/etc/plugin/caddy_frontend_ipv6_http.py
+T-2/etc/plugin/caddy_frontend_ipv6_https.py
+T-2/etc/plugin/caucase-updater.py
+T-2/etc/plugin/check-free-disk-space.py
+T-2/etc/plugin/expose-csr_id-ip-port-listening.py
+T-2/etc/plugin/frontend-caddy-configuration-promise.py
+T-2/etc/plugin/monitor-bootstrap-status.py
+T-2/etc/plugin/monitor-http-frontend.py
+T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
+T-2/etc/plugin/trafficserver-cache-availability.py
+T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_file_list_run-CADDY.txt

+T-0/var/run/monitor-httpd.pid
+T-1/var/run/kedifa.pid
+T-1/var/run/monitor-httpd.pid
+T-2/var/run/backend-haproxy-rsyslogd.pid
+T-2/var/run/backend-haproxy.pid
+T-2/var/run/backend_haproxy_configuration_last_state
+T-2/var/run/backend_haproxy_graceful_configuration_state_signature
+T-2/var/run/bhlog.sck
+T-2/var/run/graceful_configuration_state_signature
+T-2/var/run/httpd.pid
+T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
+T-0:aikc-user-caucase-updater-on-watch RUNNING
+T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
+T-0:certificate_authority-{hash-generic}-on-watch RUNNING
+T-0:crond-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-graceful EXITED
+T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
+T-1:bootstrap-monitor EXITED
+T-1:caucase-updater-on-watch RUNNING
+T-1:caucased-{hash-generic}-on-watch RUNNING
+T-1:certificate_authority-{hash-generic}-on-watch RUNNING
+T-1:crond-{hash-generic}-on-watch RUNNING
+T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-1:kedifa-{hash-generic}-on-watch RUNNING
+T-1:kedifa-reloader EXITED
+T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-1:monitor-httpd-graceful EXITED
+T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
+T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
+T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-safe-graceful EXITED
+T-2:bootstrap-monitor EXITED
+T-2:certificate_authority-{hash-generic}-on-watch RUNNING
+T-2:crond-{hash-generic}-on-watch RUNNING
+T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-2:frontend-caddy-safe-graceful EXITED
+T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
+T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
+T-2:trafficserver-{hash-generic}-on-watch RUNNING
+T-2:trafficserver-reload EXITED