caddy-frontend: Unskip some tests

By using a late instance creation, it's possible to escape requirement of
the cluster being fully instantiated by default, so it's possible to test
various interesting corner cases.

software/caddy-frontend/test/test.py

@@ -887,12 +887,12 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     return parsed_parameter_dict
 
   def getMasterPartitionPath(self):
-    return '/' + os.path.join(
-      *glob.glob(
-        os.path.join(
-          self.instance_path, '*', 'etc', 'Caddyfile-rejected-slave'
-        )
-      )[0].split('/')[:-2])
+    # partition w/o etc/trafficserver, but with buildout.cfg
+    return [
+      q for q in glob.glob(os.path.join(self.instance_path, '*',))
+      if not os.path.exists(
+        os.path.join(q, 'etc', 'trafficserver')) and os.path.exists(
+          os.path.join(q, 'buildout.cfg'))][0]
 
   def parseConnectionParameterDict(self):
     return self.parseParameterDict(
@@ -1583,14 +1583,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       got_status_code_list
     )
 
-  def getMasterPartitionPath(self):
-    # partition w/o etc/trafficserver, but with buildout.cfg
-    return [
-      q for q in glob.glob(os.path.join(self.instance_path, '*',))
-      if not os.path.exists(
-        os.path.join(q, 'etc', 'trafficserver')) and os.path.exists(
-          os.path.join(q, 'buildout.cfg'))][0]
-
   def getSlavePartitionPath(self):
     # partition w/ etc/trafficserver
     return [
@@ -4678,21 +4670,19 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
   retries 5""" in content)
 
 
-@skip('Impossible to instantiate cluster with stopped partition')
 class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {
+  instance_parameter_dict = {
       'domain': 'example.com',
-      '-frontend-quantity': 2,
-      '-sla-2-computer_guid': cls.slap._computer_id,
-      '-frontend-2-state': 'stopped',
       'port': HTTPS_PORT,
       'plain_http_port': HTTP_PORT,
       'kedifa_port': KEDIFA_PORT,
       'caucase_port': CAUCASE_PORT,
     }
 
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return cls.instance_parameter_dict
+
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -4703,6 +4693,29 @@ class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     }
 
   def test(self):
+    # now instantiate 2nd partition in started state
+    # and due to port collision, stop the first one...
+    self.instance_parameter_dict.update({
+      '-frontend-quantity': 2,
+      '-sla-2-computer_guid': self.slap._computer_id,
+      '-frontend-1-state': 'stopped',
+      '-frontend-2-state': 'started',
+    })
+    self.requestDefaultInstance()
+    self.requestSlaves()
+    self.slap.waitForInstance(self.instance_max_retry)
+    # ...and be nice, put back the first one online
+    self.instance_parameter_dict.update({
+      '-frontend-1-state': 'started',
+      '-frontend-2-state': 'stopped',
+    })
+    self.requestDefaultInstance()
+    self.slap.waitForInstance(self.instance_max_retry)
+    self.slap.waitForInstance(self.instance_max_retry)
+    self.slap.waitForInstance(self.instance_max_retry)
+
+    self.updateSlaveConnectionParameterDictDict()
+    # the real assertions follow...
     parameter_dict = self.parseSlaveParameterDict('replicate')
     self.assertLogAccessUrlWithPop(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict)
@@ -4729,7 +4742,7 @@ class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     result_http = fakeHTTPResult(
       parameter_dict['domain'], 'test-path')
-    self.assertEqualResultJson(result_http, 'Path', '/test-path')
+    self.assertEqual(httplib.FOUND, result_http.status_code)
 
     # prove 2nd frontend by inspection of the instance
     slave_configuration_name = '_replicate.conf'
@@ -4744,21 +4757,19 @@ class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       2, len(slave_configuration_file_list), slave_configuration_file_list)
 
 
-@skip('Impossible to instantiate cluster with destroyed partition')
 class TestReplicateSlaveOtherDestroyed(SlaveHttpFrontendTestCase):
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {
+  instance_parameter_dict = {
       'domain': 'example.com',
-      '-frontend-quantity': 2,
-      '-sla-2-computer_guid': cls.slap._computer_id,
-      '-frontend-2-state': 'destroyed',
       'port': HTTPS_PORT,
       'plain_http_port': HTTP_PORT,
       'kedifa_port': KEDIFA_PORT,
       'caucase_port': CAUCASE_PORT,
     }
 
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return cls.instance_parameter_dict
+
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -4769,6 +4780,28 @@ class TestReplicateSlaveOtherDestroyed(SlaveHttpFrontendTestCase):
     }
 
   def test_extra_slave_instance_list_not_present_destroyed_request(self):
+    # now instantiate 2nd partition in started state
+    # and due to port collision, stop the first one
+    self.instance_parameter_dict.update({
+      '-frontend-quantity': 2,
+      '-sla-2-computer_guid': self.slap._computer_id,
+      '-frontend-1-state': 'stopped',
+      '-frontend-2-state': 'started',
+
+    })
+    self.requestDefaultInstance()
+    self.slap.waitForInstance(self.instance_max_retry)
+
+    # now start back first instance, and destroy 2nd one
+    self.instance_parameter_dict.update({
+      '-frontend-1-state': 'started',
+      '-frontend-2-state': 'destroyed',
+    })
+    self.requestDefaultInstance()
+    self.slap.waitForInstance(self.instance_max_retry)
+    self.slap.waitForInstance(self.instance_max_retry)
+    self.slap.waitForInstance(self.instance_max_retry)
+
     buildout_file = os.path.join(
       self.getMasterPartitionPath(), 'buildout-switch-softwaretype.cfg')
     with open(buildout_file) as fh:
@@ -5021,19 +5054,20 @@ class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase,
     )
 
 
-@skip('New test system cannot be used with failing promises')
 class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
                                     TestDataMixin):
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {
+  instance_parameter_dict = {
       'port': HTTPS_PORT,
+      'domain': 'example.com',
       'plain_http_port': HTTP_PORT,
-      're6st-verification-url': 'some-re6st-verification-url',
       'kedifa_port': KEDIFA_PORT,
       'caucase_port': CAUCASE_PORT,
     }
 
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return cls.instance_parameter_dict
+
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -5044,16 +5078,28 @@ class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
     }
 
   def test_default(self):
+    self.instance_parameter_dict[
+      're6st-verification-url'] = 'some-re6st-verification-url'
+    # re-request instance with updated parameters
+    self.requestDefaultInstance()
+
+    # run once instance, it's only needed for later checks
+    try:
+      self.slap.waitForInstance()
+    except Exception:
+      pass
+
     parameter_dict = self.parseSlaveParameterDict('default')
     self.assertLogAccessUrlWithPop(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict)
     self.assertEqual(
       {
-        'domain': 'default.None',
+        'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
+        'domain': 'default.example.com',
         'replication_number': '1',
-        'url': 'http://default.None',
-        'site_url': 'http://default.None',
-        'secure_access': 'https://default.None',
+        'url': 'http://default.example.com',
+        'site_url': 'http://default.example.com',
+        'secure_access': 'https://default.example.com',
       },
       parameter_dict
     )

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_etc_cron_d-CADDY.txt

+T-0/etc/cron.d/logrotate
+T-0/etc/cron.d/monitor-configurator
+T-0/etc/cron.d/monitor-globalstate
+T-0/etc/cron.d/monitor_collect
+T-1/etc/cron.d/logrotate
+T-1/etc/cron.d/monitor-configurator
+T-1/etc/cron.d/monitor-globalstate
+T-1/etc/cron.d/monitor_collect
+T-2/etc/cron.d/logrotate
+T-2/etc/cron.d/monitor-configurator
+T-2/etc/cron.d/monitor-globalstate
+T-2/etc/cron.d/monitor_collect
+T-2/etc/cron.d/trafficserver-logrotate

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_log-CADDY.txt

+T-0/var/log/monitor-httpd-access.log
+T-0/var/log/monitor-httpd-error.log
+T-0/var/log/slapgrid-T-0-error.log
+T-1/var/log/expose-csr_id.log
+T-1/var/log/kedifa.log
+T-1/var/log/monitor-httpd-access.log
+T-1/var/log/monitor-httpd-error.log
+T-2/var/log/backend-haproxy.log
+T-2/var/log/expose-csr_id.log
+T-2/var/log/frontend-access.log
+T-2/var/log/frontend-error.log
+T-2/var/log/httpd/_default_access_log
+T-2/var/log/httpd/_default_backend_log
+T-2/var/log/httpd/_default_error_log
+T-2/var/log/monitor-httpd-access.log
+T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
+T-2/var/log/trafficserver/manager.log

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_plugin-CADDY.txt

+T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
+T-0/etc/plugin/aikc-user-caucase-updater.py
+T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
+T-0/etc/plugin/check-backend-haproxy-statistic-url-caddy-frontend-1.py
+T-0/etc/plugin/check-free-disk-space.py
+T-0/etc/plugin/monitor-bootstrap-status.py
+T-0/etc/plugin/monitor-http-frontend.py
+T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
+T-0/etc/plugin/rejected-slave.py
+T-1/etc/plugin/__init__.py
+T-1/etc/plugin/buildout-T-1-status.py
+T-1/etc/plugin/caucased.py
+T-1/etc/plugin/check-free-disk-space.py
+T-1/etc/plugin/expose-csr_id-ip-port-listening.py
+T-1/etc/plugin/kedifa-http-reply.py
+T-1/etc/plugin/monitor-bootstrap-status.py
+T-1/etc/plugin/monitor-http-frontend.py
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
+T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend_haproxy_http.py
+T-2/etc/plugin/backend_haproxy_https.py
+T-2/etc/plugin/buildout-T-2-status.py
+T-2/etc/plugin/caddy_frontend_ipv4_http.py
+T-2/etc/plugin/caddy_frontend_ipv4_https.py
+T-2/etc/plugin/caddy_frontend_ipv6_http.py
+T-2/etc/plugin/caddy_frontend_ipv6_https.py
+T-2/etc/plugin/caucase-updater.py
+T-2/etc/plugin/check-free-disk-space.py
+T-2/etc/plugin/expose-csr_id-ip-port-listening.py
+T-2/etc/plugin/frontend-caddy-configuration-promise.py
+T-2/etc/plugin/monitor-bootstrap-status.py
+T-2/etc/plugin/monitor-http-frontend.py
+T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
+T-2/etc/plugin/trafficserver-cache-availability.py
+T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_run-CADDY.txt

+T-0/var/run/monitor-httpd.pid
+T-1/var/run/kedifa.pid
+T-1/var/run/monitor-httpd.pid
+T-2/var/run/backend-haproxy-rsyslogd.pid
+T-2/var/run/backend-haproxy.pid
+T-2/var/run/backend_haproxy_configuration_last_state
+T-2/var/run/backend_haproxy_graceful_configuration_state_signature
+T-2/var/run/bhlog.sck
+T-2/var/run/graceful_configuration_state_signature
+T-2/var/run/httpd.pid
+T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
+T-0:aikc-user-caucase-updater-on-watch RUNNING
+T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
+T-0:certificate_authority-{hash-generic}-on-watch RUNNING
+T-0:crond-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-graceful EXITED
+T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
+T-1:bootstrap-monitor EXITED
+T-1:caucase-updater-on-watch RUNNING
+T-1:caucased-{hash-generic}-on-watch RUNNING
+T-1:certificate_authority-{hash-generic}-on-watch RUNNING
+T-1:crond-{hash-generic}-on-watch RUNNING
+T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-1:kedifa-{hash-generic}-on-watch RUNNING
+T-1:kedifa-reloader EXITED
+T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-1:monitor-httpd-graceful EXITED
+T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
+T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
+T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-safe-graceful EXITED
+T-2:bootstrap-monitor EXITED
+T-2:certificate_authority-{hash-generic}-on-watch RUNNING
+T-2:crond-{hash-generic}-on-watch RUNNING
+T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-2:frontend-caddy-safe-graceful EXITED
+T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
+T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
+T-2:trafficserver-{hash-generic}-on-watch RUNNING
+T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_etc_cron_d-CADDY.txt

+T-0/etc/cron.d/logrotate
+T-0/etc/cron.d/monitor-configurator
+T-0/etc/cron.d/monitor-globalstate
+T-0/etc/cron.d/monitor_collect
+T-1/etc/cron.d/logrotate
+T-1/etc/cron.d/monitor-configurator
+T-1/etc/cron.d/monitor-globalstate
+T-1/etc/cron.d/monitor_collect
+T-2/etc/cron.d/logrotate
+T-2/etc/cron.d/monitor-configurator
+T-2/etc/cron.d/monitor-globalstate
+T-2/etc/cron.d/monitor_collect
+T-2/etc/cron.d/trafficserver-logrotate
+T-3/etc/cron.d/logrotate
+T-3/etc/cron.d/monitor-configurator
+T-3/etc/cron.d/monitor-globalstate
+T-3/etc/cron.d/monitor_collect
+T-3/etc/cron.d/trafficserver-logrotate

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_log-CADDY.txt

+T-0/var/log/monitor-httpd-access.log
+T-0/var/log/monitor-httpd-error.log
+T-0/var/log/slapgrid-T-0-error.log
+T-1/var/log/expose-csr_id.log
+T-1/var/log/kedifa.log
+T-1/var/log/monitor-httpd-access.log
+T-1/var/log/monitor-httpd-error.log
+T-2/var/log/backend-haproxy.log
+T-2/var/log/expose-csr_id.log
+T-2/var/log/frontend-access.log
+T-2/var/log/frontend-error.log
+T-2/var/log/httpd/_replicate_access_log
+T-2/var/log/httpd/_replicate_backend_log
+T-2/var/log/httpd/_replicate_error_log
+T-2/var/log/monitor-httpd-access.log
+T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
+T-2/var/log/trafficserver/manager.log
+T-3/var/log/backend-haproxy.log
+T-3/var/log/expose-csr_id.log
+T-3/var/log/frontend-access.log
+T-3/var/log/frontend-error.log
+T-3/var/log/httpd/_replicate_access_log
+T-3/var/log/httpd/_replicate_error_log
+T-3/var/log/monitor-httpd-access.log
+T-3/var/log/monitor-httpd-error.log
+T-3/var/log/slave-introspection-access.log
+T-3/var/log/slave-introspection-error.log
+T-3/var/log/trafficserver/manager.log

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_plugin-CADDY.txt

+T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
+T-0/etc/plugin/aikc-user-caucase-updater.py
+T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
+T-0/etc/plugin/check-backend-haproxy-statistic-url-caddy-frontend-1.py
+T-0/etc/plugin/check-backend-haproxy-statistic-url-caddy-frontend-2.py
+T-0/etc/plugin/check-free-disk-space.py
+T-0/etc/plugin/monitor-bootstrap-status.py
+T-0/etc/plugin/monitor-http-frontend.py
+T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
+T-0/etc/plugin/rejected-slave.py
+T-1/etc/plugin/__init__.py
+T-1/etc/plugin/buildout-T-1-status.py
+T-1/etc/plugin/caucased.py
+T-1/etc/plugin/check-free-disk-space.py
+T-1/etc/plugin/expose-csr_id-ip-port-listening.py
+T-1/etc/plugin/kedifa-http-reply.py
+T-1/etc/plugin/monitor-bootstrap-status.py
+T-1/etc/plugin/monitor-http-frontend.py
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
+T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend_haproxy_http.py
+T-2/etc/plugin/backend_haproxy_https.py
+T-2/etc/plugin/buildout-T-2-status.py
+T-2/etc/plugin/caddy_frontend_ipv4_http.py
+T-2/etc/plugin/caddy_frontend_ipv4_https.py
+T-2/etc/plugin/caddy_frontend_ipv6_http.py
+T-2/etc/plugin/caddy_frontend_ipv6_https.py
+T-2/etc/plugin/caucase-updater.py
+T-2/etc/plugin/check-free-disk-space.py
+T-2/etc/plugin/expose-csr_id-ip-port-listening.py
+T-2/etc/plugin/frontend-caddy-configuration-promise.py
+T-2/etc/plugin/monitor-bootstrap-status.py
+T-2/etc/plugin/monitor-http-frontend.py
+T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
+T-2/etc/plugin/trafficserver-cache-availability.py
+T-2/etc/plugin/trafficserver-port-listening.py
+T-3/etc/plugin/__init__.py
+T-3/etc/plugin/backend-client-caucase-updater.py
+T-3/etc/plugin/backend-haproxy-configuration.py
+T-3/etc/plugin/backend_haproxy_http.py
+T-3/etc/plugin/backend_haproxy_https.py
+T-3/etc/plugin/buildout-T-3-status.py
+T-3/etc/plugin/caddy_frontend_ipv4_http.py
+T-3/etc/plugin/caddy_frontend_ipv4_https.py
+T-3/etc/plugin/caddy_frontend_ipv6_http.py
+T-3/etc/plugin/caddy_frontend_ipv6_https.py
+T-3/etc/plugin/caucase-updater.py
+T-3/etc/plugin/check-free-disk-space.py
+T-3/etc/plugin/expose-csr_id-ip-port-listening.py
+T-3/etc/plugin/frontend-caddy-configuration-promise.py
+T-3/etc/plugin/monitor-bootstrap-status.py
+T-3/etc/plugin/monitor-http-frontend.py
+T-3/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-3/etc/plugin/promise-logrotate-setup.py
+T-3/etc/plugin/re6st-connectivity.py
+T-3/etc/plugin/slave-introspection-configuration.py
+T-3/etc/plugin/slave_introspection_https.py
+T-3/etc/plugin/trafficserver-cache-availability.py
+T-3/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_run-CADDY.txt

+T-0/var/run/monitor-httpd.pid
+T-1/var/run/kedifa.pid
+T-1/var/run/monitor-httpd.pid
+T-2/var/run/backend-haproxy-rsyslogd.pid
+T-2/var/run/backend-haproxy.pid
+T-2/var/run/backend_haproxy_configuration_last_state
+T-2/var/run/backend_haproxy_graceful_configuration_state_signature
+T-2/var/run/bhlog.sck
+T-2/var/run/graceful_configuration_state_signature
+T-2/var/run/httpd.pid
+T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature
+T-3/var/run/backend-haproxy.pid
+T-3/var/run/backend_haproxy_configuration_last_state
+T-3/var/run/backend_haproxy_graceful_configuration_state_signature
+T-3/var/run/graceful_configuration_state_signature
+T-3/var/run/slave_introspection_configuration_last_state
+T-3/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
+T-0:aikc-user-caucase-updater-on-watch RUNNING
+T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
+T-0:certificate_authority-{hash-generic}-on-watch RUNNING
+T-0:crond-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-graceful EXITED
+T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
+T-1:bootstrap-monitor EXITED
+T-1:caucase-updater-on-watch RUNNING
+T-1:caucased-{hash-generic}-on-watch RUNNING
+T-1:certificate_authority-{hash-generic}-on-watch RUNNING
+T-1:crond-{hash-generic}-on-watch RUNNING
+T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-1:kedifa-{hash-generic}-on-watch RUNNING
+T-1:kedifa-reloader EXITED
+T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-1:monitor-httpd-graceful EXITED
+T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
+T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
+T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-safe-graceful EXITED
+T-2:bootstrap-monitor EXITED
+T-2:certificate_authority-{hash-generic}-on-watch RUNNING
+T-2:crond-{hash-generic}-on-watch RUNNING
+T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-2:frontend-caddy-safe-graceful EXITED
+T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
+T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
+T-2:trafficserver-{hash-generic}-on-watch RUNNING
+T-2:trafficserver-reload EXITED
+T-3:6tunnel-11080-{hash-generic}-on-watch STOPPED
+T-3:6tunnel-11443-{hash-generic}-on-watch STOPPED
+T-3:backend-client-login-certificate-caucase-updater-on-watch STOPPED
+T-3:backend-haproxy-{hash-generic}-on-watch STOPPED
+T-3:backend-haproxy-rsyslogd-{hash-generic}-on-watch STOPPED
+T-3:backend-haproxy-safe-graceful EXITED
+T-3:bootstrap-monitor EXITED
+T-3:certificate_authority-{hash-generic}-on-watch STOPPED
+T-3:crond-{hash-generic}-on-watch STOPPED
+T-3:expose-csr_id-{hash-generic}-on-watch STOPPED
+T-3:frontend-caddy-safe-graceful EXITED
+T-3:frontend_caddy-{hash-caddy-T-3}-on-watch STOPPED
+T-3:kedifa-login-certificate-caucase-updater-on-watch STOPPED
+T-3:kedifa-updater-{hash-generic}-on-watch STOPPED
+T-3:monitor-httpd-{hash-generic}-on-watch STOPPED
+T-3:monitor-httpd-graceful EXITED
+T-3:slave-instrospection-nginx-{hash-generic}-on-watch STOPPED
+T-3:slave-introspection-safe-graceful EXITED
+T-3:trafficserver-{hash-generic}-on-watch STOPPED
+T-3:trafficserver-reload EXITED