software/caddy-frontend: auto-restart services on SR upgrade.

/reviewed-on nexedi/slapos!413

software/caddy-frontend/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 750e2b1c922bf14511a3bc8a42468b1b
+md5sum = 3bf520c34753cf9ee9e665e9ef7fb469
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -11,7 +11,9 @@ parts =
   cron
   cron-entry-logrotate
   ca-frontend
+  ca-frontend-service
   certificate-authority
+  certificate-authority-service
   logrotate-entry-caddy
   logrotate-entry-nginx
   caddy-frontend
@@ -240,6 +242,7 @@ wrapper-path = ${directory:service}/frontend_caddy
 wait-for-files =
 	       ${ca-frontend:cert-file}
 	       ${ca-frontend:key-file}
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 
 [not-found-html]
 recipe = slapos.cookbook:symbolic.link
@@ -281,12 +284,18 @@ recipe = slapos.cookbook:certificate_authority
 openssl-binary = {{ parameter_dict['openssl'] }}/bin/openssl
 ca-dir = ${directory:ca-dir}
 requests-directory = ${cadirectory:requests}
-wrapper = ${directory:service}/certificate_authority
+wrapper = ${directory:bin}/certificate_authority
 ca-private = ${cadirectory:private}
 ca-certs = ${cadirectory:certs}
 ca-newcerts = ${cadirectory:newcerts}
 ca-crl = ${cadirectory:crl}
 
+[certificate-authority-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${certificate-authority:wrapper}
+wrapper-path = ${directory:service}/certificate_authority
+hash-files = ${buildout:directory}/software_release/buildout.cfg
+
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
 requests = ${directory:ca-dir}/requests/
@@ -301,12 +310,18 @@ recipe = slapos.cookbook:certificate_authority.request
 key-file = ${cadirectory:certs}/frontend.key
 cert-file = ${cadirectory:certs}/frontend.crt
 executable = ${directory:service}/frontend_caddy
-wrapper = ${directory:service}/frontend_caddy
+wrapper = ${directory:bin}/frontend_caddy
 key-content = ${configuration:apache-key}
 cert-content = ${configuration:apache-certificate}
 # Put domain name
 name = ${configuration:domain}
 
+[ca-frontend-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${ca-frontend:wrapper}
+wrapper-path = ${directory:service}/frontend_caddy
+hash-files = ${buildout:directory}/software_release/buildout.cfg
+
 [ca-custom-frontend]
 < = jinja2-template-base
 template = {{ parameter_dict['template_empty'] }}
@@ -411,6 +426,7 @@ recipe = slapos.cookbook:wrapper
 command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_cop
 wrapper-path = ${trafficserver-variable:wrapper-path}
 environment = TS_ROOT=${buildout:directory}
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 
 [trafficserver-reload]
 recipe = slapos.cookbook:wrapper
@@ -644,6 +660,7 @@ wrapper-path = ${directory:bin}/nginx-wrapper
 recipe = slapos.cookbook:wrapper
 command-line = ${nginx-wrapper:wrapper-path} -pidfile ${nginx-configuration:pid-file}
 wrapper-path = ${directory:service}/frontend_nginx
+hash-files = ${buildout:directory}/software_release/buildout.cfg
 
 [dynamic-nginx-frontend-template]
 < = jinja2-template-base

software/caddy-frontend/test/test.py

@@ -97,12 +97,24 @@ def isHTTP2(domain, ip):
 
 
 class TestDataMixin(object):
+  @staticmethod
+  def generateHashFromFiles(file_list):
+    import hashlib
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
   def getTrimmedProcessInfo(self):
     return '\n'.join(sorted([
       '%(group)s:%(name)s %(statename)s' % q for q
       in self.getSupervisorRPCServer().supervisor.getAllProcessInfo()]))
 
-  def assertTestData(self, runtime_data):
+  def assertTestData(self, runtime_data, **kw):
     filename = '%s-%s.txt' % (self.id(), 'CADDY')
     test_data_file = os.path.join(
       os.path.dirname(os.path.realpath(__file__)), 'test_data', filename)
@@ -112,6 +124,9 @@ class TestDataMixin(object):
     except IOError:
       test_data = ''
 
+    if len(kw) > 0:
+      test_data = test_data.format(**kw)
+
     maxDiff = self.maxDiff
     self.maxDiff = None
     try:
@@ -185,8 +200,16 @@ class TestDataMixin(object):
   def test_supervisor_state(self):
     # give a chance for etc/run scripts to finish
     time.sleep(1)
+
+    hash_files = [
+      'software_release/buildout.cfg',
+    ]
+    hash_files = [os.path.join(self.computer_partition_root_path, path)
+                  for path in hash_files]
+    h = self.generateHashFromFiles(hash_files)
+
     runtime_data = self.getTrimmedProcessInfo()
-    self.assertTestData(runtime_data)
+    self.assertTestData(runtime_data, hash=h)
 
 
 class HttpFrontendTestCase(SlapOSInstanceTestCase):
@@ -3387,3 +3410,55 @@ class TestDuplicateSiteKeyProtection(SlaveHttpFrontendTestCase, TestDataMixin):
         '\'duplicate.example.com\' clashes"]'
       }
     )
+
+class AutoRestartTestCase(SlaveHttpFrontendTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      '-frontend-1-state': 'stopped',
+    }
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    return {
+      'test': {
+        'url': cls.backend_url,
+      },
+    }
+
+  @staticmethod
+  def generateHashFromFiles(file_list):
+    import hashlib
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
+  def test_hashes(self):
+    hash_files = [
+      'software_release/buildout.cfg',
+    ]
+    expected_process_names = [
+      'frontend_caddy-{hash}-on-watch',
+      'frontend_nginx-{hash}-on-watch',
+      'trafficserver-{hash}-on-watch',
+      'certificate_authority-{hash}-on-watch',
+      'crond-{hash}',
+    ]
+
+    supervisor = self.getSupervisorRPCServer().supervisor
+    process_names = [process['name']
+                     for process in supervisor.getAllProcessInfo()]
+
+    hash_files = [os.path.join(self.computer_partition_root_path, path)
+                  for path in hash_files]
+
+    for name in expected_process_names:
+      h = self.generateHashFromFiles(hash_files)
+      expected_process_name = name.format(hash=h)
+
+      self.assertIn(expected_process_name, process_names)

software/caddy-frontend/test/test_data/test.TestDefaultMonitorHttpdPort.test_supervisor_state-CADDY.txt

 TestDefaultMonitorHttpdPort-0:bootstrap-monitor EXITED
 TestDefaultMonitorHttpdPort-0:certificate_authority-on-watch RUNNING
-TestDefaultMonitorHttpdPort-0:crond RUNNING
+TestDefaultMonitorHttpdPort-0:crond-{hash} RUNNING
+TestDefaultMonitorHttpdPort-0:monitor-httpd-{hash}-on-watch EXITED
 TestDefaultMonitorHttpdPort-0:monitor-httpd-graceful EXITED
-TestDefaultMonitorHttpdPort-0:monitor-httpd-on-watch EXITED
 TestDefaultMonitorHttpdPort-1:6tunnel-26011-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:6tunnel-26012-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:6tunnel-4443-on-watch STOPPED
@@ -10,14 +10,15 @@ TestDefaultMonitorHttpdPort-1:6tunnel-8080-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:6tunnel-8081-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:6tunnel-9443-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:bootstrap-monitor STOPPED
-TestDefaultMonitorHttpdPort-1:certificate_authority-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:certificate_authority-{hash}-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:crond-{hash} STOPPED
 TestDefaultMonitorHttpdPort-1:crond-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:frontend-caddy-safe-graceful STOPPED
 TestDefaultMonitorHttpdPort-1:frontend-nginx-safe-graceful STOPPED
-TestDefaultMonitorHttpdPort-1:frontend_caddy-on-watch STOPPED
-TestDefaultMonitorHttpdPort-1:frontend_nginx-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:frontend_caddy-{hash}-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:frontend_nginx-{hash}-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:monitor-httpd-{hash}-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:monitor-httpd-graceful STOPPED
-TestDefaultMonitorHttpdPort-1:monitor-httpd-on-watch STOPPED
-TestDefaultMonitorHttpdPort-1:trafficserver-on-watch STOPPED
+TestDefaultMonitorHttpdPort-1:trafficserver-{hash}-on-watch STOPPED
 TestDefaultMonitorHttpdPort-1:trafficserver-reload STOPPED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestDuplicateSiteKeyProtection.test_supervisor_state-CADDY.txt

 TestDuplicateSiteKeyProtection-0:bootstrap-monitor EXITED
 TestDuplicateSiteKeyProtection-0:certificate_authority-on-watch RUNNING
-TestDuplicateSiteKeyProtection-0:crond RUNNING
+TestDuplicateSiteKeyProtection-0:crond-{hash} RUNNING
+TestDuplicateSiteKeyProtection-0:monitor-httpd-{hash}-on-watch RUNNING
 TestDuplicateSiteKeyProtection-0:monitor-httpd-graceful EXITED
-TestDuplicateSiteKeyProtection-0:monitor-httpd-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:6tunnel-11080-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:6tunnel-11443-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestDuplicateSiteKeyProtection-1:6tunnel-12443-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:6tunnel-26011-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:6tunnel-26012-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:bootstrap-monitor EXITED
-TestDuplicateSiteKeyProtection-1:certificate_authority-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:certificate_authority-{hash}-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:crond-{hash} RUNNING
 TestDuplicateSiteKeyProtection-1:crond-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:frontend-caddy-safe-graceful EXITED
 TestDuplicateSiteKeyProtection-1:frontend-nginx-safe-graceful EXITED
-TestDuplicateSiteKeyProtection-1:frontend_caddy-on-watch RUNNING
-TestDuplicateSiteKeyProtection-1:frontend_nginx-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:frontend_caddy-{hash}-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:frontend_nginx-{hash}-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:monitor-httpd-{hash}-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:monitor-httpd-graceful EXITED
-TestDuplicateSiteKeyProtection-1:monitor-httpd-on-watch RUNNING
-TestDuplicateSiteKeyProtection-1:trafficserver-on-watch RUNNING
+TestDuplicateSiteKeyProtection-1:trafficserver-{hash}-on-watch RUNNING
 TestDuplicateSiteKeyProtection-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_supervisor_state-CADDY.txt

 TestEnableHttp2ByDefaultDefaultSlave-0:bootstrap-monitor EXITED
 TestEnableHttp2ByDefaultDefaultSlave-0:certificate_authority-on-watch RUNNING
-TestEnableHttp2ByDefaultDefaultSlave-0:crond RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-0:crond-{hash} RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-0:monitor-httpd-graceful EXITED
-TestEnableHttp2ByDefaultDefaultSlave-0:monitor-httpd-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-11080-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-11443-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-12443-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-26011-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:6tunnel-26012-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:bootstrap-monitor EXITED
-TestEnableHttp2ByDefaultDefaultSlave-1:certificate_authority-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:crond-{hash} RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:crond-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:frontend-caddy-safe-graceful EXITED
 TestEnableHttp2ByDefaultDefaultSlave-1:frontend-nginx-safe-graceful EXITED
-TestEnableHttp2ByDefaultDefaultSlave-1:frontend_caddy-on-watch RUNNING
-TestEnableHttp2ByDefaultDefaultSlave-1:frontend_nginx-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:monitor-httpd-graceful EXITED
-TestEnableHttp2ByDefaultDefaultSlave-1:monitor-httpd-on-watch RUNNING
-TestEnableHttp2ByDefaultDefaultSlave-1:trafficserver-on-watch RUNNING
+TestEnableHttp2ByDefaultDefaultSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultDefaultSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_supervisor_state-CADDY.txt

 TestEnableHttp2ByDefaultFalseSlave-0:bootstrap-monitor EXITED
 TestEnableHttp2ByDefaultFalseSlave-0:certificate_authority-on-watch RUNNING
-TestEnableHttp2ByDefaultFalseSlave-0:crond RUNNING
+TestEnableHttp2ByDefaultFalseSlave-0:crond-{hash} RUNNING
+TestEnableHttp2ByDefaultFalseSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-0:monitor-httpd-graceful EXITED
-TestEnableHttp2ByDefaultFalseSlave-0:monitor-httpd-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-11080-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-11443-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-12443-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-26011-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:6tunnel-26012-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:bootstrap-monitor EXITED
-TestEnableHttp2ByDefaultFalseSlave-1:certificate_authority-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:crond-{hash} RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:crond-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:frontend-caddy-safe-graceful EXITED
 TestEnableHttp2ByDefaultFalseSlave-1:frontend-nginx-safe-graceful EXITED
-TestEnableHttp2ByDefaultFalseSlave-1:frontend_caddy-on-watch RUNNING
-TestEnableHttp2ByDefaultFalseSlave-1:frontend_nginx-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:monitor-httpd-graceful EXITED
-TestEnableHttp2ByDefaultFalseSlave-1:monitor-httpd-on-watch RUNNING
-TestEnableHttp2ByDefaultFalseSlave-1:trafficserver-on-watch RUNNING
+TestEnableHttp2ByDefaultFalseSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestEnableHttp2ByDefaultFalseSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestMalformedBackenUrlSlave.test_supervisor_state-CADDY.txt

 TestMalformedBackenUrlSlave-0:bootstrap-monitor EXITED
 TestMalformedBackenUrlSlave-0:certificate_authority-on-watch RUNNING
-TestMalformedBackenUrlSlave-0:crond RUNNING
+TestMalformedBackenUrlSlave-0:crond-{hash} RUNNING
+TestMalformedBackenUrlSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestMalformedBackenUrlSlave-0:monitor-httpd-graceful EXITED
-TestMalformedBackenUrlSlave-0:monitor-httpd-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:6tunnel-11080-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:6tunnel-11443-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestMalformedBackenUrlSlave-1:6tunnel-12443-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:6tunnel-26011-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:6tunnel-26012-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:bootstrap-monitor EXITED
-TestMalformedBackenUrlSlave-1:certificate_authority-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:crond-{hash} RUNNING
 TestMalformedBackenUrlSlave-1:crond-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:frontend-caddy-safe-graceful EXITED
 TestMalformedBackenUrlSlave-1:frontend-nginx-safe-graceful EXITED
-TestMalformedBackenUrlSlave-1:frontend_caddy-on-watch RUNNING
-TestMalformedBackenUrlSlave-1:frontend_nginx-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:monitor-httpd-graceful EXITED
-TestMalformedBackenUrlSlave-1:monitor-httpd-on-watch RUNNING
-TestMalformedBackenUrlSlave-1:trafficserver-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestMalformedBackenUrlSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_supervisor_state-CADDY.txt

 TestMasterRequest-0:bootstrap-monitor EXITED
 TestMasterRequest-0:certificate_authority-on-watch RUNNING
-TestMasterRequest-0:crond RUNNING
+TestMasterRequest-0:crond-{hash} RUNNING
+TestMasterRequest-0:monitor-httpd-{hash}-on-watch RUNNING
 TestMasterRequest-0:monitor-httpd-graceful EXITED
-TestMasterRequest-0:monitor-httpd-on-watch RUNNING
 watchdog:watchdog RUNNING
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_supervisor_state-CADDY.txt

 TestMasterRequestDomain-0:bootstrap-monitor EXITED
 TestMasterRequestDomain-0:certificate_authority-on-watch RUNNING
-TestMasterRequestDomain-0:crond RUNNING
+TestMasterRequestDomain-0:crond-{hash} RUNNING
+TestMasterRequestDomain-0:monitor-httpd-{hash}-on-watch RUNNING
 TestMasterRequestDomain-0:monitor-httpd-graceful EXITED
-TestMasterRequestDomain-0:monitor-httpd-on-watch RUNNING
 watchdog:watchdog RUNNING
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestQuicEnabled.test_supervisor_state-CADDY.txt

 TestQuicEnabled-0:bootstrap-monitor EXITED
 TestQuicEnabled-0:certificate_authority-on-watch RUNNING
-TestQuicEnabled-0:crond RUNNING
+TestQuicEnabled-0:crond-{hash} RUNNING
+TestQuicEnabled-0:monitor-httpd-{hash}-on-watch RUNNING
 TestQuicEnabled-0:monitor-httpd-graceful EXITED
-TestQuicEnabled-0:monitor-httpd-on-watch RUNNING
 TestQuicEnabled-1:6tunnel-11080-on-watch RUNNING
 TestQuicEnabled-1:6tunnel-11443-on-watch RUNNING
 TestQuicEnabled-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestQuicEnabled-1:6tunnel-12443-on-watch RUNNING
 TestQuicEnabled-1:6tunnel-26011-on-watch RUNNING
 TestQuicEnabled-1:6tunnel-26012-on-watch RUNNING
 TestQuicEnabled-1:bootstrap-monitor EXITED
-TestQuicEnabled-1:certificate_authority-on-watch RUNNING
+TestQuicEnabled-1:certificate_authority-{hash}-on-watch RUNNING
+TestQuicEnabled-1:crond-{hash} RUNNING
 TestQuicEnabled-1:crond-on-watch RUNNING
 TestQuicEnabled-1:frontend-caddy-safe-graceful EXITED
 TestQuicEnabled-1:frontend-nginx-safe-graceful EXITED
-TestQuicEnabled-1:frontend_caddy-on-watch RUNNING
-TestQuicEnabled-1:frontend_nginx-on-watch RUNNING
+TestQuicEnabled-1:frontend_caddy-{hash}-on-watch RUNNING
+TestQuicEnabled-1:frontend_nginx-{hash}-on-watch RUNNING
+TestQuicEnabled-1:monitor-httpd-{hash}-on-watch RUNNING
 TestQuicEnabled-1:monitor-httpd-graceful EXITED
-TestQuicEnabled-1:monitor-httpd-on-watch RUNNING
-TestQuicEnabled-1:trafficserver-on-watch RUNNING
+TestQuicEnabled-1:trafficserver-{hash}-on-watch RUNNING
 TestQuicEnabled-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_supervisor_state-CADDY.txt

 TestRe6stVerificationUrlDefaultSlave-0:bootstrap-monitor EXITED
 TestRe6stVerificationUrlDefaultSlave-0:certificate_authority-on-watch RUNNING
-TestRe6stVerificationUrlDefaultSlave-0:crond RUNNING
+TestRe6stVerificationUrlDefaultSlave-0:crond-{hash} RUNNING
+TestRe6stVerificationUrlDefaultSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-0:monitor-httpd-graceful EXITED
-TestRe6stVerificationUrlDefaultSlave-0:monitor-httpd-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:6tunnel-11080-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:6tunnel-11443-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestRe6stVerificationUrlDefaultSlave-1:6tunnel-12443-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:6tunnel-26011-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:6tunnel-26012-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:bootstrap-monitor EXITED
-TestRe6stVerificationUrlDefaultSlave-1:certificate_authority-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:crond-{hash} RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:crond-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:frontend-caddy-safe-graceful EXITED
 TestRe6stVerificationUrlDefaultSlave-1:frontend-nginx-safe-graceful EXITED
-TestRe6stVerificationUrlDefaultSlave-1:frontend_caddy-on-watch RUNNING
-TestRe6stVerificationUrlDefaultSlave-1:frontend_nginx-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:monitor-httpd-graceful EXITED
-TestRe6stVerificationUrlDefaultSlave-1:monitor-httpd-on-watch RUNNING
-TestRe6stVerificationUrlDefaultSlave-1:trafficserver-on-watch RUNNING
+TestRe6stVerificationUrlDefaultSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlDefaultSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_supervisor_state-CADDY.txt

 TestRe6stVerificationUrlSlave-0:bootstrap-monitor EXITED
 TestRe6stVerificationUrlSlave-0:certificate_authority-on-watch RUNNING
-TestRe6stVerificationUrlSlave-0:crond RUNNING
+TestRe6stVerificationUrlSlave-0:crond-{hash} RUNNING
+TestRe6stVerificationUrlSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlSlave-0:monitor-httpd-graceful EXITED
-TestRe6stVerificationUrlSlave-0:monitor-httpd-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:6tunnel-11080-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:6tunnel-11443-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestRe6stVerificationUrlSlave-1:6tunnel-12443-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:6tunnel-26011-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:6tunnel-26012-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:bootstrap-monitor EXITED
-TestRe6stVerificationUrlSlave-1:certificate_authority-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:crond-{hash} RUNNING
 TestRe6stVerificationUrlSlave-1:crond-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:frontend-caddy-safe-graceful EXITED
 TestRe6stVerificationUrlSlave-1:frontend-nginx-safe-graceful EXITED
-TestRe6stVerificationUrlSlave-1:frontend_caddy-on-watch RUNNING
-TestRe6stVerificationUrlSlave-1:frontend_nginx-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:monitor-httpd-graceful EXITED
-TestRe6stVerificationUrlSlave-1:monitor-httpd-on-watch RUNNING
-TestRe6stVerificationUrlSlave-1:trafficserver-on-watch RUNNING
+TestRe6stVerificationUrlSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestRe6stVerificationUrlSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_supervisor_state-CADDY.txt

 TestReplicateSlave-0:bootstrap-monitor EXITED
 TestReplicateSlave-0:certificate_authority-on-watch RUNNING
-TestReplicateSlave-0:crond RUNNING
+TestReplicateSlave-0:crond-{hash} RUNNING
+TestReplicateSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestReplicateSlave-0:monitor-httpd-graceful EXITED
-TestReplicateSlave-0:monitor-httpd-on-watch RUNNING
 TestReplicateSlave-1:6tunnel-11080-on-watch RUNNING
 TestReplicateSlave-1:6tunnel-11443-on-watch RUNNING
 TestReplicateSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,15 +10,16 @@ TestReplicateSlave-1:6tunnel-12443-on-watch RUNNING
 TestReplicateSlave-1:6tunnel-26011-on-watch RUNNING
 TestReplicateSlave-1:6tunnel-26012-on-watch RUNNING
 TestReplicateSlave-1:bootstrap-monitor EXITED
-TestReplicateSlave-1:certificate_authority-on-watch RUNNING
+TestReplicateSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestReplicateSlave-1:crond-{hash} RUNNING
 TestReplicateSlave-1:crond-on-watch RUNNING
 TestReplicateSlave-1:frontend-caddy-safe-graceful EXITED
 TestReplicateSlave-1:frontend-nginx-safe-graceful EXITED
-TestReplicateSlave-1:frontend_caddy-on-watch RUNNING
-TestReplicateSlave-1:frontend_nginx-on-watch RUNNING
+TestReplicateSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestReplicateSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestReplicateSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestReplicateSlave-1:monitor-httpd-graceful EXITED
-TestReplicateSlave-1:monitor-httpd-on-watch RUNNING
-TestReplicateSlave-1:trafficserver-on-watch RUNNING
+TestReplicateSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestReplicateSlave-1:trafficserver-reload EXITED
 TestReplicateSlave-2:6tunnel-11080-on-watch STOPPED
 TestReplicateSlave-2:6tunnel-11443-on-watch STOPPED
@@ -27,14 +28,15 @@ TestReplicateSlave-2:6tunnel-12443-on-watch STOPPED
 TestReplicateSlave-2:6tunnel-26011-on-watch STOPPED
 TestReplicateSlave-2:6tunnel-26012-on-watch STOPPED
 TestReplicateSlave-2:bootstrap-monitor STOPPED
-TestReplicateSlave-2:certificate_authority-on-watch STOPPED
+TestReplicateSlave-2:certificate_authority-{hash}-on-watch STOPPED
+TestReplicateSlave-2:crond-{hash} STOPPED
 TestReplicateSlave-2:crond-on-watch STOPPED
 TestReplicateSlave-2:frontend-caddy-safe-graceful STOPPED
 TestReplicateSlave-2:frontend-nginx-safe-graceful STOPPED
-TestReplicateSlave-2:frontend_caddy-on-watch STOPPED
-TestReplicateSlave-2:frontend_nginx-on-watch STOPPED
+TestReplicateSlave-2:frontend_caddy-{hash}-on-watch STOPPED
+TestReplicateSlave-2:frontend_nginx-{hash}-on-watch STOPPED
+TestReplicateSlave-2:monitor-httpd-{hash}-on-watch STOPPED
 TestReplicateSlave-2:monitor-httpd-graceful STOPPED
-TestReplicateSlave-2:monitor-httpd-on-watch STOPPED
-TestReplicateSlave-2:trafficserver-on-watch STOPPED
+TestReplicateSlave-2:trafficserver-{hash}-on-watch STOPPED
 TestReplicateSlave-2:trafficserver-reload STOPPED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestSlave.test_supervisor_state-CADDY.txt

 TestSlave-0:bootstrap-monitor EXITED
 TestSlave-0:certificate_authority-on-watch RUNNING
-TestSlave-0:crond RUNNING
+TestSlave-0:crond-{hash} RUNNING
+TestSlave-0:monitor-httpd-{hash}-on-watch RUNNING
 TestSlave-0:monitor-httpd-graceful EXITED
-TestSlave-0:monitor-httpd-on-watch RUNNING
 TestSlave-1:6tunnel-11080-on-watch RUNNING
 TestSlave-1:6tunnel-11443-on-watch RUNNING
 TestSlave-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestSlave-1:6tunnel-12443-on-watch RUNNING
 TestSlave-1:6tunnel-26011-on-watch RUNNING
 TestSlave-1:6tunnel-26012-on-watch RUNNING
 TestSlave-1:bootstrap-monitor EXITED
-TestSlave-1:certificate_authority-on-watch RUNNING
+TestSlave-1:certificate_authority-{hash}-on-watch RUNNING
+TestSlave-1:crond-{hash} RUNNING
 TestSlave-1:crond-on-watch RUNNING
 TestSlave-1:frontend-caddy-safe-graceful EXITED
 TestSlave-1:frontend-nginx-safe-graceful EXITED
-TestSlave-1:frontend_caddy-on-watch RUNNING
-TestSlave-1:frontend_nginx-on-watch RUNNING
+TestSlave-1:frontend_caddy-{hash}-on-watch RUNNING
+TestSlave-1:frontend_nginx-{hash}-on-watch RUNNING
+TestSlave-1:monitor-httpd-{hash}-on-watch RUNNING
 TestSlave-1:monitor-httpd-graceful EXITED
-TestSlave-1:monitor-httpd-on-watch RUNNING
-TestSlave-1:trafficserver-on-watch RUNNING
+TestSlave-1:trafficserver-{hash}-on-watch RUNNING
 TestSlave-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING

software/caddy-frontend/test/test_data/test.TestSlaveBadParameters.test_supervisor_state-CADDY.txt

 TestSlaveBadParameters-0:bootstrap-monitor EXITED
 TestSlaveBadParameters-0:certificate_authority-on-watch RUNNING
-TestSlaveBadParameters-0:crond RUNNING
+TestSlaveBadParameters-0:crond-{hash} RUNNING
+TestSlaveBadParameters-0:monitor-httpd-{hash}-on-watch RUNNING
 TestSlaveBadParameters-0:monitor-httpd-graceful EXITED
-TestSlaveBadParameters-0:monitor-httpd-on-watch RUNNING
 TestSlaveBadParameters-1:6tunnel-11080-on-watch RUNNING
 TestSlaveBadParameters-1:6tunnel-11443-on-watch RUNNING
 TestSlaveBadParameters-1:6tunnel-12080-on-watch RUNNING
@@ -10,14 +10,15 @@ TestSlaveBadParameters-1:6tunnel-12443-on-watch RUNNING
 TestSlaveBadParameters-1:6tunnel-26011-on-watch RUNNING
 TestSlaveBadParameters-1:6tunnel-26012-on-watch RUNNING
 TestSlaveBadParameters-1:bootstrap-monitor EXITED
-TestSlaveBadParameters-1:certificate_authority-on-watch RUNNING
+TestSlaveBadParameters-1:certificate_authority-{hash}-on-watch RUNNING
+TestSlaveBadParameters-1:crond-{hash} RUNNING
 TestSlaveBadParameters-1:crond-on-watch RUNNING
 TestSlaveBadParameters-1:frontend-caddy-safe-graceful EXITED
 TestSlaveBadParameters-1:frontend-nginx-safe-graceful EXITED
-TestSlaveBadParameters-1:frontend_caddy-on-watch RUNNING
-TestSlaveBadParameters-1:frontend_nginx-on-watch RUNNING
+TestSlaveBadParameters-1:frontend_caddy-{hash}-on-watch RUNNING
+TestSlaveBadParameters-1:frontend_nginx-{hash}-on-watch RUNNING
+TestSlaveBadParameters-1:monitor-httpd-{hash}-on-watch RUNNING
 TestSlaveBadParameters-1:monitor-httpd-graceful EXITED
-TestSlaveBadParameters-1:monitor-httpd-on-watch RUNNING
-TestSlaveBadParameters-1:trafficserver-on-watch RUNNING
+TestSlaveBadParameters-1:trafficserver-{hash}-on-watch RUNNING
 TestSlaveBadParameters-1:trafficserver-reload EXITED
 watchdog:watchdog RUNNING