caddy-frontend: Pick up kedifa with async updater

Instead of fetching certificates on each slapos node instance use new
kedifa-updater, which is a tool to asynchronously fetch certificates and
has a hook to reload the server in case if new certificate is available.

custom_ssl_directory is NOT BBB

software/caddy-frontend/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 55f607a1cc8059db76b3c9057435a5ab
+md5sum = 0e96242cac04534435b07e27bd9d5096
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,7 +30,7 @@ md5sum = 0f5af15a0cc024ff181c15e946d92808
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 016094d0a251092bf12659cb992d693d
+md5sum = 121e46e105f350c2995d5188cf340ad1
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -42,7 +42,7 @@ md5sum = 38e9994be01ea1b8a379f8ff7aa05438
 
 [template-caddy-frontend-configuration]
 filename = templates/Caddyfile.in
-md5sum = e3adb1b48862e57de160b167ad1402b8
+md5sum = 9bd48b38abc48825388dd6ec6ab19b9a
 
 [caddy-backend-url-validator]
 filename = templates/caddy-backend-url-validator.in
@@ -58,7 +58,7 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = 4308b63820d3682511ce54040d1ae60e
+md5sum = 1d3c7dc1c0d6261c6fd4cc9d756eb19a
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in
@@ -90,7 +90,7 @@ md5sum = cd6bb9bd0734f17469b0ca88f8b1a531
 
 [template-nginx-configuration]
 filename = templates/nginx.cfg.in
-md5sum = 30f30ef3539fe6b7ab99162ae8e71a87
+md5sum = b3a971c3700cb5175f6c1e388cb88775
 
 [template-nginx-eventsource-slave-virtualhost]
 filename = templates/nginx-eventsource-slave.conf.in
@@ -98,7 +98,7 @@ md5sum = 217a6c801b8330b0b825f7b8b4c77184
 
 [template-nginx-notebook-slave-virtualhost]
 filename = templates/nginx-notebook-slave.conf.in
-md5sum = ac17212a53be2c08ab84682ec665148d
+md5sum = b864b17a304f0fad6d8dcb0f1893145b
 
 [template-caddy-lazy-script-call]
 filename = templates/apache-lazy-script-call.sh.in

software/caddy-frontend/common.cfg

@@ -35,7 +35,7 @@ parts +=
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/kedifa.git
 git-executable = ${git:location}/bin/git
-revision = 67bd60ea1bfb4fc6aafdfe4fa204f725731f20cf
+revision = 73a14b0e88afe7512f2fefe6ee9e0000fa523d5d
 
 [kedifa-develop]
 recipe = zc.recipe.egg:develop
@@ -111,7 +111,7 @@ openssl_cnf = ${openssl:location}/etc/ssl/openssl.cnf
 trafficserver = ${trafficserver7:location}
 sha256sum = ${coreutils:location}/bin/sha256sum
 kedifa = ${:bin_directory}/kedifa
-kedifa-getter = ${:bin_directory}/kedifa-getter
+kedifa-updater = ${:bin_directory}/kedifa-updater
 kedifa-csr = ${:bin_directory}/kedifa-csr
 
 monitor_template = ${monitor-template:output}

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -210,7 +210,9 @@ bin_directory = {{ parameter_dict['bin_directory'] }}
 caddy_executable = {{ parameter_dict['caddy'] }}
 caucase_url = {{ slapparameter_dict['kedifa-caucase-url'] }}
 sixtunnel_executable = {{ parameter_dict['sixtunnel'] }}/bin/6tunnel
-kedifa-getter = {{ parameter_dict['kedifa-getter'] }}
+kedifa-updater = {{ parameter_dict['kedifa-updater'] }}
+kedifa-updater-mapping-file = ${directory:etc}/kedifa_updater_mapping.txt
+kedifa-updater-state-file = ${directory:srv}/kedifa_updater_state.json
 kedifa-csr = {{ parameter_dict['kedifa-csr'] }}
 service_directory = ${directory:service}
 extra-context =
@@ -220,7 +222,9 @@ extra-context =
     key nginx_configuration_directory caddy-directory:nginx-slave-configuration
     key caddy_cached_configuration_directory caddy-directory:slave-with-cache-configuration
     key slave_with_cache_configuration_directory caddy-directory:slave-with-cache-configuration
-    key kedifa_getter :kedifa-getter
+    key kedifa_updater :kedifa-updater
+    key kedifa_updater_mapping_file :kedifa-updater-mapping-file
+    key kedifa_updater_state_file :kedifa-updater-state-file
     key kedifa_csr :kedifa-csr
     key caddy_executable :caddy_executable
     key caucase_url :caucase_url
@@ -257,6 +261,8 @@ extra-context =
     key template_notebook_slave_configuration software-release-path:template-nginx-notebook-slave-virtualhost
     key software_type :software_type
     key frontend_lazy_graceful_reload frontend-caddy-lazy-graceful:rendered
+    key frontend_graceful_reload caddy-configuration:frontend-graceful-command
+    key nginx_graceful_reload nginx-configuration:nginx-graceful-command
     section frontend_configuration frontend-configuration
     section caddy_configuration caddy-configuration
     section nginx_configuration nginx-configuration
@@ -275,10 +281,10 @@ extra-context =
     key service_directory directory:service
     key run_directory directory:etc-run
     key not_found_file caddy-configuration:not-found-file
-# BBB: SlapOS Master non-zero knowledge BEGIN
     key custom_ssl_directory caddy-directory:custom-ssl-directory
+# BBB: SlapOS Master non-zero knowledge BEGIN
+    key bbb_ssl_directory directory:bbb-ssl-dir
     key apache_certificate apache-certificate:rendered
-    key apache_key apache-key:rendered
 # BBB: SlapOS Master non-zero knowledge END
 
 [dynamic-virtualhost-template-slave]
@@ -321,7 +327,6 @@ extra-context =
     key password monitor-htpasswd:passwd
 # BBB: SlapOS Master non-zero knowledge BEGIN
     key apache_certificate apache-certificate:rendered
-    key apache_key apache-key:rendered
 # BBB: SlapOS Master non-zero knowledge END
 
 [caddy-wrapper]
@@ -367,9 +372,7 @@ slave-log = ${directory:log}/httpd
 nginx-slave-configuration = ${directory:etc}/nginx-slave-conf.d/
 autocert = ${directory:srv}/autocert
 master-autocert-dir = ${:autocert}/master-autocert
-# BBB: SlapOS Master non-zero knowledge BEGIN
 custom-ssl-directory = ${:slave-configuration}/ssl
-# BBB: SlapOS Master non-zero knowledge END
 
 [caddy-configuration]
 frontend-configuration = ${directory:etc}/Caddyfile
@@ -391,21 +394,17 @@ command-line = ${frontend-caddy-validate:rendered}
 wrapper-path = ${directory:bin}/caddy-configtest
 
 # BBB: SlapOS Master non-zero knowledge BEGIN
-[apache-key]
-< = jinja2-template-base
-template = {{ parameter_dict['template_empty'] }}
-rendered = ${directory:bbb-ssl-dir}/frontend.key
-content = ${configuration:apache-key}
-extra-context =
-    key content :content
-
 [apache-certificate]
-< = jinja2-template-base
-template = {{ parameter_dict['template_empty'] }}
+recipe = slapos.recipe.template:jinja2
+template = inline:
+{% raw %}
+  {{ certificate }}
+  {{ key }}
+{% endraw %}
+context =
+  key certificate configuration:apache-certificate
+  key key configuration:apache-key
 rendered = ${directory:bbb-ssl-dir}/frontend.crt
-content = ${configuration:apache-certificate}
-extra-context =
-    key content :content
 # BBB: SlapOS Master non-zero knowledge END
 
 [logrotate-entry-caddy]
@@ -555,7 +554,7 @@ template = {{ parameter_dict['template_graceful_script'] }}
 rendered = ${directory:etc-run}/frontend-caddy-safe-graceful
 mode = 0700
 
-path_list = ${caddy-configuration:frontend-configuration} ${frontend-configuration:log-access-configuration} ${caddy-directory:slave-configuration}/*.conf ${caddy-directory:slave-with-cache-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*/*.pem ${caddy-directory:custom-ssl-directory}/*.key ${caddy-directory:custom-ssl-directory}/*.crt ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.key ${directory:bbb-ssl-dir}/*.crt
+path_list = ${caddy-configuration:frontend-configuration} ${frontend-configuration:log-access-configuration} ${caddy-directory:slave-configuration}/*.conf ${caddy-directory:slave-with-cache-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*.pem ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.crt
 sha256sum = {{ parameter_dict['sha256sum'] }}
 signature_file = ${directory:run}/caddy_graceful_signature
 extra-context =
@@ -569,7 +568,7 @@ extra-context =
 template = {{ parameter_dict['template_graceful_script'] }}
 rendered = ${directory:etc-run}/frontend-nginx-safe-graceful
 mode = 0700
-path_list = ${dynamic-nginx-frontend-template:rendered} ${caddy-directory:nginx-slave-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*/*.pem ${caddy-directory:custom-ssl-directory}/*.key ${caddy-directory:custom-ssl-directory}/*.crt ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.key ${directory:bbb-ssl-dir}/*.crt
+path_list = ${dynamic-nginx-frontend-template:rendered} ${caddy-directory:nginx-slave-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*.pem ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.crt
 sha256sum = {{ parameter_dict['sha256sum'] }}
 signature_file = ${directory:run}/nginx_graceful_signature
 extra-context =
@@ -772,7 +771,6 @@ extra-context =
   key master_certificate caddy-configuration:master-certificate
 # BBB: SlapOS Master non-zero knowledge BEGIN
   key apache_certificate apache-certificate:rendered
-  key apache_key apache-key:rendered
 # BBB: SlapOS Master non-zero knowledge END
 
 [nginx-configuration]

software/caddy-frontend/software.cfg

@@ -2,6 +2,8 @@
 extends = common.cfg
 
 [versions]
+# Modern KeDiFa requires zc.lockfile
+zc.lockfile = 1.4
 # Versions pinned for kedifa need urllib3 >= 1.18
 urllib3 = 1.24
 requests = 2.20.0

software/caddy-frontend/templates/Caddyfile.in

@@ -9,9 +9,9 @@ import {{ slave_with_cache_configuration_directory }}/*.conf
 {%-   do ssl.__setitem__('certificate', master_certificate) -%}
 {%-   do ssl.__setitem__('key', master_certificate) -%}
 {#- BBB: SlapOS Master non-zero knowledge BEGIN -#}
-{%- elif os_module.path.getsize(apache_certificate) > 0 and os_module.path.getsize(apache_key) > 0 -%}
+{%- elif os_module.path.getsize(apache_certificate) > 0 -%}
 {%-   do ssl.__setitem__('certificate', apache_certificate) -%}
-{%-   do ssl.__setitem__('key', apache_key) -%}
+{%-   do ssl.__setitem__('key', apache_certificate) -%}
 {%- endif -%}
 {#- BBB: SlapOS Master non-zero knowledge END #}
 # Catch-all and 404 for not configured instances

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

 {% if software_type == slap_software_type %}
 
+{% set kedifa_updater_mapping = [] %}
 {% set cached_server_dict = {} %}
 {% set part_list = [] %}
 {% set cache_port = caddy_configuration.get('cache-port') %}
@@ -32,12 +33,7 @@ notifempty = true
 create = true
 
 {% if master_key_download_url %}
-{%   do part_list.append('master-key-download') %}
-[master-key-download]
-recipe = plone.recipe.command
-destination = {{ master_certificate }}
-command = {{ kedifa_getter }} --out ${:destination} --server-ca-certificate {{ kedifa_caucase_ca_certificate }} --identity {{ kedifa_login_certificate }} {{ master_key_download_url }}
-update-command = ${:command}
+{%   do kedifa_updater_mapping.append((master_key_download_url, master_certificate)) %}
 {% endif %}
 
 {% if slave_kedifa_information %}
@@ -174,34 +170,12 @@ bytes = 8
 
 {# ################################################## #}
 {# Set Slave Certificates if needed                   #}
-{%   set cert_dirname = slave_reference.replace('-','.') %}
-{%   set autocert_dir = '/'.join([autocert, cert_dirname]) %}
-[{{ slave_reference }}-path]
-recipe = slapos.cookbook:mkdirectory
-cert = {{ autocert_dir }}
 {# Set certificate key for custom configuration       #}
-{% set certificate = '%s/certificate.pem' % (autocert_dir, ) %}
+{% set cert_name = slave_reference.replace('-','.') + '.pem' %}
+{% set certificate = '%s/%s' % (autocert, cert_name) %}
 {% do slave_parameter_dict.__setitem__('certificate', certificate )%}
+{% do kedifa_updater_mapping.append((key_download_url, certificate)) %}
 
-[{{ slave_reference }}-key-download]
-recipe = plone.recipe.command
-destination = {{ '${' + slave_reference + '-path:cert}/downloaded.pem' }}
-used = {{ '${' + slave_reference + '-path:cert}/certificate.pem' }}
-source-master = ${master-key-download:destination}
-command =
-  {{ kedifa_getter }} --out ${:destination} --server-ca-certificate {{ kedifa_caucase_ca_certificate }} --identity {{ kedifa_login_certificate }} {{ key_download_url }}
-  if [ -f ${:destination} ] ; then
-    # if the slave specific certificate is available, use it
-    ln -sf ${:destination} ${:used}
-  elif [ -f ${:source-master} ] ; then
-    # if the master provided certificate is available, use it
-    ln -sf ${:source-master} ${:used}
-  else
-    rm -f ${:used}
-  fi
-update-command = ${:command}
-
-# BBB: SlapOS Master non-zero knowledge BEGIN
 {# Set ssl certificates for each slave #}
 {%   for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%}
 {%     if cert_name in slave_instance %}
@@ -217,6 +191,7 @@ template = {{ empty_template }}
 rendered = {{ cert_file }}
 extra-context =
     key content {{ cert_title + '-config:value' }}
+# BBB: SlapOS Master non-zero knowledge BEGIN
 # Store certificate in config
 [{{ cert_title + '-config' }}]
 value = {{ dumps(slave_instance.get(cert_name)) }}
@@ -225,32 +200,18 @@ value = {{ dumps(slave_instance.get(cert_name)) }}
 
 {#-   Set Up Certs #}
 {%   do slave_instance.__setitem__('apache_certificate', apache_certificate) %}
-{%   do slave_instance.__setitem__('apache_key', apache_key) %}
 {%   if 'ssl_key' in slave_instance and 'ssl_crt' in slave_instance %}
 {%     set cert_title = '%s-crt' % (slave_reference) %}
-{%     set key_title = '%s-key' % (slave_reference) %}
-{%     set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %}
-{%     set key_file = '/'.join([custom_ssl_directory, key_title.replace('-','.')]) %}
+{%     set cert_file = '/'.join([bbb_ssl_directory, cert_title.replace('-','.')]) %}
 {%     do part_list.append(cert_title) %}
-{%     do part_list.append(key_title) %}
 {%     do slave_parameter_dict.__setitem__("ssl_crt", cert_file) %}
-{%     do slave_parameter_dict.__setitem__("ssl_key", key_file) %}
 {%     do slave_instance.__setitem__('path_to_ssl_crt', cert_file) %}
-{%     do slave_instance.__setitem__('path_to_ssl_key', key_file) %}
-
-[{{key_title}}]
-< = jinja2-template-base
-template = {{ empty_template }}
-rendered = {{ key_file }}
-key-content = {{ dumps(slave_instance.get('ssl_key')) }}
-extra-context =
-    key content :key-content
 
 [{{cert_title}}]
 < = jinja2-template-base
 template = {{ empty_template }}
 rendered = {{ cert_file }}
-cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.get('ssl_ca_crt', '')) }}
+cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.get('ssl_ca_crt', '') + '\n' + slave_instance.get('ssl_key')) }}
 extra-context =
     key content :cert-content
 {%     endif %}
@@ -259,7 +220,7 @@ extra-context =
 {# ########################################## #}
 {# Set Slave Configuration                    #}
 [{{ slave_configuration_section_name }}]
-certificate = {{ '${' + slave_reference + '-key-download:used}' }}
+certificate = {{ certificate }}
 https_port = {{ dumps('' ~ https_port) }}
 http_port = {{ dumps('' ~ http_port) }}
 local_ipv4 = {{ dumps('' ~ local_ipv4) }}
@@ -492,12 +453,36 @@ monitor-base-url = {{ monitor_base_url }}
 csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/csr_id.txt
 csr_id-certificate = ${get-csr_id-certificate:certificate}
 
+[kedifa-updater]
+recipe = slapos.cookbook:wrapper
+command-line = {{ kedifa_updater }}
+  --server-ca-certificate {{ kedifa_caucase_ca_certificate }}
+  --identity {{ kedifa_login_certificate }}
+  --master-certificate {{ master_certificate }}
+  --on-update "{{ frontend_graceful_reload }} ; {{ nginx_graceful_reload }}"
+  ${kedifa-updater-mapping:file}
+  {{ kedifa_updater_state_file }}
+
+wrapper-path = {{ service_directory }}/kedifa-updater
+hash-files = ${buildout:directory}/software_release/buildout.cfg
+
+[kedifa-updater-mapping]
+recipe = slapos.recipe.template:jinja2
+file = {{ kedifa_updater_mapping_file }}
+template = inline:
+{% for mapping in kedifa_updater_mapping %}
+  {{ mapping[0] }} {{ mapping[1] }}
+{% endfor %}
+
+rendered = ${:file}
+
 [buildout]
 extends =
   {{ common_profile }}
   {{ logrotate_base_instance }} 
 
 parts +=
+    kedifa-updater
 {% for part in part_list %}
 {{ '    %s' % part }}
 {% endfor %}

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -31,12 +31,12 @@
 {%   do ssl.__setitem__('certificate', slave_parameter['certificate']) %}
 {%   do ssl.__setitem__('key', slave_parameter['certificate']) %}
 {#- BBB: SlapOS Master non-zero knowledge BEGIN -#}
-{% elif 'path_to_ssl_crt' in slave_parameter and 'path_to_ssl_key' in slave_parameter %}
+{% elif 'path_to_ssl_crt' in slave_parameter  %}
 {%   do ssl.__setitem__('certificate', slave_parameter['path_to_ssl_crt']) %}
-{%   do ssl.__setitem__('key', slave_parameter['path_to_ssl_key']) %}
-{% elif os_module.path.getsize(slave_parameter['apache_certificate']) > 0 and os_module.path.getsize(slave_parameter['apache_key']) > 0 %}
+{%   do ssl.__setitem__('key', slave_parameter['path_to_ssl_crt']) %}
+{% elif os_module.path.getsize(slave_parameter['apache_certificate']) > 0 %}
 {%   do ssl.__setitem__('certificate', slave_parameter['apache_certificate']) %}
-{%   do ssl.__setitem__('key', slave_parameter['apache_key']) %}
+{%   do ssl.__setitem__('key', slave_parameter['apache_certificate']) %}
 {% endif %}
 {#- BBB: SlapOS Master non-zero knowledge END -#}
 {% if 'key' in ssl %}

software/caddy-frontend/templates/nginx-notebook-slave.conf.in

@@ -10,12 +10,12 @@
 {%   do ssl.__setitem__('certificate', slave_parameter['certificate']) %}
 {%   do ssl.__setitem__('key', slave_parameter['certificate']) %}
 {#- BBB: SlapOS Master non-zero knowledge BEGIN -#}
-{% elif 'path_to_ssl_crt' in slave_parameter and 'path_to_ssl_key' in slave_parameter %}
+{% elif 'path_to_ssl_crt' in slave_parameter  %}
 {%   do ssl.__setitem__('certificate', slave_parameter['path_to_ssl_crt']) %}
-{%   do ssl.__setitem__('key', slave_parameter['path_to_ssl_key']) %}
-{% elif os_module.path.getsize(slave_parameter['apache_certificate']) > 0 and os_module.path.getsize(slave_parameter['apache_key']) > 0 %}
+{%   do ssl.__setitem__('key', slave_parameter['path_to_ssl_crt']) %}
+{% elif os_module.path.getsize(slave_parameter['apache_certificate']) > 0 %}
 {%   do ssl.__setitem__('certificate', slave_parameter['apache_certificate']) %}
-{%   do ssl.__setitem__('key', slave_parameter['apache_key']) %}
+{%   do ssl.__setitem__('key', slave_parameter['apache_certificate']) %}
 {% endif %}
 {#- BBB: SlapOS Master non-zero knowledge END -#}
 {% if 'key' in ssl %}

software/caddy-frontend/templates/nginx.cfg.in

@@ -63,9 +63,9 @@ import {{ slave_configuration_directory }}/*.conf
 {%-   do ssl.__setitem__('certificate', master_certificate) -%}
 {%-   do ssl.__setitem__('key', master_certificate) -%}
 {#- BBB: SlapOS Master non-zero knowledge BEGIN -#}
-{%- elif os_module.path.getsize(apache_certificate) > 0 and os_module.path.getsize(apache_key) > 0 -%}
+{%- elif os_module.path.getsize(apache_certificate) > 0 -%}
 {%-   do ssl.__setitem__('certificate', apache_certificate) -%}
-{%-   do ssl.__setitem__('key', apache_key) -%}
+{%-   do ssl.__setitem__('key', apache_certificate) -%}
 {%- endif -%}
 {#- BBB: SlapOS Master non-zero knowledge END -#}
 # Catch-all and 404 for not configured instances

software/caddy-frontend/test/test.py

@@ -690,6 +690,31 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
       verify=cls.ca_certificate_file)
     assert upload.status_code == httplib.CREATED
 
+  @classmethod
+  def runKedifaUpdater(cls):
+    kedifa_updater = None
+    for kedifa_updater in sorted(glob.glob(
+        os.path.join(
+          cls.instance_path, '*', 'etc', 'service', 'kedifa-updater*'))):
+      # fetch first kedifa-updater, as by default most of the tests are using
+      # only one running partition; in case if test does not need
+      # kedifa-updater this method can be overridden
+      break
+    if kedifa_updater is not None:
+      # try few times kedifa_updater
+      for i in range(10):
+        return_code, output = subprocess_status_output(
+          [kedifa_updater, '--once'])
+        if return_code == 0:
+          break
+        # wait for the other updater to work
+        time.sleep(2)
+      # assert that in the worst case last run was correct
+      assert return_code == 0, output
+      # give caddy a moment to refresh its config, as sending signal does not
+      # block until caddy is refreshed
+      time.sleep(2)
+
   @classmethod
   def untilSlavePartitionReady(cls):
     for slave_reference, partition_parameter_kw in cls\
@@ -724,6 +749,7 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
     # run partition for slaves to be setup
     cls.runComputerPartitionUntil(
       cls.untilSlavePartitionReady)
+    cls.runKedifaUpdater()
     for slave_reference, partition_parameter_kw in cls\
             .getSlaveParameterDictDict().items():
       slave_instance = request(
@@ -1597,9 +1623,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       data=data,
       verify=self.ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
-
-    # after partitions being processed the key will be used for this slave
-    self.runComputerPartition(max_quantity=1)
+    self.runKedifaUpdater()
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
@@ -1612,7 +1636,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     certificate_file_list = glob.glob(os.path.join(
       self.instance_path, '*', 'srv', 'autocert',
-      '_custom_domain_ssl_crt_ssl_key_ssl_ca_crt', 'certificate.pem'))
+      '_custom_domain_ssl_crt_ssl_key_ssl_ca_crt.pem'))
     self.assertEqual(1, len(certificate_file_list))
     certificate_file = certificate_file_list[0]
     with open(certificate_file) as out:
@@ -1681,9 +1705,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       verify=self.ca_certificate_file)
 
     self.assertEqual(httplib.CREATED, upload.status_code)
-
-    # after partitions being processed the key will be used for this slave
-    self.runComputerPartition(max_quantity=1)
+    self.runKedifaUpdater()
 
     with self.assertRaises(requests.exceptions.SSLError):
       self.fakeHTTPSResult(
@@ -1691,7 +1713,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     certificate_file_list = glob.glob(os.path.join(
       self.instance_path, '*', 'srv', 'autocert',
-      '_ssl_ca_crt_garbage', 'certificate.pem'))
+      '_ssl_ca_crt_garbage.pem'))
     self.assertEqual(1, len(certificate_file_list))
     certificate_file = certificate_file_list[0]
     with open(certificate_file) as out:
@@ -1727,9 +1749,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       verify=self.ca_certificate_file)
 
     self.assertEqual(httplib.CREATED, upload.status_code)
-
-    # after partitions being processed the key will be used for this slave
-    self.runComputerPartition(max_quantity=1)
+    self.runKedifaUpdater()
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
@@ -1742,7 +1762,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     certificate_file_list = glob.glob(os.path.join(
       self.instance_path, '*', 'srv', 'autocert',
-      '_ssl_ca_crt_does_not_match', 'certificate.pem'))
+      '_ssl_ca_crt_does_not_match.pem'))
     self.assertEqual(1, len(certificate_file_list))
     certificate_file = certificate_file_list[0]
     with open(certificate_file) as out:
@@ -1844,9 +1864,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       data=data,
       verify=self.ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
-
-    # after partitions being processed the key will be used for this slave
-    self.runComputerPartition(max_quantity=1)
+    self.runKedifaUpdater()
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
@@ -3574,6 +3592,10 @@ class TestDefaultMonitorHttpdPort(SlaveHttpFrontendTestCase, TestDataMixin):
       'caucase_port': CAUCASE_PORT,
     }
 
+  @classmethod
+  def runKedifaUpdater(cls):
+    return
+
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -4468,9 +4490,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster(
       master_parameter_dict['master-key-upload-url'] + auth.text,
       data=key_pem + certificate_pem,
       verify=self.ca_certificate_file)
-
-    # after partitions being processed the key will be used for this slave
-    self.runComputerPartition(max_quantity=1)
+    self.runKedifaUpdater()
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
@@ -4761,9 +4781,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
       data=data,
       verify=self.ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
-
-    # after partitions being processed the key will be used for this slave
-    self.runComputerPartition(max_quantity=1)
+    self.runKedifaUpdater()
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
@@ -4855,8 +4873,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
       verify=self.ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
 
-    # after partitions being processed the key will be used for this slave
-    self.runComputerPartition(max_quantity=1)
+    self.runKedifaUpdater()
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
@@ -4940,8 +4957,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
       verify=self.ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
 
-    # after partitions being processed the key will be used for this slave
-    self.runComputerPartition(max_quantity=1)
+    self.runKedifaUpdater()
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
@@ -5034,8 +5050,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
       verify=self.ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
 
-    # after partitions being processed the key will be used for this slave
-    self.runComputerPartition(max_quantity=1)
+    self.runKedifaUpdater()
 
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',

software/caddy-frontend/test/test_data/test.TestDefaultMonitorHttpdPort.test_promise_run_plugin-CADDY.txt

@@ -12,10 +12,10 @@ T-2/etc/plugin/caddy_ssl_cached.py: OK
 T-2/etc/plugin/check-_test-error-log-last-day.py: OK
 T-2/etc/plugin/check-_test-error-log-last-hour.py: OK
 T-2/etc/plugin/check-free-disk-space.py: OK
-T-2/etc/plugin/frontend-caddy-configuration-promise.py: OK
+T-2/etc/plugin/frontend-caddy-configuration-promise.py: ERROR
 T-2/etc/plugin/monitor-bootstrap-status.py: OK
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
-T-2/etc/plugin/nginx-configuration-promise.py: OK
+T-2/etc/plugin/nginx-configuration-promise.py: ERROR
 T-2/etc/plugin/nginx_frontend_ipv4_http.py: OK
 T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
 T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK

software/caddy-frontend/test/test_data/test.TestDefaultMonitorHttpdPort.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful STOPPED
 T-2:frontend_caddy-{hash}-on-watch STOPPED
 T-2:frontend_nginx-{hash}-on-watch STOPPED
 T-2:kedifa-login-certificate-caucase-updater-on-watch STOPPED
+T-2:kedifa-updater-{hash}-on-watch STOPPED
 T-2:monitor-httpd-{hash}-on-watch STOPPED
 T-2:monitor-httpd-graceful STOPPED
 T-2:trafficserver-{hash}-on-watch STOPPED

software/caddy-frontend/test/test_data/test.TestDuplicateSiteKeyProtection.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestMalformedBackenUrlSlave.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_promise_run_plugin-CADDY.txt

@@ -15,7 +15,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py: OK
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
 T-2/etc/plugin/nginx-configuration-promise.py: OK
 T-2/etc/plugin/nginx_frontend_ipv4_http.py: OK
-T-2/etc/plugin/nginx_frontend_ipv4_https.py: ERROR
+T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
 T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
 T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
 T-2/etc/plugin/re6st-connectivity.py: OK

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch EXITED
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch EXITED
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_promise_run_plugin-CADDY.txt

@@ -15,7 +15,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py: OK
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
 T-2/etc/plugin/nginx-configuration-promise.py: OK
 T-2/etc/plugin/nginx_frontend_ipv4_http.py: OK
-T-2/etc/plugin/nginx_frontend_ipv4_https.py: ERROR
+T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
 T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
 T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
 T-2/etc/plugin/re6st-connectivity.py: OK

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch EXITED
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch EXITED
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestQuicEnabled.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_promise_run_plugin-CADDY.txt

@@ -33,14 +33,14 @@ T-3/etc/plugin/caddy_ssl_cached.py: OK
 T-3/etc/plugin/check-_replicate-error-log-last-day.py: OK
 T-3/etc/plugin/check-_replicate-error-log-last-hour.py: OK
 T-3/etc/plugin/check-free-disk-space.py: OK
-T-3/etc/plugin/frontend-caddy-configuration-promise.py: OK
+T-3/etc/plugin/frontend-caddy-configuration-promise.py: ERROR
 T-3/etc/plugin/monitor-bootstrap-status.py: OK
 T-3/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
-T-3/etc/plugin/nginx-configuration-promise.py: OK
+T-3/etc/plugin/nginx-configuration-promise.py: ERROR
 T-3/etc/plugin/nginx_frontend_ipv4_http.py: OK
 T-3/etc/plugin/nginx_frontend_ipv4_https.py: OK
 T-3/etc/plugin/nginx_frontend_ipv6_http.py: OK
 T-3/etc/plugin/nginx_frontend_ipv6_https.py: OK
 T-3/etc/plugin/re6st-connectivity.py: OK
-T-3/etc/plugin/trafficserver-cache-availability.py: ERROR
+T-3/etc/plugin/trafficserver-cache-availability.py: OK
 T-3/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING
@@ -42,6 +43,7 @@ T-3:frontend-nginx-safe-graceful STOPPED
 T-3:frontend_caddy-{hash}-on-watch STOPPED
 T-3:frontend_nginx-{hash}-on-watch STOPPED
 T-3:kedifa-login-certificate-caucase-updater-on-watch STOPPED
+T-3:kedifa-updater-{hash}-on-watch STOPPED
 T-3:monitor-httpd-{hash}-on-watch STOPPED
 T-3:monitor-httpd-graceful STOPPED
 T-3:trafficserver-{hash}-on-watch STOPPED

software/caddy-frontend/test/test_data/test.TestSlave.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestSlaveBadParameters.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_supervisor_state-CADDY.txt

@@ -23,6 +23,7 @@ T-2:frontend-nginx-safe-graceful EXITED
 T-2:frontend_caddy-{hash}-on-watch RUNNING
 T-2:frontend_nginx-{hash}-on-watch RUNNING
 T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash}-on-watch RUNNING
 T-2:monitor-httpd-{hash}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
 T-2:trafficserver-{hash}-on-watch RUNNING