caddy-frontend: Use publish_failsafe for slave publishing

Due to various reasons there might be an error while publishing information
about a slave, and such error would end up with stopping the processing of
the partition. This is undesired effect, thus switch to publish_failsafe style
in order to continue processing.

software/caddy-frontend/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = 1ec349747487b45dd6b30bb2aa45acbe
 
 [profile-master]
 filename = instance-master.cfg.in
-md5sum = 50dbd9bd25fbc608b576ac83f4e13b0f
+md5sum = 4edf417511a7bdb9f95c04640fcca4df
 
 [profile-slave-list]
 filename = instance-slave-list.cfg.in
@@ -34,7 +34,7 @@ md5sum = aba91817a1b58377597500f676603d23
 
 [profile-master-publish-slave-information]
 filename = instance-master-publish-slave-information.cfg.in
-md5sum = e03eef8feee481256928a0e9249642c5
+md5sum = cba4d995962f7fbeae3f61c9372c4181
 
 [template-caddy-frontend-configuration]
 _update_hash_filename_ = templates/Caddyfile.in

software/caddy-frontend/instance-master-publish-slave-information.cfg.in

@@ -67,7 +67,8 @@
 {%     do part_list.append(publish_section_title) %}
 [{{ publish_section_title }}]
 <= common-frontend-information
-recipe = slapos.cookbook:publish
+recipe = slapos.cookbook:publish_failsafe
+-error-status-file = {{ publish_failsafe_error }}/{{ slave_reference }}-error.status
 -slave-reference = {{ slave_reference }}
 {%     set log_access_url = slave_information.pop('log-access-urls', None) %}
 {%     if log_access_url %}

software/caddy-frontend/instance-master.cfg.in

@@ -546,6 +546,7 @@ extra-context =
     section active_slave_instance_dict active-slave-instance
     section warning_slave_information warning-slave-information
     key slave_kedifa_information request-kedifa:connection-slave-kedifa-information
+    key publish_failsafe_error directory:publish-failsafe-error-var
 
 [monitor-base-url-dict]
 kedifa = ${request-kedifa:connection-monitor-base-url}
@@ -564,6 +565,8 @@ caucased = ${:srv}/caucased
 backup-caucased = ${:backup}/caucased
 # NGINX
 rejected-var = ${:var}/rejected-nginx
+# slaves
+publish-failsafe-error-var = ${:var}/publish-failsafe-error
 
 {% if aikc_enabled %}
 [directory]
@@ -986,6 +989,19 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
       key_len=2048
 )}}
 
+[check-publish-failsafe-error]
+recipe = plone.recipe.command
+file = ${directory:var}/publish-failsafe-error.check
+command = {{ software_parameter_dict['findutils'] }}/bin/find ${directory:publish-failsafe-error-var} -type f -printf "%f\n" > ${:file}
+update-command = ${:command}
+
+[publish-failsafe-error-promise]
+<= monitor-promise-base
+promise = check_file_state
+name = publish-failsafe-error.py
+config-filename = ${check-publish-failsafe-error:file}
+config-state = empty
+
 [buildout]
 extends =
   {{ software_parameter_dict['profile_common'] }}
@@ -993,6 +1009,8 @@ extends =
 parts =
   monitor-base
   publish-slave-information
+  check-publish-failsafe-error
+  publish-failsafe-error-promise
   publish-information
   request-kedifa
   rejected-slave-promise

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestMasterAIKCDisabledAIBCCDisabledRequest.test_file_list_plugin.txt

@@ -9,6 +9,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_plugin.txt

@@ -14,6 +14,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestSlaveHealthCheck.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestSlaveHostHaproxyClash.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_plugin.txt

@@ -13,6 +13,7 @@ T-0/etc/plugin/master-key-upload-url-ready-promise.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
 T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
 T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
 T-0/etc/plugin/rejected-slave.py
 T-1/etc/plugin/__init__.py