caddy-frontend: Setup backend client auth

By default do not offer authentication certificate, the switch
authenticate-to-backend can be used on cluster or slave level to control
this feature.

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = db3f99b99f7db132928871acefc5b56a
+md5sum = 99f2f6d8818da4a98ca48412453c4f90
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,15 +22,15 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = ae7c867ebbfe3e2881f4b1cdbf414152
+md5sum = 23237969bbd9e974ac674b2052e8d67c
 
 [template-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 6d7113ebf0c46b0e4c72c128ebb647db
+md5sum = 5dabdbf51d20bf9e9e277e5b84d58b7e
 
 [template-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 5deaf0c4cbe84216d2b061f21494539c
+md5sum = be95a8ff3b0c2db22d80aa07c47504e3
 
 [template-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -54,7 +54,7 @@ md5sum = 266f175dbdfc588af7a86b0b1884fe73
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = ad130d64ac6e4fc0dc5a028494573d62
+md5sum = 8daeaaa02190d90e0df91e364b1d2fc0
 
 [template-log-access]
 _update_hash_filename_ = templates/template-log-access.conf.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -14,6 +14,8 @@ parts =
   switch-caddy-softwaretype
   caucase-updater
   caucase-updater-promise
+  backend-client-caucase-updater
+  backend-client-caucase-updater-promise
   frontend-caddy-graceful
   port-redirection
   promise-frontend-caddy-configuration
@@ -67,6 +69,7 @@ service = ${:etc}/service
 etc-run = ${:etc}/run
 
 ca-dir = ${:srv}/ssl
+backend-client-dir = ${:srv}/backend-client
 # BBB: SlapOS Master non-zero knowledge BEGIN
 bbb-ssl-dir = ${:srv}/bbb-ssl
 # BBB: SlapOS Master non-zero knowledge END
@@ -210,6 +213,47 @@ kedifa-updater-mapping-file = ${directory:etc}/kedifa_updater_mapping.txt
 kedifa-updater-state-file = ${directory:srv}/kedifa_updater_state.json
 slave_kedifa_information = {{ dumps(slapparameter_dict['slave-kedifa-information']) }}
 
+[backend-client-login-config]
+d = ${directory:backend-client-dir}
+template-csr = ${:d}/csr.pem
+key = ${:d}/certificate.pem
+certificate = ${:key}
+ca-certificate = ${:d}/ca.pem
+cas-ca-certificate = ${:d}/cas-ca.pem
+crl = ${:d}/crl.pem
+
+[backend-client-login-csr]
+recipe = plone.recipe.command
+organization = {{ slapparameter_dict['cluster-identification'] }}
+organizational_unit = {{ instance_parameter['configuration.frontend-name'] }}
+command =
+{% if slapparameter_dict['backend-client-caucase-url'] %}
+  if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ]  ; then
+    {{ parameter_dict['openssl'] }} req -new -sha256 \
+      -newkey rsa:2048 -nodes -keyout ${:key} \
+      -subj "/O=${:organization}/OU=${:organizational_unit}" \
+      -out ${:template-csr}
+  fi
+{% endif %}
+  test -f ${:key} && test -f ${:template-csr}
+update-command = ${:command}
+template-csr = ${backend-client-login-config:template-csr}
+key = ${backend-client-login-config:key}
+stop-on-error = True
+
+{{ caucase.updater(
+     prefix='backend-client-caucase-updater',
+     buildout_bin_directory=parameter_dict['bin_directory'],
+     updater_path='${directory:service}/backend-client-login-certificate-caucase-updater',
+     url=slapparameter_dict['backend-client-caucase-url'],
+     data_dir='${directory:srv}/backend-client-caucase-updater',
+     crt_path='${backend-client-login-config:certificate}',
+     ca_path='${backend-client-login-config:ca-certificate}',
+     crl_path='${backend-client-login-config:crl}',
+     key_path='${backend-client-login-csr:key}',
+     template_csr='${backend-client-login-csr:template-csr}'
+)}}
+
 [dynamic-custom-personal-template-slave-list]
 < = jinja2-template-base
 template = {{ parameter_dict['template_slave_list'] }}
@@ -225,8 +269,10 @@ caddy_executable = {{ parameter_dict['caddy'] }}
 sixtunnel_executable = {{ parameter_dict['sixtunnel'] }}/bin/6tunnel
 organization = {{ slapparameter_dict['cluster-identification'] }}
 organizational-unit = {{ instance_parameter['configuration.frontend-name'] }}
+backend-client-caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
 extra-context =
     key caddy_configuration_directory caddy-directory:slave-configuration
+    key backend_client_caucase_url :backend-client-caucase-url
     import urlparse_module urlparse
     key caddy_executable :caddy_executable
     key http_port configuration:plain_http_port
@@ -656,6 +702,13 @@ log-socket = ${backend-haproxy-rsyslogd:log-socket}
 graceful-command = ${backend-haproxy-validate:rendered} && kill -USR2 $(cat ${:pid-file})
 http-port = ${configuration:backend-haproxy-http-port}
 https-port = ${configuration:backend-haproxy-https-port}
+# Caucase related configuration
+caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
+ca-certificate = ${backend-client-login-config:ca-certificate}
+certificate = ${backend-client-login-config:certificate}
+cas-ca-certificate = ${backend-client-login-config:cas-ca-certificate}
+csr = ${backend-client-caucase-updater-csr:csr}
+crl = ${backend-client-login-config:crl}
 
 [backend-haproxy]
 recipe = slapos.cookbook:wrapper
@@ -676,7 +729,7 @@ template = {{ parameter_dict['template_configuration_state_script'] }}
 rendered = ${directory:bin}/${:_buildout_section_name_}
 mode = 0700
 
-path_list = ${backend-haproxy-configuration:file}
+path_list = ${backend-haproxy-configuration:file} ${backend-client-login-config:certificate}
 sha256sum = {{ parameter_dict['sha256sum'] }}
 
 extra-context =

software/caddy-frontend/instance-caddy-input-schema.json

@@ -88,6 +88,16 @@
       "title": "Automatic Internal KeDiFa's Caucase CSR",
       "type": "string"
     },
+    "automatic-internal-backend-client-caucase-csr": {
+      "default": "true",
+      "description": "Automatically signs CSRs sent to Backend Client's caucase, based on csr_id and matching certificate.",
+      "enum": [
+        "true",
+        "false"
+      ],
+      "title": "Automatic Internal Backend Client's Caucase CSR",
+      "type": "string"
+    },
     "ciphers": {
       "description": "List of ciphers. Empty defaults to Caddy list of ciphers. See https://caddyserver.com/docs/tls for more information.",
       "title": "Ordered space separated list of ciphers",
@@ -98,6 +108,16 @@
       "description": "Timeout for HTTP requests.",
       "title": "HTTP Request timeout in seconds",
       "type": "integer"
+    },
+    "authenticate-to-backend": {
+      "default": "false",
+      "description": "If set to true the frontend certificate will be used as authentication certificate to the backend. Note: backend might have to know the frontend CA, available with 'backend-client-caucase-url'.",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "title": "Authenticate to backend",
+      "type": "string"
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/instance-output-schema.json

@@ -73,6 +73,10 @@
     "warning-list": {
       "description": "List of warning found during the request.",
       "type": "array"
+    },
+    "backend-client-caucase-url": {
+      "description": "URL to caucase used by authentication to the backend.",
+      "type": "string"
     }
   },
   "type": "object"

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -214,6 +214,15 @@
       "description": "List of ciphers. Empty defaults to cluster list of ciphers, which by default are Caddy list of ciphers. See https://caddyserver.com/docs/tls for more information.",
       "title": "Ordered space separated list of ciphers",
       "type": "string"
+    },
+    "authenticate-to-backend": {
+      "description": "If set to true the frontend certificate will be used as authentication certificate to the backend. Note: backend might have to know the frontend CA, available with 'backend-client-caucase-url'.",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "title": "Authenticate to backend",
+      "type": "string"
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/instance-slave-output-schema.json

@@ -49,6 +49,10 @@
     "kedifa-caucase-url": {
       "description": "URL to caucase used by KeDiFa",
       "type": "string"
+    },
+    "backend-client-caucase-url": {
+      "description": "URL to caucase used by authentication to the backend.",
+      "type": "string"
     }
   },
   "type": "object"

software/caddy-frontend/instance.cfg.in

@@ -103,6 +103,7 @@ configuration.nginx_port = 9443
 configuration.kedifa_port = 7879
 # Warning: Caucase takes also cacuase_port+1
 configuration.caucase_port = 8890
+configuration.caucase_backend_client_port = 8990
 configuration.apache-key =
 configuration.apache-certificate =
 configuration.open-port = 80 443
@@ -121,3 +122,4 @@ configuration.backend-connect-timeout = 5
 configuration.backend-connect-retries = 3
 configuration.backend-haproxy-http-port = 21080
 configuration.backend-haproxy-https-port = 21443
+configuration.authenticate-to-backend = False

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -110,11 +110,12 @@ create = true
 {%-   do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
 {%-   do slave_instance.__setitem__('global_disable_http2', global_disable_http2) %}
 {#-   Pass backend timeout values #}
-{%-   for key in ['backend-connect-timeout', 'backend-connect-retries', 'request-timeout'] %}
+{%-   for key in ['backend-connect-timeout', 'backend-connect-retries', 'request-timeout', 'authenticate-to-backend'] %}
 {%-     if slave_instance.get(key, '') == '' %}
 {%-       do slave_instance.__setitem__(key, configuration[key]) %}
 {%-     endif %}
 {%-   endfor %}
+{%-   do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %}
 {#-   Set Up log files #}
 {%-   do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %}
 {%-   do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}
@@ -126,6 +127,7 @@ create = true
 {%-   do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
 {%-   do slave_publish_dict.__setitem__('slave-reference', slave_reference) %}
 {%-   do slave_publish_dict.__setitem__('public-ipv4', public_ipv4) %}
+{%-   do slave_publish_dict.__setitem__('backend-client-caucase-url', backend_client_caucase_url) %}
 {#-   Set slave domain if none was defined #}
 {%-   if slave_instance.get('custom_domain', None) == None %}
 {%-     set domain_prefix = slave_instance.get('slave_reference').replace("-", "").replace("_", "").lower() %}
@@ -342,6 +344,7 @@ slave-instance-information-list = {{ json_module.dumps(slave_instance_informatio
 {%- endif %}
 monitor-base-url = {{ monitor_base_url }}
 csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/csr_id.txt
+backend-client-csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/backend-haproxy-csr_id.txt
 csr_id-certificate = ${get-csr_id-certificate:certificate}
 
 [kedifa-updater]
@@ -400,6 +403,25 @@ https-port = {{ ('' ~ backend_haproxy_configuration['https-port']) }}
 request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
 backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
 backend-connect-retries =  {{ dumps('' ~ configuration['backend-connect-retries']) }}
+certificate = {{ dumps('' ~ backend_haproxy_configuration['certificate']) }}
+
+[store-backend-haproxy-csr_id]
+recipe = plone.recipe.command
+
+csr_id_path = {{ directory['csr_id'] }}/backend-haproxy-csr_id.txt
+csr_work_path = {{ directory['tmp'] }}/${:_buildout_section_name_}
+
+stop-on-error = False
+update-command = ${:command}
+command =
+  {{ bin_directory }}/caucase \
+    --ca-url {{ backend_haproxy_configuration['caucase-url'] }} \
+    --ca-crt {{ backend_haproxy_configuration['cas-ca-certificate'] }} \
+    --crl {{ backend_haproxy_configuration['crl'] }} \
+    --mode service \
+    --send-csr {{ backend_haproxy_configuration['csr'] }} > ${:csr_work_path} && \
+  cut -d ' ' -f 1 ${:csr_work_path} > ${:csr_id_path}
+
 ##<Backend haproxy>
 
 [buildout]
@@ -483,7 +505,9 @@ config-hostname = ${expose-csr_id-configuration:ip}
 config-port = ${expose-csr_id-configuration:port}
 
 [expose-csr_id]
-depends = ${store-csr_id:command}
+depends =
+  ${store-csr_id:command}
+  ${store-backend-haproxy-csr_id:command}
 recipe = slapos.cookbook:wrapper
 command-line = {{ caddy_executable }}
   -conf ${expose-csr_id-template:rendered}

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -48,7 +48,11 @@ frontend https-backend
 {%-   for (scheme, prefix) in [('http', 'http_backend'), ('https', 'https_backend')] %}
 {%-     set info_dict = slave_instance[prefix] %}
 {%-     if info_dict['scheme'] == 'https' %}
-{%-       set ssl = ['ssl verify'] %}
+{%-       set ssl = [] %}
+{%-       if slave_instance['authenticate-to-backend'] %}
+{%-         set ssl = ['crt %s' % (configuration['certificate'],)] %}
+{%-       endif %}
+{%-       do ssl.append('ssl verify') %}
 {%-       set path_to_ssl_proxy_ca_crt = slave_instance.get('path_to_ssl_proxy_ca_crt') %}
 {%-       if slave_instance['ssl_proxy_verify'] %}
 {%-         if path_to_ssl_proxy_ca_crt %}

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -11,6 +11,12 @@ T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
 T-2/var/log/httpd/_Url_access_log
 T-2/var/log/httpd/_Url_error_log
+T-2/var/log/httpd/_auth-to-backend-backend-ignore_access_log
+T-2/var/log/httpd/_auth-to-backend-backend-ignore_error_log
+T-2/var/log/httpd/_auth-to-backend-not-configured_access_log
+T-2/var/log/httpd/_auth-to-backend-not-configured_error_log
+T-2/var/log/httpd/_auth-to-backend_access_log
+T-2/var/log/httpd/_auth-to-backend_error_log
 T-2/var/log/httpd/_ciphers_access_log
 T-2/var/log/httpd/_ciphers_error_log
 T-2/var/log/httpd/_custom_domain_access_log

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestSlave.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -11,6 +11,12 @@ T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
 T-2/var/log/httpd/_Url_access_log
 T-2/var/log/httpd/_Url_error_log
+T-2/var/log/httpd/_auth-to-backend-backend-ignore_access_log
+T-2/var/log/httpd/_auth-to-backend-backend-ignore_error_log
+T-2/var/log/httpd/_auth-to-backend-not-configured_access_log
+T-2/var/log/httpd/_auth-to-backend-not-configured_error_log
+T-2/var/log/httpd/_auth-to-backend_access_log
+T-2/var/log/httpd/_auth-to-backend_error_log
 T-2/var/log/httpd/_ciphers_access_log
 T-2/var/log/httpd/_ciphers_error_log
 T-2/var/log/httpd/_custom_domain_access_log

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_plugin-CADDY.txt

 T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
 T-0/etc/plugin/aikc-user-caucase-updater.py
 T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
 T-0/etc/plugin/check-free-disk-space.py
 T-0/etc/plugin/monitor-bootstrap-status.py
 T-0/etc/plugin/monitor-http-frontend.py
@@ -17,6 +19,7 @@ T-1/etc/plugin/monitor-bootstrap-status.py
 T-1/etc/plugin/monitor-http-frontend.py
 T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
 T-0:aikc-user-caucase-updater-on-watch RUNNING
 T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
 T-0:certificate_authority-{hash-generic}-on-watch RUNNING
 T-0:crond-{hash-generic}-on-watch RUNNING
 T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
@@ -17,6 +19,7 @@ T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-1:monitor-httpd-graceful EXITED
 T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
 T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
 T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
 T-2:backend-haproxy-safe-graceful EXITED