feature/caddy-frontend: Expose slaves' log files with real frontend

See merge request !791

component/nginx/buildout.cfg

@@ -13,8 +13,8 @@ parts = nginx-output
 [nginx-common]
 recipe = slapos.recipe.cmmi
 shared = false
-url = https://nginx.org/download/nginx-1.17.6.tar.gz
-md5sum = 55022aa5715386c994f6773478822853
+url = https://nginx.org/download/nginx-1.19.2.tar.gz
+md5sum = 3dc55f6451ed6f819f1c796f4e5e9617
 
 [nginx]
 <= nginx-common
@@ -26,8 +26,8 @@ configure-options=
   --with-http_realip_module
   --with-mail
   --with-mail_ssl_module
-  --with-ld-opt="-L ${openssl-1.0:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
-  --with-cc-opt="-I ${openssl-1.0:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
+  --with-ld-opt="-L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
+  --with-cc-opt="-I ${openssl:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
 
 [nginx-dav-ext-module]
 recipe = slapos.recipe.build:download-unpacked
@@ -46,8 +46,8 @@ configure-options =
   --with-mail
   --with-mail_ssl_module
   --error-log-path=var/log/nginx.error.log
-  --with-ld-opt=" -L ${libexpat:location}/lib -L ${openssl-1.0:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
-  --with-cc-opt="-I ${libexpat:location}/include -I ${openssl-1.0:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
+  --with-ld-opt=" -L ${libexpat:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
+  --with-cc-opt="-I ${libexpat:location}/include -I ${openssl:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
   --with-http_dav_module 
   --add-module='${nginx-dav-ext-module:location}'
 
@@ -73,8 +73,8 @@ strip-top-level-dir = true
 configure-options=
   --with-ipv6
   --with-http_ssl_module
-  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl-1.0:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib"
-  --with-cc-opt="-I ${pcre:location}/include -I ${openssl-1.0:location}/include -I ${zlib:location}/include"
+  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
+  --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include"
   --add-module=${hexaglobe-nginx-module:location}/sub_module
 # --add-module=${hexaglobe-nginx-module:location}/nginx-upstream-fair
 
@@ -91,8 +91,8 @@ configure-options=
   --with-http_ssl_module
   --with-http_v2_module
   --with-http_gzip_static_module
-  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl-1.0:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib"
-  --with-cc-opt="-I ${pcre:location}/include -I ${openssl-1.0:location}/include -I ${zlib:location}/include -Wno-error"
+  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
+  --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include -Wno-error"
   --add-module=${nginx-push-stream-module:location}
 
 [nginx-push-stream-output]

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = dae3bf6daf851b5610a1a6bd83057b29
+md5sum = 30eb6db0d2df9ffb3a50fb64e73df5be
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,7 +22,7 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 43b3763d4086283e9c7c886e3b89e54b
+md5sum = 9a8064a771de806e7cbddac067957ea9
 
 [template-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,7 +30,7 @@ md5sum = 19debfbc27c464f451b1eb5bb5ce3c84
 
 [template-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 8f518a7b543126ce91fe218255fb201c
+md5sum = da4e3de1c1861459273c6e8d32ead21e
 
 [template-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -106,7 +106,7 @@ md5sum = 8c150e1e6c993708d31936742f3a7302
 
 [caddyprofiledeps-setup]
 filename = setup.py
-md5sum = d9b6476bb0b36cf463fddb00d41dfbaa
+md5sum = 8e1c6c06c09beb921965b3ce98c67c9e
 
 [caddyprofiledeps-dummy]
 filename = caddyprofiledummy.py
@@ -119,3 +119,7 @@ md5sum = f77744e52f9d028c39f99cfbf31eadaf
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
 md5sum = be899b04e1aa652ed510f20d4ea523dd
+
+[template-slave-introspection-httpd-nginx]
+_update_hash_filename_ = templates/slave-introspection-httpd-nginx.conf.in
+md5sum = 3067e6ba6c6901821d57d2109517d39c

software/caddy-frontend/common.cfg

@@ -12,6 +12,7 @@ extends =
   ../../component/xz-utils/buildout.cfg
   ../../component/rsyslogd/buildout.cfg
   ../../component/haproxy/buildout.cfg
+  ../../component/nginx/buildout.cfg
 
   ../../stack/caucase/buildout.cfg
 # Monitoring stack (keep on bottom)
@@ -94,6 +95,8 @@ logrotate_base_instance = ${template-logrotate-base:rendered}
 bin_directory = ${buildout:bin-directory}
 
 sixtunnel = ${6tunnel:location}
+nginx = ${nginx-output:nginx}
+nginx_mime = ${nginx-output:mime}
 caddy = ${caddy:output}
 caddy_location = ${caddy:location}
 haproxy_executable = ${haproxy:location}/sbin/haproxy
@@ -110,6 +113,7 @@ kedifa = ${:bin_directory}/kedifa
 kedifa-updater = ${:bin_directory}/kedifa-updater
 kedifa-csr = ${:bin_directory}/kedifa-csr
 xz_location = ${xz-utils:location}
+htpasswd = ${:bin_directory}/htpasswd
 
 monitor_template = ${monitor-template:output}
 template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
@@ -129,6 +133,7 @@ template_trafficserver_records_config = ${template-trafficserver-records-config:
 template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
 template_trafficserver_logging_config = ${template-trafficserver-logging-config:target}
 template_wrapper = ${template-wrapper:output}
+template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
 
 [template]
 recipe = slapos.recipe.template:jinja2
@@ -198,6 +203,9 @@ mode = 640
 [template-empty]
 <=download-template
 
+[template-slave-introspection-httpd-nginx]
+<=download-template
+
 [template-wrapper]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/templates/wrapper.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -52,6 +52,15 @@ parts =
   promise-backend-haproxy-https
   promise-backend-haproxy-configuration
 
+  slave-introspection-frontend
+  slave-introspection-graceful
+  promise-slave-introspection-https
+  promise-slave-introspection-configuration
+  logrotate-entry-slave-introspection
+
+[caddyprofiledeps]
+recipe = caddyprofiledeps
+
 # Create all needed directories
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -83,6 +92,9 @@ csr_id = ${:srv}/csr_id
 caddy-csr_id = ${:etc}/caddy-csr_id
 caddy-csr_id-log = ${:log}/httpd-csr_id
 
+# slave introspection
+slave-introspection-var = ${:var}/slave-introspection
+
 [switch-caddy-softwaretype]
 recipe = slapos.cookbook:softwaretype
 single-default = ${dynamic-custom-personal-template-slave-list:rendered}
@@ -95,6 +107,14 @@ ip-access-certificate = ${self-signed-ip-access:certificate}
 caddy-directory = {{ parameter_dict['caddy_location'] }}
 caddy-ipv6 = {{ instance_parameter['ipv6-random'] }}
 caddy-https-port = ${configuration:port}
+nginx = {{ parameter_dict['nginx'] }}
+nginx_mime = {{ parameter_dict['nginx_mime'] }}
+htpasswd = {{ parameter_dict['htpasswd'] }}
+slave-introspection-template = {{ parameter_dict['template_slave_introspection_httpd_nginx'] }}
+slave-introspection-configuration = ${directory:etc}/slave-introspection-httpd-nginx.conf
+slave-introspection-https-port = ${configuration:slave-introspection-https-port}
+slave-introspection-secure_access = ${slave-introspection-frontend:connection-secure_access}
+slave-introspection-domain = ${slave-introspection-frontend:connection-domain}
 
 [self-signed-ip-access]
 # Self Signed certificate for HTTPS IP accesses to the frontend
@@ -258,6 +278,7 @@ stop-on-error = True
 
 [dynamic-custom-personal-template-slave-list]
 < = jinja2-template-base
+depends = ${caddyprofiledeps:recipe}
 template = {{ parameter_dict['template_slave_list'] }}
 filename = custom-personal-instance-slave-list.cfg
 slave_instance_list = {{ dumps(instance_parameter['slave-instance-list']) }}
@@ -276,6 +297,7 @@ extra-context =
     key caddy_configuration_directory caddy-directory:slave-configuration
     key backend_client_caucase_url :backend-client-caucase-url
     import urlparse_module urlparse
+    import furl_module furl
     key caddy_executable :caddy_executable
     key http_port configuration:plain_http_port
     key https_port configuration:port
@@ -400,6 +422,11 @@ not-found-file = ${caddy-directory:document-root}/${not-found-html:filename}
 master-certificate = ${caddy-directory:master-autocert-dir}/master.pem
 # Communication with ATS
 cache-port = ${trafficserver-variable:input-port}
+# slave instrspection
+slave-introspection-access-log = ${directory:log}/slave-introspection-access.log
+slave-introspection-error-log = ${directory:log}/slave-introspection-error.log
+slave-introspection-pid-file = ${directory:run}/slave-introspection.pid
+slave-introspection-graceful-command = ${slave-introspection-validate:rendered} && kill -HUP $(cat ${:slave-introspection-pid-file})
 
 # BBB: SlapOS Master non-zero knowledge BEGIN
 [get-self-signed-fallback-access]
@@ -892,6 +919,91 @@ extra-context =
     key http_port configuration:plain_http_port
     key https_port configuration:port
 
+[slave-introspection-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Slave Introspection Frontend {{ instance_parameter['configuration.frontend-name'] }}
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter['configuration.slave-introspection-https-port'] }}/
+config-https-only = true
+return = domain secure_access
+
+[slave-introspection-configuration-state]
+<= jinja2-template-base
+template = {{ parameter_dict['template_configuration_state_script'] }}
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+
+path_list = ${frontend-configuration:slave-introspection-configuration} ${frontend-configuration:ip-access-certificate}
+sha256sum = {{ parameter_dict['sha256sum'] }}
+
+extra-context =
+    key path_list :path_list
+    key sha256sum :sha256sum
+    key signature_file :signature_file
+
+[slave-introspection-configuration-state-graceful]
+<= slave-introspection-configuration-state
+signature_file = ${directory:run}/slave_introspection_graceful_configuration_state_signature
+
+[slave-introspection-configuration-state-validate]
+<= slave-introspection-configuration-state
+signature_file = ${directory:run}/slave_introspection_validate_configuration_state_signature
+
+[slave-introspection-graceful]
+< = jinja2-template-base
+template = {{ parameter_dict['template_graceful_script'] }}
+rendered = ${directory:etc-run}/slave-introspection-safe-graceful
+mode = 0700
+
+extra-context =
+    key graceful_reload_command caddy-configuration:slave-introspection-graceful-command
+    key caddy_configuration_state slave-introspection-configuration-state-graceful:rendered
+
+[slave-introspection-validate]
+<= jinja2-template-base
+template = {{ parameter_dict['template_validate_script'] }}
+rendered = ${directory:bin}/slave-introspection-validate
+mode = 0700
+last_state_file = ${directory:run}/slave_introspection_configuration_last_state
+validate_command = {{ parameter_dict['nginx'] }} -c ${frontend-configuration:slave-introspection-configuration} -t
+extra-context =
+    key validate_command :validate_command
+    key configuration_state_command slave-introspection-configuration-state-validate:rendered
+    key last_state_file :last_state_file
+
+[promise-slave-introspection-configuration]
+<= monitor-promise-base
+module = validate_frontend_configuration
+name = slave-introspection-configuration.py
+config-verification-script = ${promise-slave-introspection-configuration-helper:rendered}
+
+[promise-slave-introspection-configuration-helper]
+< = jinja2-template-base
+template = {{ parameter_dict['template_empty'] }}
+rendered = ${directory:bin}/slave-introspection-read-last-configuration-state
+mode = 0700
+content =
+  #!/bin/sh
+  exit `cat ${slave-introspection-validate:last_state_file}`
+context =
+    key content :content
+
+[promise-slave-introspection-https]
+<= monitor-promise-base
+module = check_port_listening
+name = slave_introspection_https.py
+config-hostname = {{ instance_parameter['ipv6-random'] }}
+config-port = ${frontend-configuration:slave-introspection-https-port}
+
+[logrotate-entry-slave-introspection]
+<= logrotate-entry-base
+name = slave-introspection
+log = ${backend-haproxy-rsyslogd-config:log-file}
+rotate-num = ${configuration:rotate-num}
+post = kill -USR1 $(cat ${caddy-configuration:slave-introspection-pid-file})
+
 [configuration]
 {%- for key, value in instance_parameter.iteritems() -%}
 {%-   if key.startswith('configuration.') %}

software/caddy-frontend/instance.cfg.in

@@ -41,6 +41,7 @@ template = {{ template_caddy_frontend }}
 filename = instance-caddy-frontend.cfg
 extensions = jinja2.ext.do
 extra-context =
+  import furl_module furl
   section parameter_dict dynamic-template-caddy-frontend-parameters
   raw software_type single-custom-personal
 caucase-jinja2-library = {{ caucase_jinja2_library }}
@@ -124,3 +125,4 @@ configuration.backend-haproxy-http-port = 21080
 configuration.backend-haproxy-https-port = 21443
 configuration.authenticate-to-backend = False
 configuration.rotate-num = 4000
+configuration.slave-introspection-https-port = 22443

software/caddy-frontend/setup.py

@@ -7,6 +7,8 @@ setup(
   name='caddyprofiledeps',
   install_requires=[
     'validators',
+    'furl',
+    'orderedmultidict',
   ],
   entry_points={
     'zc.buildout': [

software/caddy-frontend/software.cfg

@@ -27,3 +27,5 @@ smmap = 0.9.0
 numpy = 1.16.4
 
 websockify = 0.8.0
+furl = 2.1.0
+orderedmultidict = 1.0.1

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -36,6 +36,7 @@ context =
 # empty sections if no slaves are available
 [slave-log-directory-dict]
 [slave-password]
+[slave-htpasswd]
 
 {#- Loop thought slave list to set up slaves #}
 {%- set DEFAULT_PORT = {'http': 80, 'https': 443, '': None} %}
@@ -94,6 +95,7 @@ context =
 {%-   set slave_configuration_section_name = 'slave-instance-%s-configuration' % slave_reference %}
 {%-   set slave_logrotate_section = slave_reference + "-logs" %}
 {%-   set slave_password_section = slave_reference + "-password" %}
+{%-   set slave_htpasswd_section = slave_reference + "-htpasswd" %}
 {%-   set slave_ln_section = slave_reference + "-ln" %}
 {#-   extend parts #}
 {%-   do part_list.extend([slave_ln_section]) %}
@@ -118,7 +120,12 @@ context =
 {%-   do slave_instance.__setitem__('backend_log', slave_parameter_dict.get('backend_log')) %}
 {#-   Add slave log directory to the slave log access dict #}
 {%-   do slave_log_dict.__setitem__(slave_reference, slave_log_folder) %}
-{%-   set slave_log_access_url = 'https://' + slave_reference.lower() + ':${'+ slave_password_section +':passwd}@[' + frontend_configuration.get('caddy-ipv6') + ']:' + frontend_configuration.get('caddy-https-port') + '/' + slave_reference.lower() + '/' %}
+{%-   set furled = furl_module.furl(frontend_configuration['slave-introspection-secure_access']) %}
+{%-   do furled.set(username = slave_reference.lower()) %}
+{%-   do furled.set(password = '${'+ slave_password_section +':passwd}') %}
+{%-   do furled.set(path = slave_reference.lower() + '/') %}
+{#-   We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
+{%-   set slave_log_access_url = urlparse_module.unquote(furled.tostr()) %}
 {%-   do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
 {%-   do slave_publish_dict.__setitem__('slave-reference', slave_reference) %}
 {%-   do slave_publish_dict.__setitem__('public-ipv4', public_ipv4) %}
@@ -139,6 +146,9 @@ context =
 [slave-password]
 {{ slave_reference }} = {{ '${' + slave_password_section + ':passwd}' }}
 
+[slave-htpasswd]
+{{ slave_reference }} = {{ '${' + slave_htpasswd_section + ':file}' }}
+
 {#-   Set slave logrotate entry #}
 [{{slave_logrotate_section}}]
 <= logrotate-entry-base
@@ -162,6 +172,13 @@ recipe = slapos.cookbook:generate.password
 storage-path = {{caddy_configuration_directory}}/.{{slave_reference}}.passwd
 bytes = 8
 
+[{{ slave_htpasswd_section }}]
+recipe = plone.recipe.command
+stop-on-error = True
+file = {{ caddy_configuration_directory }}/.{{ slave_reference }}.htpasswd
+command = {{ frontend_configuration['htpasswd'] }} -cb ${:file} {{ slave_reference.lower() }} {{ '${' + slave_password_section + ':passwd}' }}
+update-command = ${:command}
+
 {#-   ################################################## #}
 {#-   Set Slave Certificates if needed                   #}
 {#-   Set certificate key for custom configuration       #}
@@ -287,6 +304,7 @@ recipe = slapos.cookbook:mkdirectory
 {%- do part_list.append('slave-log-directories') %}
 
 {%- do part_list.append('caddy-log-access') %}
+{%- do part_list.append('slave-introspection') %}
 {#- ############################################## #}
 {#- ## Prepare virtualhost for slaves using cache  #}
 {#- Define IPv6 to IPV4 tunneling #}
@@ -331,6 +349,35 @@ extra-context =
     section slave_password slave-password
     section parameter_dict caddy-log-access-parameters
 
+[slave-introspection-parameters]
+local-ipv4 = {{ dumps(local_ipv4) }}
+global-ipv6 = {{ dumps(global_ipv6) }}
+https-port = {{ frontend_configuration['slave-introspection-https-port'] }}
+ip-access-certificate = {{ frontend_configuration.get('ip-access-certificate') }}
+nginx-mime = {{ frontend_configuration['nginx_mime'] }}
+access-log = {{ dumps(caddy_configuration['slave-introspection-access-log']) }}
+error-log = {{ dumps(caddy_configuration['slave-introspection-error-log']) }}
+var = {{ directory['slave-introspection-var'] }}
+pid = {{ caddy_configuration['slave-introspection-pid-file'] }}
+
+[slave-introspection-config]
+<= jinja2-template-base
+template = {{ frontend_configuration['slave-introspection-template'] }}
+rendered = {{ frontend_configuration['slave-introspection-configuration'] }}
+extra-context =
+    section slave_log_directory slave-log-directory-dict
+    section slave_htpasswd slave-htpasswd
+    section parameter_dict slave-introspection-parameters
+
+[slave-introspection]
+recipe = slapos.cookbook:wrapper
+command-line = {{ frontend_configuration['nginx'] }}
+  -c ${slave-introspection-config:rendered}
+
+wrapper-path = {{ directory['service'] }}/slave-instrospection-nginx
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+
 
 {#- Publish information for the instance #}
 [publish-caddy-information]

software/caddy-frontend/templates/slave-introspection-httpd-nginx.conf.in

+# Access log configuration
+daemon off;
+pid {{ parameter_dict['pid'] }};
+error_log {{ parameter_dict['error-log'] }};
+
+events {
+}
+
+http {
+  include {{ parameter_dict['nginx-mime'] }};
+  server {
+    server_name_in_redirect off;
+    port_in_redirect off;
+    error_log {{ parameter_dict['error-log'] }};
+    access_log {{ parameter_dict['access-log'] }};
+    listen [{{ parameter_dict['global-ipv6'] }}]:{{ parameter_dict['https-port'] }} ssl;
+    listen {{ parameter_dict['local-ipv4'] }}:{{ parameter_dict['https-port'] }} ssl;
+    ssl_certificate {{ parameter_dict['ip-access-certificate'] }};
+    ssl_certificate_key {{ parameter_dict['ip-access-certificate'] }};
+    default_type application/octet-stream;
+    client_body_temp_path {{ parameter_dict['var'] }} 1 2;
+    proxy_temp_path {{ parameter_dict['var'] }} 1 2;
+    fastcgi_temp_path {{ parameter_dict['var'] }} 1 2;
+    uwsgi_temp_path {{ parameter_dict['var'] }} 1 2;
+    scgi_temp_path {{ parameter_dict['var'] }} 1 2;
+  {% for slave, directory in slave_log_directory.iteritems() %}
+    location /{{ slave }} {
+      alias {{ directory }};
+      autoindex on;
+      autoindex_format json;
+      sendfile on;
+      sendfile_max_chunk 1m;
+      auth_basic "Log Access {{ slave }}";
+      auth_basic_user_file "{{ slave_htpasswd[slave] | trim }}";
+    }
+  {% endfor %}
+  }
+}

software/caddy-frontend/test/test.py

@@ -674,6 +674,15 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
       result.status_code,
       'While accessing %r of %r the status code was %r' % (
         url, frontend, result.status_code))
+    # check that the result is correct JSON, which allows to access
+    # information about all logs
+    self.assertEqual(
+      'application/json',
+      result.headers['Content-Type']
+    )
+    self.assertEqual(
+      sorted([q['name'] for q in result.json()]),
+      ['access.log', 'backend.log', 'error.log'])
     self.assertEqual(
       httplib.OK,
       requests.get(url + 'access.log', verify=False).status_code
@@ -684,7 +693,8 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     )
     # assert only for few tests, as backend log is not available for many of
     # them, as it's created on the fly
-    for test_name in ['test_url', 'test_auth_to_backend', 'test_compressed_result']:
+    for test_name in [
+      'test_url', 'test_auth_to_backend', 'test_compressed_result']:
       if self.id().endswith(test_name):
         self.assertEqual(
           httplib.OK,

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_log-CADDY.txt

@@ -11,5 +11,7 @@ T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_log-CADDY.txt

@@ -11,5 +11,7 @@ T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_log-CADDY.txt

@@ -13,5 +13,7 @@ T-2/var/log/httpd/_default_access_log
 T-2/var/log/httpd/_default_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -177,5 +177,7 @@ T-2/var/log/httpd/_url_https-url_backend_log
 T-2/var/log/httpd/_url_https-url_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_plugin-CADDY.txt

@@ -38,5 +38,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_log-CADDY.txt

@@ -17,5 +17,7 @@ T-2/var/log/httpd/_own_ciphers_backend_log
 T-2/var/log/httpd/_own_ciphers_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -177,5 +177,7 @@ T-2/var/log/httpd/_url_https-url_backend_log
 T-2/var/log/httpd/_url_https-url_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -38,5 +38,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_log-CADDY.txt

@@ -47,5 +47,7 @@ T-2/var/log/httpd/_type-notebook-ssl_from_slave_kedifa_overrides_backend_log
 T-2/var/log/httpd/_type-notebook-ssl_from_slave_kedifa_overrides_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_log-CADDY.txt

@@ -14,5 +14,7 @@ T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_master_certificate_backend_l
 T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_master_certificate_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_log-CADDY.txt

@@ -14,5 +14,7 @@ T-2/var/log/httpd/_ssl_from_master_backend_log
 T-2/var/log/httpd/_ssl_from_master_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED