Merge remote-tracking branch 'origin/master' into t

component/mariadb/buildout.cfg

@@ -125,3 +125,34 @@ environment =
   CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include
   LDFLAGS=-L${groonga:location}/lib
   PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig
+
+
+### XXX keep using mariadb 10.3.22 because 10.4 cause crash
+### (we just override here for easier revert)
+[mariadb]
+version = 10.3.22
+url = https://downloads.mariadb.org/f/mariadb-10.3.22/source/mariadb-10.3.22.tar.gz/from/http%3A//ftp.hosteurope.de/mirror/archive.mariadb.org/?serve
+md5sum = f712a5e6fde038d0c9c6d2a2cd88b84e
+pre-configure =
+  set -e '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
+  grep -q "$@"
+  sed -i "/$1/d" "$2"
+post-install =
+  ldd=`ldd ${:location}/lib/plugin/ha_rocksdb.so`
+  for x in ${lz4:location} ${snappy:location} ${zstd:location}
+  do echo "$ldd" |grep -qF " $x/lib/"
+  done
+  mkdir -p ${:location}/include/wsrep &&
+  cp -p wsrep/wsrep_api.h ${:location}/include/wsrep
+
+[mroonga-mariadb]
+pre-configure =
+  rm -rf fake_mariadb_source
+  mkdir -p fake_mariadb_source
+  cd fake_mariadb_source
+  ln -s ${mariadb:location}/wsrep-lib
+  cp -a ${mariadb:location}/include/mysql/server include
+  cp -a include/private sql
+  chmod -R a+w include sql # so that buildout can delete this compile-dir after install
+  mkdir -p ${:plugin-dir}
+### /XXX keep using mariadb 10.3.22 because 10.4 cause crash

component/qemu-kvm/buildout.cfg

@@ -72,8 +72,8 @@ md5sum = 6097fdb9cbab47c96471274b9044e983
 # XXX: This is not the latest version because
 #      Debian does not provide a stable URL for it.
 <= debian-amd64-netinst-base
-version = 10.4.0
-md5sum = e2ddc8268e4c164c32b4ba25be52c9af
+version = 10.5.0
+md5sum = a3ebc76aec372808ad80000108a2593a
 
 [debian-amd64-testing-netinst.iso]
 <= debian-amd64-netinst-base

component/trafficserver/buildout.cfg

 [buildout]
 extends =
+  ../defaults.cfg
   ../libtool/buildout.cfg
   ../libxml2/buildout.cfg
   ../make/buildout.cfg
@@ -14,10 +15,15 @@ extends =
 parts =
   trafficserver
 
+[gcc]
+# For ATS v8.0.0 and later, a compiler with support for C++17 is required. 
+# https://cwiki.apache.org/confluence/display/TS/Building
+min_version = 8
+
 [trafficserver]
 recipe = slapos.recipe.cmmi
-url = http://apache.claz.org/trafficserver/trafficserver-7.1.11.tar.bz2
-md5sum = 74e8c1e2d7cdc0b97420d8b813321258
+url = http://apache.claz.org/trafficserver/trafficserver-8.1.0.tar.bz2
+md5sum = 99bfeb61095e55cb151ef58d884cb3f1
 shared = true
 configure-options =
   --with-openssl=${openssl:location}
@@ -33,6 +39,7 @@ configure-options =
 environment =
   PATH=${libtool:location}/bin:${make:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   LDFLAGS =-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${tcl:location}/lib -Wl,-rpath=${tcl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  
 make-target =
   check
   install

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = 30eb6db0d2df9ffb3a50fb64e73df5be
+md5sum = bfb647325103640c19c38e7b7f6e2833
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = c09755397fbb9c8d831551f1de7bc6c3
+md5sum = 02aac183352a6fd6ddd336d2e3757405
 
 [template-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = b684aacbb7b2a6eec3ca3ab06c61a92c
+md5sum = 54c0648c8593699dae0c565bc7dd8629
 
 [template-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = e9c5fe132a26e960df0acf3f5d8f877e
+md5sum = 2690fb2b5b6cefb7de53f35c214bdd52
 
 [template-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -54,7 +54,7 @@ md5sum = 266f175dbdfc588af7a86b0b1884fe73
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = eaf7491f628e72c877c31588e62eeb89
+md5sum = 80081a7ded0029bb24b4fe2d06b6ae95
 
 [template-log-access]
 _update_hash_filename_ = templates/template-log-access.conf.in
@@ -70,15 +70,15 @@ md5sum = 975177dedf677d24e14cede5d13187ce
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = 56c7e9f5cc92f8c519f48496010e2078
+md5sum = 18076ae37306aca1f6ef11e2173c887f
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2
-md5sum = baf7b89cc9ab5506100b0c900808c1ea
+md5sum = d022455a8610bac2dd51101edb035987
 
-[template-trafficserver-logging-config]
-_update_hash_filename_ = templates/trafficserver/logging.config.jinja2
-md5sum = 6aed31174dc262ced02f31624321df41
+[template-trafficserver-logging-yaml]
+_update_hash_filename_ = templates/trafficserver/logging.yaml.jinja2
+md5sum = 45f379e887de07d2b86de2f43937f856
 
 [template-nginx-eventsource-slave-virtualhost]
 _update_hash_filename_ = templates/nginx-eventsource-slave.conf.in

software/caddy-frontend/common.cfg

@@ -131,7 +131,7 @@ template_not_found_html = ${template-not-found-html:target}
 template_slave_list = ${template-slave-list:target}
 template_trafficserver_records_config = ${template-trafficserver-records-config:target}
 template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
-template_trafficserver_logging_config = ${template-trafficserver-logging-config:target}
+template_trafficserver_logging_yaml = ${template-trafficserver-logging-yaml:target}
 template_wrapper = ${template-wrapper:output}
 template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
 
@@ -218,7 +218,7 @@ mode = 0644
 [template-trafficserver-storage-config]
 <=download-template
 
-[template-trafficserver-logging-config]
+[template-trafficserver-logging-yaml]
 <=download-template
 
 [template-rotate-script]

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -33,7 +33,7 @@ parts =
   trafficserver-plugin-config
   trafficserver-storage-config
   trafficserver-ip-allow-config
-  trafficserver-logging-config
+  trafficserver-logging-yaml
   trafficserver-promise-listen-port
   trafficserver-promise-cache-availability
   cron-entry-logrotate-trafficserver
@@ -484,8 +484,6 @@ plugin-config =
 ip-allow-config = src_ip=0.0.0.0-255.255.255.255 action=ip_allow
 cache-path = ${trafficserver-directory:cache-path}
 disk-cache-size = ${configuration:disk-cache-size}
-synthetic-port = ${configuration:trafficserver-synthetic-port}
-mgmt-port = ${configuration:trafficserver-mgmt-port}
 ram-cache-size = ${configuration:ram-cache-size}
 templates-dir = {{ parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
 request-timeout = ${configuration:request-timeout}
@@ -497,7 +495,7 @@ target = ${trafficserver-directory:configuration}
 
 [trafficserver-launcher]
 recipe = slapos.cookbook:wrapper
-command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_cop
+command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_manager
 wrapper-path = ${trafficserver-variable:wrapper-path}
 environment = TS_ROOT=${buildout:directory}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
@@ -531,10 +529,10 @@ extra-context =
 template = {{ parameter_dict['template_trafficserver_storage_config'] }}
 filename = storage.config
 
-[trafficserver-logging-config]
+[trafficserver-logging-yaml]
 < = trafficserver-jinja2-template-base
-template = {{ parameter_dict['template_trafficserver_logging_config'] }}
-filename = logging.config
+template = {{ parameter_dict['template_trafficserver_logging_yaml'] }}
+filename = logging.yaml
 
 [trafficserver-remap-config]
 <= trafficserver-jinja2-template-base
@@ -740,6 +738,13 @@ certificate = ${backend-client-login-config:certificate}
 cas-ca-certificate = ${backend-client-login-config:cas-ca-certificate}
 csr = ${backend-client-caucase-updater-csr:csr}
 crl = ${backend-client-login-config:crl}
+# the statistic page
+statistic-certificate = ${self-signed-ip-access:certificate}
+statistic-port = ${configuration:backend-haproxy-statistic-port}
+statistic-username = ${monitor-instance-parameter:username}
+statistic-password = ${monitor-htpasswd:passwd}
+statistic-identification = {{ slapparameter_dict['cluster-identification'] }}
+statistic-frontend-secure_access = ${backend-haproxy-statistic-frontend:connection-secure_access}
 
 [backend-haproxy]
 recipe = slapos.cookbook:wrapper
@@ -932,6 +937,16 @@ config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_param
 config-https-only = true
 return = domain secure_access
 
+[backend-haproxy-statistic-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Backend Haproxy Statistic Frontend {{ instance_parameter['configuration.frontend-name'] }}
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter['configuration.backend-haproxy-statistic-port'] }}/
+config-https-only = true
+return = domain secure_access
+
 [slave-introspection-configuration-state]
 <= jinja2-template-base
 template = {{ parameter_dict['template_configuration_state_script'] }}

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -215,7 +215,7 @@ config-monitor-username = ${monitor-instance-parameter:username}
 config-monitor-password = ${monitor-htpasswd:passwd}
 
 software-type = {{frontend_type}}
-return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url backend-client-csr_id-url csr_id-url csr_id-certificate
+return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url backend-client-csr_id-url csr_id-url csr_id-certificate backend-haproxy-statistic-url
 
 {% for section, frontend_request in request_dict.iteritems() %}
 {%   set state = frontend_request.get('state', '') %}
@@ -292,6 +292,10 @@ kedifa-csr_id-certificate = ${request-kedifa:connection-csr_id-certificate}
 {{ frontend }}-csr_id-url = {{ section_part }}:connection-csr_id-url}
 {% endfor %}
 {% endif %}
+{% for frontend in frontend_list %}
+  {% set section_part = '${request-' + frontend %}
+{{ frontend }}-backend-haproxy-statistic-url = {{ section_part }}:connection-backend-haproxy-statistic-url}
+{% endfor %}
 {% if not aibcc_enabled %}
 {% for frontend in frontend_list %}
   {% set section_part = '${request-' + frontend %}

software/caddy-frontend/instance.cfg.in

@@ -110,8 +110,6 @@ configuration.apache-certificate =
 configuration.open-port = 80 443
 configuration.disk-cache-size = 8G
 configuration.ram-cache-size = 1G
-configuration.trafficserver-synthetic-port = 8083
-configuration.trafficserver-mgmt-port = 8084
 configuration.re6st-verification-url = http://[2001:67c:1254:4::1]/index.html
 configuration.enable-http2-by-default = true
 configuration.global-disable-http2 = false
@@ -123,6 +121,7 @@ configuration.backend-connect-timeout = 5
 configuration.backend-connect-retries = 3
 configuration.backend-haproxy-http-port = 21080
 configuration.backend-haproxy-https-port = 21443
+configuration.backend-haproxy-statistic-port = 21444
 configuration.authenticate-to-backend = False
 configuration.rotate-num = 4000
 configuration.slave-introspection-https-port = 22443

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -394,6 +394,13 @@ monitor-base-url = {{ monitor_base_url }}
 csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/csr_id.txt
 backend-client-csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/backend-haproxy-csr_id.txt
 csr_id-certificate = ${get-csr_id-certificate:certificate}
+{%-   set furled = furl_module.furl(backend_haproxy_configuration['statistic-frontend-secure_access']) %}
+{%-   do furled.set(username = backend_haproxy_configuration['statistic-username']) %}
+{%-   do furled.set(password = backend_haproxy_configuration['statistic-password']) %}
+{%-   do furled.set(path = '/') %}
+{#-   We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
+{%-   set statistic_url = urlparse_module.unquote(furled.tostr()) %}
+backend-haproxy-statistic-url = {{ statistic_url }}
 
 [kedifa-updater]
 recipe = slapos.cookbook:wrapper
@@ -442,16 +449,14 @@ extra-context =
   section configuration backend-haproxy-config
 
 [backend-haproxy-config]
-file = {{ backend_haproxy_configuration['file'] }}
-pid-file = {{ backend_haproxy_configuration['pid-file'] }}
-log-socket = {{ backend_haproxy_configuration['log-socket'] }}
+{%- for key, value in backend_haproxy_configuration.items() %}
+{{ key }} = {{ value }}
+{%- endfor %}
 local-ipv4 = {{ dumps('' ~ local_ipv4) }}
-http-port = {{ ('' ~ backend_haproxy_configuration['http-port']) }}
-https-port = {{ ('' ~ backend_haproxy_configuration['https-port']) }}
+global-ipv6 = ${slap-network-information:global-ipv6}
 request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
 backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
 backend-connect-retries =  {{ dumps('' ~ configuration['backend-connect-retries']) }}
-certificate = {{ dumps('' ~ backend_haproxy_configuration['certificate']) }}
 
 [store-backend-haproxy-csr_id]
 recipe = plone.recipe.command

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -38,6 +38,15 @@ defaults
 {%-   endif %}
 {%- endmacro %}
 
+# statistic
+frontend statistic
+  bind {{ configuration['global-ipv6']}}:{{ configuration['statistic-port'] }} ssl crt {{ configuration['statistic-certificate'] }}
+  stats enable
+  stats uri /
+  stats show-desc {{ configuration['statistic-identification'] }}
+  stats auth {{ configuration['statistic-username'] }}:{{ configuration['statistic-password'] }}
+  stats realm {{ configuration['statistic-identification'] }}
+
 frontend http-backend
   bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
 {%- for slave_instance in backend_slave_list %}

software/caddy-frontend/templates/trafficserver/logging.config.jinja2

-squid = format {
-   Format = '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<cluc> %<caun> %<phr>/%<pqsn> %<psct>'
-}
-log.ascii {
-   Format = squid,
-   Filename = 'squid',
-   RollingEnabled = 1,
-   RollingIntervalSec = 86400,
-   RollingOffsetHr = 0
-}

software/caddy-frontend/templates/trafficserver/logging.yaml.jinja2

+formats:
+  - name: squid
+    format: '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<cluc> %<caun> %<phr>/%<shn> %<psct>'
+logs:
+  - filename: squid
+    format: squid
+    mode: ascii
+    rolling_enabled: 1
+    rolling_interval_sec: 86400
+    rolling_offset_hr: 0

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -5,7 +5,7 @@
 ##############################################################################
 
 ##############################################################################
-# SlapOS Specific configuration
+# SlapOS Specific configuration not available in default records.config
 ##############################################################################
 CONFIG proxy.config.proxy_name STRING {{ ats_configuration['hostname'] }}
 CONFIG proxy.config.local_state_dir STRING {{ ats_directory['local-state'] }}
@@ -18,8 +18,6 @@ CONFIG proxy.config.output.logfile STRING traffic.out
 CONFIG proxy.config.snapshot_dir STRING snapshots
 CONFIG proxy.config.admin.user_id STRING {{ '#%s' % os_module.geteuid() }}
 CONFIG proxy.config.admin.number_config_bak INT 0
-CONFIG proxy.config.process_manager.mgmt_port INT {{ ats_configuration['mgmt-port'] }}
-CONFIG proxy.config.admin.synthetic_port INT {{ ats_configuration['synthetic-port'] }}
 LOCAL proxy.local.incoming_ip_to_bind STRING {{ ats_configuration['local-ip'] }}
 CONFIG proxy.config.log.logfile_dir STRING {{ ats_directory['log'] }}
 # Implement RFC 5861 with core
@@ -28,14 +26,6 @@ CONFIG proxy.config.body_factory.template_sets_dir STRING  {{ ats_configuration[
 # Support stale-if-error by returning cached content on backend 5xx or unavailability
 CONFIG proxy.config.http.negative_revalidating_enabled INT 1
 CONFIG proxy.config.http.negative_revalidating_lifetime INT 86400
-##############################################################################
-# Proxy users variables. Docs:
-#    https://docs.trafficserver.apache.org/records.config#proxy-user-variables
-##############################################################################
-# Do not modify headers, as it needlessly pollutes information
-CONFIG proxy.config.http.insert_client_ip INT 0
-CONFIG proxy.config.http.insert_squid_x_forwarded_for INT 0
-
 
 ##############################################################################
 # Thread configurations. Docs:
@@ -80,7 +70,7 @@ CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1
 CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120
 CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120
 CONFIG proxy.config.http.transaction_no_activity_timeout_in INT {{ ats_configuration['request-timeout'] }}
-CONFIG proxy.config.http.transaction_no_activity_timeout_out INT {{ ats_configuration['request-timeout'] }}
+CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 30 {{ ats_configuration['request-timeout'] }}
 CONFIG proxy.config.http.transaction_active_timeout_in INT 900
 CONFIG proxy.config.http.transaction_active_timeout_out INT 0
 CONFIG proxy.config.http.accept_no_activity_timeout INT 120
@@ -107,6 +97,13 @@ CONFIG proxy.config.http.down_server.abort_threshold INT 10
 CONFIG proxy.config.http.negative_caching_enabled INT 0
 CONFIG proxy.config.http.negative_caching_lifetime INT 1800
 
+##############################################################################
+# Proxy users variables. Docs:
+#    https://docs.trafficserver.apache.org/records.config#proxy-user-variables
+##############################################################################
+CONFIG proxy.config.http.insert_client_ip INT 0
+CONFIG proxy.config.http.insert_squid_x_forwarded_for INT 0
+
 ##############################################################################
 # Security. Docs:
 #    https://docs.trafficserver.apache.org/records.config#security
@@ -125,7 +122,7 @@ CONFIG proxy.config.http.cache.http INT 1
 #    https://docs.trafficserver.apache.org/en/latest/admin-guide/files/cache.config.en.html
 ##############################################################################
 CONFIG proxy.config.http.cache.ignore_client_cc_max_age INT 1
-CONFIG proxy.config.http.normalize_ae_gzip INT 0
+CONFIG proxy.config.http.normalize_ae INT 0
 CONFIG proxy.config.http.cache.cache_responses_to_cookies INT 1
 CONFIG proxy.config.http.cache.cache_urls_that_look_dynamic INT 1
     # https://docs.trafficserver.apache.org/records.config#proxy-config-http-cache-when-to-revalidate
@@ -165,7 +162,7 @@ CONFIG proxy.config.cache.min_average_object_size INT 8000
 ##############################################################################
 # Logging Config. Docs:
 #    https://docs.trafficserver.apache.org/records.config#logging-configuration
-#    https://docs.trafficserver.apache.org/en/latest/admin-guide/files/logging.config.en.html
+#    https://docs.trafficserver.apache.org/en/latest/admin-guide/files/logging.yaml.en.html
 ##############################################################################
 CONFIG proxy.config.log.logging_enabled INT 3
 CONFIG proxy.config.log.max_space_mb_for_logs INT 25000
@@ -194,14 +191,14 @@ CONFIG proxy.config.reverse_proxy.enabled INT 1
 ##############################################################################
 CONFIG proxy.config.ssl.client.verify.server INT 0
 CONFIG proxy.config.ssl.client.CA.cert.filename STRING NULL
-CONFIG proxy.config.ssl.server.cipher_suite STRING ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+CONFIG proxy.config.ssl.server.cipher_suite STRING ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
 
 ##############################################################################
 # Debugging. Docs:
 #    https://docs.trafficserver.apache.org/records.config#diagnostic-logging-configuration
 ##############################################################################
 CONFIG proxy.config.diags.debug.enabled INT 0
-CONFIG proxy.config.diags.debug.tags STRING http.*|dns.*
+CONFIG proxy.config.diags.debug.tags STRING http|dns
 # ToDo: Undocumented
 CONFIG proxy.config.dump_mem_info_frequency INT 0
 CONFIG proxy.config.http.slow.log.threshold INT 0

software/caddy-frontend/templates/trafficserver/storage.config.jinja2

@@ -12,4 +12,42 @@
 # Using a file for the cache storage
 #
 # <pathname> <size>
+#
+# Where 'pathname' is full path to the directory where you want
+# the cache-file to live and 'size' is size in bytes
+#
+# Example: 128MB cache file(/opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340/var/trafficserver/cache.db)
+#      /opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340/var/trafficserver 128M
+#
+# Example: 144MB cache file(/opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340/var/trafficserver/cache.db)
+#          assuming prefix of '/opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340'
+#      var/trafficserver 150994944
+#
+# Example: 512MB cache file(/opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340/var/trafficserver/cache.db)
+#          assuming prefix of '/opt/slapgrid/shared/trafficserver/4e294bc2396af9b22bee270330a1f340'
+#      var/trafficserver 512M
+#
+#
+#############################################################
+##              O_DIRECT Specific Configuration            ##
+#############################################################
+#
+# Examples: Using O_DIRECT on disks (Linux kernel >= 2.6.3,
+# FreeBSD > 5.3)
+#
+#      /dev/disc/by-id/[Insert_ID_Here_12345]         		# Linux
+#      /dev/disc/by-path/[Insert-Path-Here:12:34:56-1.0.0.0]	# Linux
+#
+#      /dev/ada1            					# FreeBSD
+#
+# Note that disks are identified by id or path. This is to prevent changes
+# by the kernel (which could occur if a disk was simply described as /dev/sda, sdb, etc.).
+#
+# Also note that when using these raw devices in O_DIRECT mode, you
+# do not need to specify the partition size. It's automatically
+# detected.
+#
+# A small default cache (256MB). This is set to allow for the regression test to succeed
+# most likely you'll want to use a larger cache. And, we definitely recommend the use
+# of raw devices for production caches.
 {{ ats_configuration.get("cache-path") }} {{ ats_configuration.get("disk-cache-size") }}

software/caddy-frontend/test/test.py

@@ -48,6 +48,7 @@ from slapos.recipe.librecipe import generateHashFromFiles
 import xml.etree.ElementTree as ET
 import urlparse
 import socket
+import sqlite3
 
 
 try:
@@ -307,12 +308,13 @@ class TestDataMixin(object):
       'trafficserver/.diags.log.meta',
       'trafficserver/.manager.log.meta',
       'trafficserver/.squid.log.meta',
-      'trafficserver/.traffic.out.meta',
       'trafficserver/diags.log',
       'trafficserver/squid.log',
       # not important, appears sometimes
       'trafficserver/.error.log.meta',
       'trafficserver/error.log',
+      'trafficserver/.traffic.out.meta',
+      'trafficserver/traffic.out',
     ])
 
   def test_file_list_run(self):
@@ -725,6 +727,27 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
 
     return generate_auth_url, upload_url
 
+  def assertBackendHaproxyStatisticUrl(self, parameter_dict):
+    url_key = 'caddy-frontend-1-backend-haproxy-statistic-url'
+    backend_haproxy_statistic_url_dict = {}
+    for key in parameter_dict.keys():
+      if key.startswith('caddy-frontend') and key.endswith(
+        'backend-haproxy-statistic-url'):
+        backend_haproxy_statistic_url_dict[key] = parameter_dict.pop(key)
+    self.assertEqual(
+      [url_key],
+      backend_haproxy_statistic_url_dict.keys()
+    )
+
+    backend_haproxy_statistic_url = backend_haproxy_statistic_url_dict[url_key]
+    result = requests.get(
+      backend_haproxy_statistic_url,
+      verify=False,
+    )
+    self.assertEqual(httplib.OK, result.status_code)
+    self.assertIn('testing partition 0', result.text)
+    self.assertIn('Statistics Report for HAProxy', result.text)
+
   def assertKeyWithPop(self, key, d):
     self.assertTrue(key in d, 'Key %r is missing in %r' % (key, d))
     d.pop(key)
@@ -1041,6 +1064,7 @@ class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
   def test(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -1071,6 +1095,7 @@ class TestMasterRequest(HttpFrontendTestCase, TestDataMixin):
   def test(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
     self.assertEqual(
@@ -1542,6 +1567,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -3695,7 +3721,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
     )
 
   def test_enable_cache_server_alias(self):
@@ -3737,7 +3763,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
     )
 
     result = fakeHTTPResult(
@@ -3799,7 +3825,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
     )
 
     # check stale-if-error support (assumes stale-while-revalidate is same)
@@ -3851,11 +3877,47 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       self.assertNotEqual(via, None)
       self.assertRegexpMatches(
         via,
-        r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+        r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
       )
     finally:
       self.startServerProcess()
     # END: check stale-if-error support
+    # BEGIN: Check that squid.log is correctly filled in
+    ats_log_file_list = glob.glob(
+      os.path.join(
+        self.instance_path, '*', 'var', 'log', 'trafficserver', 'squid.log'
+      ))
+    if len(ats_log_file_list) == 1:
+      ats_log_file = ats_log_file_list[0]
+    direct_pattern = re.compile(
+      r'.*TCP_MISS/200 .*test-path/deeper.*enablecache.example.com'
+      '.* - DIRECT*')
+    refresh_pattern = re.compile(
+      r'.*TCP_REFRESH_MISS/200 .*test-path/deeper.*enablecache.example.com'
+      '.* - DIRECT*')
+    # ATS needs some time to flush logs
+    timeout = 5
+    b = time.time()
+    while True:
+      direct_pattern_match = 0
+      refresh_pattern_match = 0
+      if (time.time() - b) > timeout:
+        break
+      with open(ats_log_file) as fh:
+        for line in fh.readlines():
+          if direct_pattern.match(line):
+            direct_pattern_match += 1
+          if refresh_pattern.match(line):
+            refresh_pattern_match += 1
+      if direct_pattern_match > 0 and refresh_pattern_match:
+        break
+      time.sleep(0.1)
+
+    with open(ats_log_file) as fh:
+      ats_log = fh.read()
+    self.assertRegexpMatches(ats_log, direct_pattern)
+    self.assertRegexpMatches(ats_log, refresh_pattern)
+    # END: Check that squid.log is correctly filled in
 
   def test_enable_cache_ats_timeout(self):
     parameter_dict = self.assertSlaveBase('enable_cache')
@@ -3976,7 +4038,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
     )
 
     try:
@@ -4023,7 +4085,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/8.1.0\)$'
     )
 
   def test_enable_http2_false(self):
@@ -5192,6 +5254,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -5868,6 +5931,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityUpdate(
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -5974,6 +6038,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -6169,6 +6234,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
     self.assertRejectedSlavePromiseWithPop(parameter_dict)
 
@@ -6683,3 +6749,90 @@ class TestSlaveHostHaproxyClash(SlaveHttpFrontendTestCase, TestDataMixin):
     )
     self.assertEqual(self.certificate_pem, der2pem(result_specific.peercert))
     self.assertEqualResultJson(result_specific, 'Path', '/zspecific/test-path')
+
+
+class TestPassedRequestParameter(HttpFrontendTestCase):
+  # special SRs to check out
+  frontend_2_sr = 'special_sr_for_2'
+  frontend_3_sr = 'special_sr_for_3'
+  kedifa_sr = 'special_sr_for_kedifa'
+
+  @classmethod
+  def setUpClass(cls):
+    super(TestPassedRequestParameter, cls).setUpClass()
+    cls.slap.supply(cls.frontend_2_sr, cls.slap._computer_id)
+    cls.slap.supply(cls.frontend_3_sr, cls.slap._computer_id)
+    cls.slap.supply(cls.kedifa_sr, cls.slap._computer_id)
+
+  instance_parameter_dict = {
+      'port': HTTPS_PORT,
+      'plain_http_port': HTTP_PORT,
+      'kedifa_port': KEDIFA_PORT,
+      'caucase_port': CAUCASE_PORT,
+  }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return cls.instance_parameter_dict
+
+  def test(self):
+    self.instance_parameter_dict.update({
+      '-frontend-quantity': 3,
+      '-sla-2-computer_guid': self.slap._computer_id,
+      '-frontend-2-state': 'stopped',
+      '-frontend-2-software-release-url': self.frontend_2_sr,
+      '-sla-3-computer_guid': self.slap._computer_id,
+      '-frontend-3-state': 'stopped',
+      '-frontend-3-software-release-url': self.frontend_3_sr,
+      '-kedifa-software-release-url': self.kedifa_sr,
+    })
+
+    # re-request instance with updated parameters
+    self.requestDefaultInstance()
+
+    # run once instance, it's only needed for later checks
+    try:
+      self.slap.waitForInstance()
+    except Exception:
+      pass
+
+    # inspect slapproxy, that the master correctly requested other partitions
+    sqlitedb_file = os.path.join(
+      os.path.abspath(
+        os.path.join(
+          self.slap.instance_directory, os.pardir
+        )
+      ), 'var', 'proxy.db'
+    )
+    connection = sqlite3.connect(sqlitedb_file)
+
+    def dict_factory(cursor, row):
+      d = {}
+      for idx, col in enumerate(cursor.description):
+        d[col[0]] = row[idx]
+      return d
+    connection.row_factory = dict_factory
+    cursor = connection.cursor()
+
+    cursor.execute(
+      "select partition_reference, software_release "
+      "from partition14 where slap_state='busy';")
+
+    requested_partition_information = cursor.fetchall()
+
+    base_software_url = self.getSoftwareURL()
+    self.assertEqual(
+      requested_partition_information,
+      [
+        {'software_release': base_software_url,
+         'partition_reference': 'testing partition 0'},
+        {'software_release': self.kedifa_sr,
+         'partition_reference': 'kedifa'},
+        # that one is base, as expected
+        {'software_release': base_software_url,
+         'partition_reference': 'caddy-frontend-1'},
+        {'software_release': self.frontend_2_sr,
+         'partition_reference': 'caddy-frontend-2'},
+        {'software_release': self.frontend_3_sr,
+         'partition_reference': 'caddy-frontend-3'}]
+    )

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_log-CADDY.txt

@@ -26,4 +26,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -26,4 +26,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_log-CADDY.txt

@@ -26,4 +26,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -26,4 +26,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_log-CADDY.txt

@@ -14,4 +14,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_log-CADDY.txt

@@ -14,4 +14,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_log-CADDY.txt

@@ -16,4 +16,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -180,4 +180,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_log-CADDY.txt

@@ -20,4 +20,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -180,4 +180,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveHostHaproxyClash.test_file_list_log-CADDY.txt

@@ -20,4 +20,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_log-CADDY.txt

@@ -50,4 +50,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_log-CADDY.txt

@@ -17,4 +17,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_log-CADDY.txt

@@ -17,4 +17,3 @@ T-2/var/log/monitor-httpd-error.log
 T-2/var/log/slave-introspection-access.log
 T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out

software/theia/software.cfg

@@ -120,7 +120,7 @@ eggs =
 version = 0.19.0
 recipe = plone.recipe.command
 command =
-  PATH=${git:location}/bin/:$PATH  bash -c "${python3:executable} -m venv ${:location} && \
+  PATH=${git:location}/bin/:$PATH  bash -c "${python3:executable} -m venv --clear ${:location} && \
     . ${:location}/bin/activate && \
     pip install -r ${python-language-server-requirements.txt:output}"
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
@@ -322,7 +322,7 @@ golang = ${golang1.14:location}
 
 [gowork.goinstall]
 command =
-  bash -c ". ${gowork:env.sh} && GO111MODULE=on go get golang.org/x/tools/gopls@v0.4.3"
+  bash -c ". ${gowork:env.sh} && GO111MODULE=on go get golang.org/x/tools/gopls@v0.4.3 && cd ${go_github.com_caddyserver_caddy:location} && GO111MODULE=on go install -v github.com/caddyserver/caddy/..."
 
 
 [cli-utilities]