Commit 7017a46e by Łukasz Nowak Committed by Łukasz Nowak

caddy-frontend: Adapt to ATS 7

Adapted configuration and instantiation to ATS 7.

Deployment:
 * traffic_line has been replaced with traffic_ctl
 * access log, of squid style, is ascii instead of binary, to do so
   logging.config is generated
 * ip_allow.config is configured to allow access from any host
 * RFC 5861 (stale content on error or revalidate) is implemented with core
   instead with deprecated plugin
 * trafficserver-autoconf-port renamed to trafficserver-synthetic-port
 * proxy.config.system.mmap_max removed, as it is not used by the system anymore

Tests:
 * As Via header is not returned to the client, it is dropped from the
   tests, instead its existence in the backend is checked.
 * Promise plugin trafficserver-cache-availability.py is re enabled, as
   it is expected to work immediately.
1 parent 01f30221
Showing 27 changed files with 114 additions and 20 deletions
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum = 9906c7722c4fe7534cb5a397c9692553
md5sum = 111ff0794c90657b658e3d50525e7fed
[template-common]
filename = instance-common.cfg.in
......@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
[template-apache-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = ef1a1b0c1f2466ff81b8d19c212187cf
md5sum = e5fa926695de04c5bff54cd8128227a3
[template-apache-replicate]
filename = instance-apache-replicate.cfg.in
......@@ -78,11 +78,15 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
[template-trafficserver-records-config]
filename = templates/trafficserver/records.config.jinja2
md5sum = 84baef0a49c9a65e8f2d2ffdb8c1d39c
md5sum = 15cbdba7ab7fe3c3d7c26c1718e47bbd
[template-trafficserver-storage-config]
filename = templates/trafficserver/storage.config.jinja2
md5sum = 117238225b3fc3c5b5be381815f44c67
md5sum = baf7b89cc9ab5506100b0c900808c1ea
[template-trafficserver-logging-config]
filename = templates/trafficserver/logging.config.jinja2
md5sum = cd6bb9bd0734f17469b0ca88f8b1a531
[template-nginx-configuration]
filename = templates/nginx.cfg.in
......
......@@ -136,6 +136,8 @@ template_trafficserver_records_config_filename = ${template-trafficserver-record
template_trafficserver_records_config_location = ${template-trafficserver-records-config:location}
template_trafficserver_storage_config_filename = ${template-trafficserver-storage-config:filename}
template_trafficserver_storage_config_location = ${template-trafficserver-storage-config:location}
template_trafficserver_logging_config_filename = ${template-trafficserver-logging-config:filename}
template_trafficserver_logging_config_location = ${template-trafficserver-logging-config:location}
template_wrapper = ${template-wrapper:output}
[template]
......@@ -243,6 +245,11 @@ filename = records.config.jinja2
url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
filename = storage.config.jinja2
[template-trafficserver-logging-config]
<=download-template
url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
filename = logging.config.jinja2
# NGINX Configuration
[template-nginx-configuration]
recipe = slapos.recipe.template
......
......@@ -44,6 +44,8 @@ parts =
trafficserver-remap-config
trafficserver-plugin-config
trafficserver-storage-config
trafficserver-ip-allow-config
trafficserver-logging-config
trafficserver-promise-listen-port
trafficserver-promise-cache-availability
......@@ -495,12 +497,14 @@ hostname = ${configuration:frontend-name}
remap = map /HTTPS/ http://{{ instance_parameter['ipv4-random'] }}:${caddy-configuration:ssl-cache-through-port}
map / http://{{ instance_parameter['ipv4-random'] }}:${caddy-configuration:cache-through-port}
plugin-config = {{ parameter_dict['trafficserver'] }}/libexec/trafficserver/rfc5861.so
plugin-config =
ip-allow-config = src_ip=0.0.0.0-255.255.255.255 action=ip_allow
cache-path = ${trafficserver-directory:cache-path}
disk-cache-size = ${configuration:disk-cache-size}
autoconf-port = ${configuration:trafficserver-autoconf-port}
synthetic-port = ${configuration:trafficserver-synthetic-port}
mgmt-port = ${configuration:trafficserver-mgmt-port}
ram-cache-size = ${configuration:ram-cache-size}
templates-dir = {{ parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
[trafficserver-configuration-directory]
recipe = plone.recipe.command
......@@ -516,7 +520,7 @@ hash-files = ${buildout:directory}/software_release/buildout.cfg
[trafficserver-reload]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_line -x
command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_ctl config reload
wrapper-path = ${trafficserver-variable:reload-path}
environment = TS_ROOT=${buildout:directory}
......@@ -543,6 +547,11 @@ extra-context =
template = {{ parameter_dict['template_trafficserver_storage_config_location'] }}/{{ parameter_dict['template_trafficserver_storage_config_filename'] }}
filename = storage.config
[trafficserver-logging-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_trafficserver_logging_config_location'] }}/{{ parameter_dict['template_trafficserver_logging_config_filename'] }}
filename = logging.config
[trafficserver-remap-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
......@@ -557,6 +566,13 @@ filename = plugin.config
context =
key content trafficserver-variable:plugin-config
[trafficserver-ip-allow-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
filename = ip_allow.config
context =
key content trafficserver-variable:ip-allow-config
[promise-plugin-base]
recipe = slapos.cookbook:promise.plugin
eggs =
......@@ -573,17 +589,17 @@ name = trafficserver-port-listening.py
config-hostname = ${trafficserver-variable:local-ip}
config-port = ${trafficserver-variable:input-port}
[trafficserver-line]
[trafficserver-ctl]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_line
wrapper-path = ${directory:bin}/traffic_line
command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_ctl
wrapper-path = ${directory:bin}/traffic_ctl
environment = TS_ROOT=${buildout:directory}
[trafficserver-promise-cache-availability]
<= promise-plugin-base
module = trafficserver_cache_availability
name = trafficserver-cache-availability.py
config-wrapper-path = ${trafficserver-line:wrapper-path}
config-wrapper-path = ${trafficserver-ctl:wrapper-path}
### End of ATS sections
......
......@@ -111,7 +111,7 @@ configuration.apache-certificate =
configuration.open-port = 80 443
configuration.disk-cache-size = 8G
configuration.ram-cache-size = 1G
configuration.trafficserver-autoconf-port = 8083
configuration.trafficserver-synthetic-port = 8083
configuration.trafficserver-mgmt-port = 8084
configuration.re6st-verification-url = http://[2001:67c:1254:4::1]/index.html
configuration.enable-http2-by-default = true
......
squid = format {
Format = '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<pqsn> %<psct>'
}
log.ascii {
Format = squid,
Filename = 'squid',
RollingEnabled = 1,
RollingIntervalSec = 3600,
RollingOffsetHr = 0
}
{{ ats_configuration.get("cache-path") }} {{ ats_configuration.get("disk-cache-size") }} volume={{ ats_configuration.get("hostname") }}
\ No newline at end of file
#
# storage.config - Storage Configuration file
#
# Documentation:
# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/storage.config.en.html
#
# The storage configuration is a list of all the storage to
# be used by the cache.
#
#
#############################################################
# Using a file for the cache storage
#
# <pathname> <size>
{{ ats_configuration.get("cache-path") }} {{ ats_configuration.get("disk-cache-size") }}
......@@ -316,10 +316,13 @@ class TestDataMixin(object):
self._test_file_list('log', [
# no control at all when cron would kick in, ignore it
'cron.log',
# appears late, not needed for assertion
# appears late and is quite unstable, no need to assert
'trafficserver/.diags.log.meta',
'trafficserver/.manager.log.meta',
'trafficserver/.squid.log.meta',
'trafficserver/.traffic.out.meta',
'trafficserver/diags.log',
'trafficserver/squid.blog',
'trafficserver/.squid.blog.meta',
'trafficserver/squid.log',
# not important, appears sometimes
'trafficserver/.error.log.meta',
'trafficserver/error.log',
......@@ -352,7 +355,6 @@ class TestDataMixin(object):
ignored_plugin_list = [
'__init__.py', # that's not a plugin
'monitor-http-frontend.py', # can't check w/o functioning frontend
'trafficserver-cache-availability.py', # randomly passing when done
]
runpromise_bin = os.path.join(
self.software_path, 'bin', 'monitor.runpromise')
......@@ -2266,7 +2268,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
headers = result.headers.copy()
self.assertKeyWithPop('Via', headers)
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
......@@ -2539,7 +2540,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
headers = result.headers.copy()
self.assertKeyWithPop('Via', headers)
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
......@@ -2558,6 +2558,14 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
headers
)
backend_headers = result.json()['Incoming Headers']
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.6\)$'
)
result_direct = self.fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
port=26011)
......@@ -2614,7 +2622,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
headers = result.headers.copy()
self.assertKeyWithPop('Via', headers)
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
......@@ -2633,6 +2640,14 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
headers
)
backend_headers = result.json()['Incoming Headers']
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.6\)$'
)
try:
j = result.json()
except Exception:
......@@ -2671,6 +2686,14 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
headers
)
backend_headers = result.json()['Incoming Headers']
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.6\)$'
)
def test_enable_http2_false(self):
parameter_dict = self.assertSlaveBase('enable-http2-false')
......
......@@ -21,4 +21,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: ERROR
T-2/etc/plugin/nginx_frontend_ipv6_http.py: ERROR
T-2/etc/plugin/nginx_frontend_ipv6_https.py: ERROR
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: ERROR
T-2/etc/plugin/trafficserver-port-listening.py: ERROR
\ No newline at end of file
......@@ -21,4 +21,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -25,4 +25,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -25,4 +25,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -25,4 +25,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -25,4 +25,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -21,4 +21,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -19,4 +19,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: ERROR
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -19,4 +19,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: ERROR
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -21,4 +21,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -21,4 +21,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -21,4 +21,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: ERROR
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -21,6 +21,7 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
T-3/etc/plugin/buildout-T-3-status.py: OK
T-3/etc/plugin/caddy_cached.py: ERROR
......@@ -41,4 +42,5 @@ T-3/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-3/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-3/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-3/etc/plugin/re6st-connectivity.py: OK
T-3/etc/plugin/trafficserver-cache-availability.py: ERROR
T-3/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -112,4 +112,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -38,4 +38,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -112,4 +112,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -43,4 +43,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -21,4 +21,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
......@@ -21,4 +21,5 @@ T-2/etc/plugin/nginx_frontend_ipv4_https.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_http.py: OK
T-2/etc/plugin/nginx_frontend_ipv6_https.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
T-2/etc/plugin/trafficserver-cache-availability.py: OK
T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
  • @luke

    • As Via header is not returned to the client, it is dropped from the tests, instead its existence in the backend is checked.

    It's not longer possible to have this again ? I remember sometimes using that header to check if requests were cached

Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!