From 92131e9cdb50818ab71e5f5ea3e50d72f8942d67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 22 Aug 2018 05:20:22 +0200 Subject: [PATCH 1/5] erp5: Zope2 patch to support ipv6 hostnames in VirtualHostMonster https://bugs.launchpad.net/zope2/+bug/699865 --- ...Make-VirtualHostMonster-support-IPv6.patch | 110 ++++++++++++++++++ stack/erp5/buildout.cfg | 3 + 2 files changed, 113 insertions(+) create mode 100644 component/egg-patch/Zope2/0001-SiteAccess-Make-VirtualHostMonster-support-IPv6.patch diff --git a/component/egg-patch/Zope2/0001-SiteAccess-Make-VirtualHostMonster-support-IPv6.patch b/component/egg-patch/Zope2/0001-SiteAccess-Make-VirtualHostMonster-support-IPv6.patch new file mode 100644 index 0000000000..24162f9cdf --- /dev/null +++ b/component/egg-patch/Zope2/0001-SiteAccess-Make-VirtualHostMonster-support-IPv6.patch @@ -0,0 +1,110 @@ +From 4e4ffe940a8927abb0c0c8a2b704eb0a0218bf26 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C5=81ukasz=20Nowak?= +Date: Wed, 22 Aug 2018 03:23:47 +0200 +Subject: [PATCH] SiteAccess: Make VirtualHostMonster support IPv6 + +from https://bugs.launchpad.net/zope2/+bug/699865 : + +VirtualHostMonster does not work with IPv6 named hosts. + +In case of such rewrite configuration: + +RewriteRule (.*) http://10.0.243.129:9280/VirtualHostBase/https/[%{SERVER_ADDR}]:4080$1 [L,P] + +When SERVER_ADDR is fd00::74ba VirtualHostMonster dies with: + +Traceback (most recent call last): + File "/eggs/Zope2-2.12.14-py2.6-linux-x86_64.egg/ZPublisher/BeforeTraverse.py", line 145, in __call__ + meth(*(container, request, None)[:args]) + File "/eggs/Zope2-2.12.14-py2.6-linux-x86_64.egg/Products/SiteAccess/VirtualHostMonster.py", line 154, in __call__ + host, port = host.split(':') +ValueError: too many values to unpack + +This is because IPv6 addresses contain ":" in them. +--- + src/Products/SiteAccess/VirtualHostMonster.py | 7 +++- + .../SiteAccess/tests/testVirtualHostMonster.py | 43 ++++++++++++++++++++++ + 2 files changed, 49 insertions(+), 1 deletion(-) + +diff --git a/src/Products/SiteAccess/VirtualHostMonster.py b/src/Products/SiteAccess/VirtualHostMonster.py +index a455766be..c284df153 100644 +--- a/src/Products/SiteAccess/VirtualHostMonster.py ++++ b/src/Products/SiteAccess/VirtualHostMonster.py +@@ -151,7 +151,12 @@ class VirtualHostMonster(Persistent, Item, Implicit): + protocol = stack.pop() + host = stack.pop() + if ':' in host: +- host, port = host.split(':') ++ if host.startswith('['): ++ # IPv6 address passed ++ host, port = host.rsplit(':', 1) ++ else: ++ # Name or IPv4 address passed ++ host, port = host.split(':') + request.setServerURL(protocol, host, port) + else: + request.setServerURL(protocol, host) +diff --git a/src/Products/SiteAccess/tests/testVirtualHostMonster.py b/src/Products/SiteAccess/tests/testVirtualHostMonster.py +index 6a9e41815..32d7313ce 100644 +--- a/src/Products/SiteAccess/tests/testVirtualHostMonster.py ++++ b/src/Products/SiteAccess/tests/testVirtualHostMonster.py +@@ -139,6 +139,48 @@ for i, (vaddr, vr, _vh, p, ubase) in enumerate(gen_cases()): + + setattr(VHMRegressions, 'testTraverse%s' % i, test) + ++class VHMPort(unittest.TestCase): ++ ++ def setUp(self): ++ import transaction ++ from Testing.makerequest import makerequest ++ from Testing.ZopeTestCase.ZopeLite import app ++ transaction.begin() ++ self.app = makerequest(app()) ++ if 'virtual_hosting' not in self.app.objectIds(): ++ # If ZopeLite was imported, we have no default virtual ++ # host monster ++ from Products.SiteAccess.VirtualHostMonster \ ++ import manage_addVirtualHostMonster ++ manage_addVirtualHostMonster(self.app, 'virtual_hosting') ++ self.app.manage_addFolder('folder') ++ self.app.folder.manage_addDTMLMethod('doc', '') ++ self.app.REQUEST.set('PARENTS', [self.app]) ++ self.traverse = self.app.REQUEST.traverse ++ ++ def tearDown(self): ++ import transaction ++ transaction.abort() ++ self.app._p_jar.close() ++ ++ def testPassedPort(self): ++ ob = self.traverse('/VirtualHostBase/http/www.mysite.com:81' ++ '/folder/') ++ self.assertEqual(self.app.REQUEST['ACTUAL_URL'], ++ 'http://www.mysite.com:81/folder/') ++ ++ def testIPv6(self): ++ ob = self.traverse('/VirtualHostBase/http/[::1]:80' ++ '/folder/') ++ self.assertEqual(self.app.REQUEST['ACTUAL_URL'], ++ 'http://[::1]/folder/') ++ ++ def testIPv6PassedPort(self): ++ ob = self.traverse('/VirtualHostBase/http/[::1]:81' ++ '/folder/') ++ self.assertEqual(self.app.REQUEST['ACTUAL_URL'], ++ 'http://[::1]:81/folder/') ++ + + class VHMAddingTests(unittest.TestCase): + +@@ -200,6 +242,7 @@ class VHMAddingTests(unittest.TestCase): + + def test_suite(): + suite = unittest.TestSuite() ++ suite.addTest(unittest.makeSuite(VHMPort)) + suite.addTest(unittest.makeSuite(VHMRegressions)) + suite.addTest(unittest.makeSuite(VHMAddingTests)) + return suite +-- +2.11.0 + diff --git a/stack/erp5/buildout.cfg b/stack/erp5/buildout.cfg index 392700b180..79462d4e61 100644 --- a/stack/erp5/buildout.cfg +++ b/stack/erp5/buildout.cfg @@ -591,6 +591,8 @@ extra-paths = patch-binary = ${patch:location}/bin/patch Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#1d9a56e9af4371f5b6951ebf217a15d7 Acquisition-patch-options = -p1 +Zope2-patches = ${:_profile_base_location_}/../../component/egg-patch/Zope2/0001-SiteAccess-Make-VirtualHostMonster-support-IPv6.patch#5b8a5f7e4e4d418ae3eab43390506972 +Zope2-patch-options = -p1 Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899 Products.DCWorkflow-patch-options = -p1 python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068 @@ -634,6 +636,7 @@ scripts += # patched eggs Acquisition = 2.13.12+SlapOSPatched001 Products.DCWorkflow = 2.2.4+SlapOSPatched001 +Zope2 = 2.13.28+SlapOSPatched001 ocropy = 1.0+SlapOSPatched001 pysvn = 1.7.10+SlapOSPatched002 python-ldap = 2.4.32+SlapOSPatched001 -- 2.30.9 From 654477c02ef3aa923cb4a279424821912e8d7a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Tue, 16 Oct 2018 08:39:08 +0200 Subject: [PATCH 2/5] component/apache: enable mod_reqtimeout quoting https://en.wikipedia.org/wiki/Slowloris_(computer_security) : > Since Apache 2.2.15, Apache ships the module mod_reqtimeout as the > official solution supported by the developers From https://httpd.apache.org/docs/2.4/mod/mod_reqtimeout.html default is header=20-40,MinRate=500 body=20,MinRate=500 which seem to be good default. --- component/apache/apache-backend.conf.in | 1 + component/apache/buildout.hash.cfg | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/component/apache/apache-backend.conf.in b/component/apache/apache-backend.conf.in index 32886fc860..3b18f07a18 100644 --- a/component/apache/apache-backend.conf.in +++ b/component/apache/apache-backend.conf.in @@ -102,6 +102,7 @@ LoadModule rewrite_module modules/mod_rewrite.so LoadModule headers_module modules/mod_headers.so LoadModule deflate_module modules/mod_deflate.so LoadModule filter_module modules/mod_filter.so +LoadModule reqtimeout_module modules/mod_reqtimeout.so AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml application/x-font-ttf application/font-woff application/font-woff2 application/x-font-opentype diff --git a/component/apache/buildout.hash.cfg b/component/apache/buildout.hash.cfg index 089abcf6ab..41648d91f6 100644 --- a/component/apache/buildout.hash.cfg +++ b/component/apache/buildout.hash.cfg @@ -14,5 +14,5 @@ # not need these here). [template-apache-backend-conf] filename = apache-backend.conf.in -md5sum = 3b430ca726a2707e1b6a2ae41a6c8e21 +md5sum = aaa2a9ea1de045597518fb416469535a -- 2.30.9 From 7671b6ff091ccc30bbfbc491f5b87f9be0a4f123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Tue, 16 Oct 2018 09:07:27 +0200 Subject: [PATCH 3/5] component/apache: increase Timeout for direct access case In "direct zope access" ports, the shared frontend is not used, so the argument that long timeout consume resources on shared server does not apply here. A timeout of one hour was choosen arbitrarily, a value that should be large enough for normal requests and more than the default 60s timeout that we hit in the "wait for activities" step when running zelenium tests. --- component/apache/apache-backend.conf.in | 1 + component/apache/buildout.hash.cfg | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/component/apache/apache-backend.conf.in b/component/apache/apache-backend.conf.in index 3b18f07a18..8194e5dfcd 100644 --- a/component/apache/apache-backend.conf.in +++ b/component/apache/apache-backend.conf.in @@ -185,6 +185,7 @@ Listen {{ ip }}:{{ port }} Listen {{ ip }}:{{ port }} SSLEngine on + Timeout 3600 {% if enable_authentication and parameter_dict['ca-cert'] and parameter_dict['crl'] -%} SSLVerifyClient require SSLCACertificateFile {{ parameter_dict['ca-cert'] }} diff --git a/component/apache/buildout.hash.cfg b/component/apache/buildout.hash.cfg index 41648d91f6..03c12b6e4c 100644 --- a/component/apache/buildout.hash.cfg +++ b/component/apache/buildout.hash.cfg @@ -14,5 +14,5 @@ # not need these here). [template-apache-backend-conf] filename = apache-backend.conf.in -md5sum = aaa2a9ea1de045597518fb416469535a +md5sum = 4777a443a8f5dc9b13e05b859f3db706 -- 2.30.9 From e75f31523680e8da1769339ed943d30ac5050f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Wed, 17 Oct 2018 08:56:12 +0200 Subject: [PATCH 4/5] component/apache: support zope path with a / /a/b/$1 must become VirtualHostRoot/_vh_a/vh_b/$1 for zope's virtual host monster to rewrite url properly --- component/apache/apache-backend.conf.in | 2 +- component/apache/buildout.hash.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/component/apache/apache-backend.conf.in b/component/apache/apache-backend.conf.in index 8194e5dfcd..ba3c291cad 100644 --- a/component/apache/apache-backend.conf.in +++ b/component/apache/apache-backend.conf.in @@ -201,7 +201,7 @@ Listen {{ ip }}:{{ port }} {% endif -%} {% for path, backend in path_mapping.items() %} - RewriteRule ^/{{path}}(.*) {{ backend }}/VirtualHostBase/https/{{ ip }}:{{ port }}/VirtualHostRoot/_vh_{{ path }}$1 [L,P] + RewriteRule ^/{{path}}(.*) {{ backend }}/VirtualHostBase/https/{{ ip }}:{{ port }}/VirtualHostRoot/_vh_{{ path.replace('/', '/_vh_') }}$1 [L,P] {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/component/apache/buildout.hash.cfg b/component/apache/buildout.hash.cfg index 03c12b6e4c..7f0bc34c57 100644 --- a/component/apache/buildout.hash.cfg +++ b/component/apache/buildout.hash.cfg @@ -14,5 +14,5 @@ # not need these here). [template-apache-backend-conf] filename = apache-backend.conf.in -md5sum = 4777a443a8f5dc9b13e05b859f3db706 +md5sum = 8bbba80016d8a0bebef93282ded87cda -- 2.30.9 From 7c4ceb0afa40fbe0966c83bb143af03b21038dfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Tue, 21 Aug 2018 06:37:48 +0200 Subject: [PATCH 5/5] erp5/balancer: add entries for direct access to zopes --- stack/erp5/buildout.hash.cfg | 2 +- stack/erp5/instance-balancer.cfg.in | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/stack/erp5/buildout.hash.cfg b/stack/erp5/buildout.hash.cfg index 1192f40948..0d133fd86a 100644 --- a/stack/erp5/buildout.hash.cfg +++ b/stack/erp5/buildout.hash.cfg @@ -86,7 +86,7 @@ md5sum = d1257e7e942307be0a79e34aa4320e9f [template-balancer] filename = instance-balancer.cfg.in -md5sum = f0fd49c7d6d9f7c6936afba0d18b7691 +md5sum = 384d6f167836e98944860fb62158c4c7 [template-haproxy-cfg] filename = haproxy.cfg.in diff --git a/stack/erp5/instance-balancer.cfg.in b/stack/erp5/instance-balancer.cfg.in index bd869a016f..05abf6eacd 100644 --- a/stack/erp5/instance-balancer.cfg.in +++ b/stack/erp5/instance-balancer.cfg.in @@ -99,6 +99,7 @@ ipv4 = {{ ipv4 }} {% set apache_dict = {} -%} {% set zope_virtualhost_monster_backend_dict = {} %} {% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #} +{% set direct_zope_url_dict = {} %} {# family_name => list of apache URLs #} {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%} {% for family_name, parameter_id_list in sorted( slapparameter_dict['zope-family-dict'].iteritems()) -%} @@ -163,6 +164,18 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }} {% set external_scheme = 'https' -%} {% endif -%} {% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%} + +{# Direct access to backend zopes. #} +{% set external_port = next_port() %} +{% set direct_zope_url_list = [] %} +{% set direct_zope_backend_mapping = {} %} +{% for i, (zope_family_address, _, _) in enumerate(zope_family_address_list) %} +{% do direct_zope_url_list.append('https://[' ~ ipv6 ~ ']:' ~ external_port ~ '/' ~ family_name ~ '/zope-' ~ i ) %} +{% do direct_zope_backend_mapping.__setitem__(family_name ~ '/zope-' ~ i, 'http://' ~ zope_family_address ) %} +{% endfor -%} +{% do zope_virtualhost_monster_backend_dict.__setitem__(('[' ~ ipv6 ~ ']', external_port), (ssl_authentication, direct_zope_backend_mapping) ) -%} +{% do direct_zope_url_dict.__setitem__(family_name, direct_zope_url_list) %} + {% endfor -%} [haproxy-cfg-parameter-dict] @@ -250,6 +263,10 @@ recipe = slapos.cookbook:publish.serialised {% for family_name, test_runner_url_list in test_runner_url_dict.items() -%} {{ family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }} {% endfor -%} +{% for family_name, direct_zope_url_list in direct_zope_url_dict.items() -%} +{{ family_name ~ '-direct-zope-url-list' }} = {{ dumps(direct_zope_url_list) }} +{% endfor -%} + monitor-base-url = ${monitor-publish-parameters:monitor-base-url} -- 2.30.9