diff --git a/component/cloudooo/buildout.cfg b/component/cloudooo/buildout.cfg index a02e4aa896843709ae1dac375724d73cba9ba01d..a5465f3965a0c3bb03a65c34a20e7e43a38a6ea5 100644 --- a/component/cloudooo/buildout.cfg +++ b/component/cloudooo/buildout.cfg @@ -14,6 +14,7 @@ recipe = slapos.recipe.build:gitclone repository = https://lab.nexedi.com/nexedi/cloudooo.git branch = master git-executable = ${git:location}/bin/git +revision = 0ff799ebcfea1013342f5450e88ff5c3b8536e89 [cloudooo] recipe = zc.recipe.egg diff --git a/component/libffi/buildout.cfg b/component/libffi/buildout.cfg index d96874cc511ba253e35d145c1cec5f7cd07fe46f..5e7ead3b6760bb9b4b618e09c7df7130615b34ca 100644 --- a/component/libffi/buildout.cfg +++ b/component/libffi/buildout.cfg @@ -16,6 +16,8 @@ patches = ${:_profile_base_location_}/includedir.diff#e430307d16a0d215a24ec2acff23d184 configure-options = --disable-static + --enable-portable-binary + --with-gcc-arch=generic # on x86_64 OpenSuse, libraries are installed under parts/libffi/lib64. make-targets = install && cp -av ${:location}/lib64/* ${:location}/lib/ || true diff --git a/component/slapos/buildout.cfg b/component/slapos/buildout.cfg index f35612ecdcd654443a4ff20530f7c6a01673b99d..01070eecb2332913648bd82b1797f0f29edd047b 100644 --- a/component/slapos/buildout.cfg +++ b/component/slapos/buildout.cfg @@ -20,6 +20,7 @@ extends = ../sqlite3/buildout.cfg ../swig/buildout.cfg ../zlib/buildout.cfg + ../socat/buildout.cfg parts = slapos @@ -31,7 +32,7 @@ parts = [environment] # Note: For now original PATH is appended to the end, as not all tools are # provided by SlapOS -PATH=${bison:location}/bin:${bzip2:location}/bin:${gettext:location}/bin:${glib:location}/bin:${libxml2:location}/bin:${libxslt:location}/bin:${m4:location}/bin:${ncurses:location}/bin:${openssl:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin:${readline:location}/bin:${sqlite3:location}/bin:${swig:location}/bin:${buildout:bin-directory}:${patch:location}/bin:$PATH +PATH=${bison:location}/bin:${bzip2:location}/bin:${gettext:location}/bin:${glib:location}/bin:${libxml2:location}/bin:${libxslt:location}/bin:${m4:location}/bin:${ncurses:location}/bin:${openssl:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin:${readline:location}/bin:${sqlite3:location}/bin:${swig:location}/bin:${buildout:bin-directory}:${patch:location}/bin:${socat:location}/bin:$PATH CFLAGS=-I${bzip2:location}/include -I${gdbm:location}/include -I${gettext:location}/include -I${glib:location}/include -I${libxml2:location}/include -I${libxslt:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline:location}/include -I${sqlite3:location}/include -I${zlib:location}/include CPPFLAGS=${:CFLAGS} LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${gdbm:location}/lib -Wl,-rpath=${gdbm:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${libxslt:location}/lib -Wl,-rpath=${libxslt:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib diff --git a/component/socat/buildout.cfg b/component/socat/buildout.cfg new file mode 100644 index 0000000000000000000000000000000000000000..20fc0e5304a45f37224e4d44844de001ba3e3c75 --- /dev/null +++ b/component/socat/buildout.cfg @@ -0,0 +1,9 @@ +[buildout] +parts = + socat + +[socat] +recipe = slapos.recipe.cmmi +url = http://www.dest-unreach.org/socat/download/socat-${:version}.tar.gz +version = 1.7.3.2 +md5sum = aec3154f7854580cfab0c2d81e910519 diff --git a/software/agent/software.cfg b/software/agent/software.cfg index 3d174f659b0a88428622abab6927c5c6b9fb0952..2fb28ab56e125ac7c34414cf704971c688144673 100644 --- a/software/agent/software.cfg +++ b/software/agent/software.cfg @@ -45,7 +45,7 @@ eggs = [versions] apache-libcloud = 0.18.0 ecdsa = 0.13 -erp5.util = 0.4.49 +erp5.util = 0.4.51 gitdb = 0.6.4 pycrypto = 2.6.1 pycurl = 7.43.0 diff --git a/software/apache-frontend/buildout.hash.cfg b/software/apache-frontend/buildout.hash.cfg index eb35758d6f5ff27f5ef37eea58c4a843c0b82638..d9e5c6204abf998f0d36e21d5959147cba37ef62 100644 --- a/software/apache-frontend/buildout.hash.cfg +++ b/software/apache-frontend/buildout.hash.cfg @@ -38,7 +38,7 @@ md5sum = 665e83d660c9b779249b2179d7ce4b4e [template-apache-frontend-configuration] filename = templates/apache.conf.in -md5sum = a56045e7b53ff00ab34d2a8f911fc1a1 +md5sum = 1f483a6e1a8076980e1bbbf495ee21b2 [template-custom-slave-list] filename = templates/apache-custom-slave-list.cfg.in diff --git a/software/apache-frontend/software.cfg b/software/apache-frontend/software.cfg index 3bb800148629aae39957117a777d8ec32c780447..c0c9ec7058d28ee49ba84f0353a381d0c744e5a0 100644 --- a/software/apache-frontend/software.cfg +++ b/software/apache-frontend/software.cfg @@ -24,7 +24,7 @@ dnspython = 1.15.0 # Required by: # slapos.toolbox==0.71 -erp5.util = 0.4.49 +erp5.util = 0.4.51 # Required by: # slapos.toolbox==0.71 diff --git a/software/apache-frontend/templates/apache.conf.in b/software/apache-frontend/templates/apache.conf.in index f7083bd1ee4d58674e91c08079e4947547f753f2..f888653202c070265702157da398e24528c5671a 100644 --- a/software/apache-frontend/templates/apache.conf.in +++ b/software/apache-frontend/templates/apache.conf.in @@ -138,7 +138,7 @@ ThreadsPerChild {{ slapparameter_dict.get('mpm-thread-per-child', '25') }} GracefulShutdownTimeout {{ slapparameter_dict.get('mpm-graceful-shutdown-timeout', '5') }} # Deflate -AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/x-javascript application/javascript +AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/json application/x-javascript application/javascript BrowserMatch ^Mozilla/4 gzip-only-text/html BrowserMatch ^Mozilla/4\.0[678] no-gzip BrowserMatch \bMSIE !no-gzip !gzip-only-text/html diff --git a/software/buildout-testing/software.cfg b/software/buildout-testing/software.cfg index 7d758036b1ece13d1ffe9905a57d0a3ab20b44e7..57c8e8edb76e74495ca81924ed78cf1640c672a3 100644 --- a/software/buildout-testing/software.cfg +++ b/software/buildout-testing/software.cfg @@ -57,3 +57,7 @@ template = key slapos_buildout slapos.buildout-repository:location key temp_directory directory:tmp raw runTestSuite_py ${buildout:bin-directory}/${runTestSuite_py:interpreter} + +[versions] +erp5.util = 0.4.51 +slapos.recipe.template = 4.3 diff --git a/software/caddy-frontend/README.caddy_frontend.rst b/software/caddy-frontend/README.caddy_frontend.rst index 8ef217e56c18cb412453c1a243b7836f01af83bd..400db2a6aff4a77da89710d822463eba20c871ba 100644 --- a/software/caddy-frontend/README.caddy_frontend.rst +++ b/software/caddy-frontend/README.caddy_frontend.rst @@ -157,12 +157,12 @@ Will append the specified path to the "VirtualHostRoot" of the zope's VirtualHos "path" is an optional parameter, ignored if not specified. Example of value: "/erp5/web_site_module/hosting/" -apache_custom_https -~~~~~~~~~~~~~~~~~~~ +caddy_custom_https +~~~~~~~~~~~~~~~~~~ Raw Caddy configuration in python template format (i.e. write "%%" for one "%") for the slave listening to the https port. Its content will be templatified in order to access functionalities such as cache access, ssl certificates... The list is available above. -apache_custom_http -~~~~~~~~~~~~~~~~~~ +caddy_custom_http +~~~~~~~~~~~~~~~~~ Raw Caddy configuration in python template format (i.e. write "%%" for one "%") for the slave listening to the http port. Its content will be templatified in order to access functionalities such as cache access, ssl certificates... The list is available above url @@ -286,7 +286,7 @@ Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be:: partition_parameter_kw={ "url":"https://[1:2:3:4:5:6:7:8]:1234", - "apache_custom_https":' + "caddy_custom_https":' https://www.example.com:%(https_port)s, https://example.com:%(https_port)s { bind %(local_ipv4)s tls %(ssl_crt)s %(ssl_key)s @@ -300,7 +300,7 @@ Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be:: insecure_skip_verify } } - "apache_custom_http":' + "caddy_custom_http":' http://www.example.com:%(http_port)s, http://example.com:%(http_port)s { bind %(local_ipv4)s log / %(access_log)s {combined} @@ -329,7 +329,7 @@ Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be:: "domain": "www.example.org", "enable_cache": "True", - "apache_custom_https":' + "caddy_custom_https":' ServerName www.example.org ServerAlias www.example.org ServerAlias example.org @@ -343,7 +343,7 @@ Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be:: RewriteEngine On RewriteRule ^/(.*) %(cache_access)s/$1 [L,P]', - "apache_custom_http":' + "caddy_custom_http":' ServerName www.example.org ServerAlias www.example.org ServerAlias example.org @@ -384,7 +384,7 @@ the proxy:: "path":"/erp5", "domain":"example.org", - "apache_custom_https":' + "caddy_custom_https":' ServerName www.example.org ServerAlias www.example.org ServerAdmin example.org @@ -413,7 +413,7 @@ the proxy:: # Use cache RewriteRule ^/(.*) %(cache_access)s/VirtualHostBase/https/www.example.org:443/erp5/VirtualHostRoot/$1 [L,P]', - "apache_custom_http":' + "caddy_custom_http":' ServerName www.example.org ServerAlias www.example.org ServerAlias example.org diff --git a/software/caddy-frontend/TODO.rst b/software/caddy-frontend/TODO.rst index 890c16106fbd13d9e9664b622c93ed333eeefbb6..d3da78c0a399260e2df1339527ce659ced46be77 100644 --- a/software/caddy-frontend/TODO.rst +++ b/software/caddy-frontend/TODO.rst @@ -1,24 +1,31 @@ Generally things to be done with ``caddy-frontend``: * ``apache-ca-certificate`` shall be merged with ``apache-certificate`` + + * ``apache-ca-certificate`` shall be appended to ``apache-certificate`` if not already there + * BUG?? check that changing ``apache-certificate`` on master partition results in reloading slave partition + * provide ``apache-frontend`` to ``caddy-frontend`` migration information * (new) ``type:websocket`` slave * ``type:eventsource``: * **Jérome Perrin**: *For event source, if I understand https://github.com/mholt/caddy/issues/1355 correctly, we could use caddy as a proxy in front of nginx-push-stream . If we have a "central shared" caddy instance, can it handle keeping connections opens for many clients ?* * ``ssl_ca_crt`` - * ``prefer-gzip-encoding-to-backend`` (requires writing middleware plugin for Caddy):: - - RequestHeader edit Accept-Encoding "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)" "gzip" * ``disabled-cookie-list`` (requires writing middleware plugin for Caddy):: RequestHeader edit Cookie "(^%(disabled_cookie)s=[^;]*; |; %(disabled_cookie)s=[^;]*|^%(disabled_cookie)s=[^;]*$)" ""' % dict(disabled_cookie=disabled_cookie) }} - * ``ssl_proxy_ca_crt`` for ``ssl_proxy_verify``, this is related to bug https://github.com/mholt/caddy/issues/1550, proposed solution `just adding your CA to the system's trust store` + + * there is already `MR <https://github.com/mholt/caddy/pull/2144>`_ which will allow regexp modification of headers, thus cookies + * ``ssl_proxy_ca_crt`` for ``ssl_proxy_verify``, this is related to bug `#1550 <https://github.com/mholt/caddy/issues/1550>`_, proposed solution `just adding your CA to the system's trust store` * ``check-error-on-caddy-log`` like ``check-error-on-apache-log`` * cover test suite like resilient tests for KVM and prove it works the same way as Caddy - * have ``caddy-frontend`` specific parameters, with backward compatibility to ``apache-frontend`` ones (like ``apache_custom_http`` --> ``caddy_custom_http``) - * change ``switch-softwaretype`` to way how ``software/erp5`` does, which will help with dropping jinja2 template for ``caddy-wrapper``, which is workaround for current situation https://lab.nexedi.com/nexedi/slapos/merge_requests/312#note_62678 + * have ``caddy-frontend`` specific parameters, with backward compatibility to ``apache-frontend`` ones: + + * ``apache-ca-certificate`` + * ``apache-certificate`` and ``apache-key`` + + * change ``switch-softwaretype`` to way how ``software/erp5`` does, which will help with dropping jinja2 template for ``caddy-wrapper``, which is workaround for current situation, cf `note_62678 <https://lab.nexedi.com/nexedi/slapos/merge_requests/312#note_62678>`_ * use `slapos!326 <https://lab.nexedi.com/nexedi/slapos/merge_requests/326>`_, and especially `note about complex restart scenarios <https://lab.nexedi.com/nexedi/slapos/merge_requests/326#note_60198>`_, instead of self-developed graceful restart scripts - * move out `test/utils.py` and use it from shared python distribution + * move out ``test/utils.py`` and use it from shared python distribution * provide various tricks for older browsers:: # The following directives modify normal HTTP response behavior to @@ -53,6 +60,8 @@ Generally things to be done with ``caddy-frontend``: </FilesMatch> * reduce the time of configuration validation (in ``instance-apache-frontend.cfg`` sections ``[configtest]``, ``[caddy-configuration]``, ``[nginx-configuration]``), as it is not scalable on frontend with 2000+ slaves (takes few minutes instead of few, < 5, seconds), issue posted `upstream <https://github.com/mholt/caddy/issues/2220>`_ * drop ``6tunnel`` and use ``bind`` in Caddy configuration, as soon as multiple binds will be possible, tracked in upstream `bind: support multiple values <https://github.com/mholt/caddy/pull/2128>`_ and `ipv6: does not bind on ipv4 and ipv6 for sites that resolve to both <https://github.com/mholt/caddy/issues/864>`_ + * use caddy-frontend in `standalone style playbooks <https://lab.nexedi.com/nexedi/slapos.package/tree/master/playbook/roles/standalone-shared>`_ + * ensure `QUIC <https://en.wikipedia.org/wiki/QUIC>`_ is used by caddy Things which can't be implemented: diff --git a/software/caddy-frontend/buildout.hash.cfg b/software/caddy-frontend/buildout.hash.cfg index 5431830b7901e68e0c034ea8f7cbc11e3b3cb2a1..ffb8912880f8ce7a2b2d3de8a2afdaa6bf90ddc6 100644 --- a/software/caddy-frontend/buildout.hash.cfg +++ b/software/caddy-frontend/buildout.hash.cfg @@ -18,19 +18,19 @@ md5sum = 906e5bd66b1265b8109a86b6ab46e91f [template-apache-frontend] filename = instance-apache-frontend.cfg -md5sum = 5602b4635e3da27ea6ff491824b0d8a5 +md5sum = b170d0987563b481eb71cf705c3658ab [template-apache-replicate] filename = instance-apache-replicate.cfg.in -md5sum = cd83f92b43904e1f3826072013cd682b +md5sum = 7f15b5745eda8e1f02d4bf7d886dcdad [template-slave-list] filename = templates/apache-custom-slave-list.cfg.in -md5sum = 3993419eea72ad4b62c0d479860f3c17 +md5sum = fb6c93f42f232e381174a5951c3fc222 [template-slave-configuration] filename = templates/custom-virtualhost.conf.in -md5sum = 74275ad73b03114c69f80c8f8ae73374 +md5sum = 54ae95597a126ae552c3a913ddf29e5e [template-replicate-publish-slave-information] filename = templates/replicate-publish-slave-information.cfg.in @@ -42,7 +42,7 @@ md5sum = 6689d96fc18d9aad78d77fe87770d4da [template-custom-slave-list] filename = templates/apache-custom-slave-list.cfg.in -md5sum = 3993419eea72ad4b62c0d479860f3c17 +md5sum = fb6c93f42f232e381174a5951c3fc222 [template-not-found-html] filename = templates/notfound.html @@ -50,7 +50,7 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b [template-default-slave-virtualhost] filename = templates/default-virtualhost.conf.in -md5sum = e9eccaa99077d9bc12b538d40f5421b0 +md5sum = 6da56d875f5cf396f8fd0685cf1a9a7a [template-cached-slave-virtualhost] filename = templates/cached-virtualhost.conf.in diff --git a/software/caddy-frontend/instance-apache-frontend.cfg b/software/caddy-frontend/instance-apache-frontend.cfg index 62e7b40e5800541b2af2d9ef04d2dea0e893097d..50b65c384c5d35732a7b1f9cf1164d99a7302b0a 100644 --- a/software/caddy-frontend/instance-apache-frontend.cfg +++ b/software/caddy-frontend/instance-apache-frontend.cfg @@ -110,8 +110,11 @@ configuration.plain_http_port = 8080 configuration.plain_nginx_port = 8081 configuration.nginx_port = 9443 configuration.server-admin = admin@example.com +# BBB: apache_custom_https and apache_custom_http configuration.apache_custom_https = "" configuration.apache_custom_http = "" +configuration.caddy_custom_https = "" +configuration.caddy_custom_http = "" configuration.apache-key = configuration.apache-certificate = configuration.apache-ca-certificate = @@ -124,7 +127,6 @@ configuration.trafficserver-mgmt-port = 8084 configuration.re6st-verification-url = http://[2001:67c:1254:4::1]/index.html configuration.enable-http2-by-default = true configuration.mpm-graceful-shutdown-timeout = 5 -configuration.monitor-cors-domains = configuration.monitor-httpd-port = 8072 [frontend-configuration] @@ -210,11 +212,14 @@ extra-context = template = ${template-slave-configuration:target} rendered = $${directory:template}/slave-virtualhost.conf.in extensions = jinja2.ext.do +# BBB: apache_custom_https and apache_custom_http extra-context = key https_port instance-parameter:configuration.port key http_port instance-parameter:configuration.plain_http_port key apache_custom_https instance-parameter:configuration.apache_custom_https key apache_custom_http instance-parameter:configuration.apache_custom_http + key caddy_custom_https instance-parameter:configuration.caddy_custom_https + key caddy_custom_http instance-parameter:configuration.caddy_custom_http # Deploy Caddy Frontend with Jinja power [dynamic-caddy-frontend-template] @@ -606,13 +611,17 @@ public-ipv4 = port = 4443 plain_http_port = 8080 server-admin = admin@example.com +# BBB: apache_custom_https and apache_custom_http apache_custom_https = "" apache_custom_http = "" +caddy_custom_https = "" +caddy_custom_http = "" apache-key = apache-certificate = open-port = 80 443 extra_slave_instance_list = frontend-name = +monitor-cors-domains = monitor-username = $${monitor-instance-parameter:username} monitor-password = $${monitor-htpasswd:passwd} @@ -622,7 +631,7 @@ monitor-password = $${monitor-htpasswd:passwd} [monitor-instance-parameter] monitor-httpd-port = $${instance-parameter:configuration.monitor-httpd-port} -cors-domains = $${instance-parameter:configuration.monitor-cors-domains} +cors-domains = $${slap-parameter:monitor-cors-domains} username = $${slap-parameter:monitor-username} password = $${slap-parameter:monitor-password} diff --git a/software/caddy-frontend/instance-apache-replicate.cfg.in b/software/caddy-frontend/instance-apache-replicate.cfg.in index 3169c5f3d3564767f1060b2d5d6fa8c2e02a607b..a8fb3321b9e0bc5ecb1188f3a9d3ae23750971ba 100644 --- a/software/caddy-frontend/instance-apache-replicate.cfg.in +++ b/software/caddy-frontend/instance-apache-replicate.cfg.in @@ -70,7 +70,8 @@ context = {% set authorized_slave_list = [] %} {% set rejected_slave_list = [] %} {% for slave in slave_instance_list %} -{% if not (slave.has_key('apache_custom_http') and not slave.get('slave_reference') in authorized_slave_string) %} +{# BBB: apache_custom_https AND apache_custom_http #} +{% if not ((slave.has_key('caddy_custom_http') or slave.has_key('apache_custom_http') or slave.has_key('caddy_custom_https') or slave.has_key('apache_custom_https')) and not slave.get('slave_reference') in authorized_slave_string) %} {% do authorized_slave_list.append(slave) %} {% else %} {% do rejected_slave_list.append(slave.get('slave_reference')) %} diff --git a/software/caddy-frontend/instance-slave-caddy-input-schema.json b/software/caddy-frontend/instance-slave-caddy-input-schema.json index 78580520a4df178161ae6bd45b333415d0a85a0f..4aba81cb3872689a965bc8720ca7ef1b799d7fc5 100644 --- a/software/caddy-frontend/instance-slave-caddy-input-schema.json +++ b/software/caddy-frontend/instance-slave-caddy-input-schema.json @@ -1,14 +1,14 @@ { "$schema": "http://json-schema.org/draft-04/schema", "properties": { - "apache_custom_http": { + "caddy_custom_http": { "default": "", "description": "Raw http configuration in python template format. Your site will be rejected if you use it without notification and approval of frontend administrators", "textarea": true, "title": "HTTP configuration", "type": "string" }, - "apache_custom_https": { + "caddy_custom_https": { "default": "", "description": "Raw https configuration in python template format. Your site will be rejected if you use it without notification and approval of frontend administrators", "textarea": true, diff --git a/software/caddy-frontend/software.cfg b/software/caddy-frontend/software.cfg index 3bb800148629aae39957117a777d8ec32c780447..c0c9ec7058d28ee49ba84f0353a381d0c744e5a0 100644 --- a/software/caddy-frontend/software.cfg +++ b/software/caddy-frontend/software.cfg @@ -24,7 +24,7 @@ dnspython = 1.15.0 # Required by: # slapos.toolbox==0.71 -erp5.util = 0.4.49 +erp5.util = 0.4.51 # Required by: # slapos.toolbox==0.71 diff --git a/software/caddy-frontend/templates/apache-custom-slave-list.cfg.in b/software/caddy-frontend/templates/apache-custom-slave-list.cfg.in index 043d8d0b6a2d94edc4543af51a594801b7b94a03..cbed4e2e0aa95fd5eb53ef2ac479112edc3b003d 100644 --- a/software/caddy-frontend/templates/apache-custom-slave-list.cfg.in +++ b/software/caddy-frontend/templates/apache-custom-slave-list.cfg.in @@ -99,7 +99,8 @@ crl = {{ custom_ssl_directory }}/crl/ {% do cached_server_dict.__setitem__(slave_reference, slave_configuration_section_name) %} {% endif %} -{% if not slave_instance.has_key('apache_custom_http') and not slave_instance.has_key('apache_custom_https') %} +{# BBB: apache_custom_https and apache_custom_http #} +{% if not slave_instance.has_key('caddy_custom_http') and not slave_instance.has_key('caddy_custom_https') and not slave_instance.has_key('apache_custom_http') and not slave_instance.has_key('apache_custom_https') %} {% do slave_publish_dict.__setitem__('domain', slave_instance.get('custom_domain')) %} {% do slave_publish_dict.__setitem__('url', "http://%s" % slave_instance.get('custom_domain')) %} {% do slave_publish_dict.__setitem__('site_url', "http://%s" % slave_instance.get('custom_domain')) %} @@ -193,10 +194,11 @@ cert-content = {{ dumps(slave_instance.get('ssl_crt')) }} {# ########################################## #} {# Set Slave Configuration #} [{{ slave_configuration_section_name }}] -{% set apache_custom_http = ((slave_instance.pop('apache_custom_http', '')) % slave_parameter_dict) %} -{% set apache_custom_https = ((slave_instance.pop('apache_custom_https', '')) % slave_parameter_dict) %} -apache_custom_http = {{ dumps(apache_custom_http) }} -apache_custom_https = {{ dumps(apache_custom_https) }} +{# BBB: apache_custom_https and apache_custom_http #} +{% set caddy_custom_http = ((slave_instance.pop('caddy_custom_http', slave_instance.pop('apache_custom_http', ''))) % slave_parameter_dict) %} +{% set caddy_custom_https = ((slave_instance.pop('caddy_custom_https', slave_instance.pop('apache_custom_https', ''))) % slave_parameter_dict) %} +caddy_custom_http = {{ dumps(caddy_custom_http) }} +caddy_custom_https = {{ dumps(caddy_custom_https) }} {{ '\n' }} {% for key, value in slave_instance.iteritems() %} {{ key }} = {{ dumps(value) }} @@ -211,7 +213,7 @@ rendered = {{ caddy_configuration_directory }}/${:filename} {% endif %} -{% if apache_custom_http %} +{% if caddy_custom_http or caddy_custom_https %} template = {{ template_custom_slave_configuration }} {% elif slave_type == 'eventsource' %} template = {{ template_eventsource_slave_configuration }} diff --git a/software/caddy-frontend/templates/custom-virtualhost.conf.in b/software/caddy-frontend/templates/custom-virtualhost.conf.in index 9e0b146adef7f681ca794e591a451b1b885368a1..b169dbef505820495f2802864c7bd3059c3de01b 100644 --- a/software/caddy-frontend/templates/custom-virtualhost.conf.in +++ b/software/caddy-frontend/templates/custom-virtualhost.conf.in @@ -1,2 +1,2 @@ -{{ slave_parameter.get('apache_custom_https', '') }} -{{ slave_parameter.get('apache_custom_http', '') }} \ No newline at end of file +{{ slave_parameter.get('caddy_custom_https', '') }} +{{ slave_parameter.get('caddy_custom_http', '') }} diff --git a/software/caddy-frontend/templates/default-virtualhost.conf.in b/software/caddy-frontend/templates/default-virtualhost.conf.in index 4725bb39cff6d53615e6d28bda6e36c69771d91c..cfad96eab7b3972cf79d934017e2f475042f805c 100644 --- a/software/caddy-frontend/templates/default-virtualhost.conf.in +++ b/software/caddy-frontend/templates/default-virtualhost.conf.in @@ -2,6 +2,10 @@ {%- set disable_no_cache_header = ('' ~ slave_parameter.get('disable-no-cache-request', '')).lower() in TRUE_VALUES %} {%- set disable_via_header = ('' ~ slave_parameter.get('disable-via-header', '')).lower() in TRUE_VALUES %} {%- set prefer_gzip = ('' ~ slave_parameter.get('prefer-gzip-encoding-to-backend', '')).lower() in TRUE_VALUES %} +{%- set proxy_append_list = [('', 'Default proxy configuration')] %} +{%- if prefer_gzip %} +{%- do proxy_append_list.append(('prefer-gzip', 'Proxy which always overrides Accept-Encoding to gzip if such is found')) %} +{%- endif %} {#- if prefer_gzip #} {%- set server_alias_list = slave_parameter.get('server-alias', '').split() %} {%- set enable_h2 = ('' ~ slave_parameter.get('enable-http2', slave_parameter['enable_http2_by_default'])).lower() in TRUE_VALUES %} {%- set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES %} @@ -15,7 +19,7 @@ {%- for host in host_list %} {%- do http_host_list.append('http://%s:%s' % (host, http_port)) %} {%- do https_host_list.append('https://%s:%s' % (host, https_port)) %} -{%- endfor %} +{%- endfor %} {#- for host in host_list #} # SSL enabled hosts {{ https_host_list|join(', ') }} { @@ -24,101 +28,119 @@ gzip {%- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %} status 501 / -{%- endif %} +{%- endif %} {#- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter #} tls {{ slave_parameter.get('path_to_ssl_crt', slave_parameter.get('login_certificate')) }} {{ slave_parameter.get('path_to_ssl_key', slave_parameter.get('login_key')) }} { {%- if slave_parameter.get('path_to_ssl_ca_crt') %} # Configuration of accepted clients clients {{ slave_parameter.get('path_to_ssl_ca_crt') }} -{%- endif %} +{%- endif %} {#- if slave_parameter.get('path_to_ssl_ca_crt') #} {%- if enable_h2 %} # Allow HTTP2 alpn h2 http/1.1 -{%- else %} +{%- else %} {#- if enable_h2 #} # Disallow HTTP2 alpn http/1.1 -{%- endif %} - } +{%- endif %} {#- if enable_h2 #} + } {# tls #} log / {{ slave_parameter.get('access_log') }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" errors {{ slave_parameter.get('error_log') }} {%- for disabled_cookie in disabled_cookie_list %} -{%- endfor %} +{%- endfor %} {#- for disabled_cookie in disabled_cookie_list #} {%- if prefer_gzip %} -{%- endif %} + rewrite { + if {>Accept-Encoding} match "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)" + to /prefer-gzip{uri} + } +{%- endif %} {#- if prefer_gzip #} {%- if slave_type == 'zope' and backend_url %} # Zope configuration - proxy / {{ backend_url }} { +{%- for (proxy_name, proxy_comment) in proxy_append_list %} + # {{ proxy_comment }} + proxy /{{ proxy_name }} {{ backend_url }} { +{%- if proxy_name == 'prefer-gzip' %} + without /prefer-gzip + header_upstream Accept-Encoding gzip +{%- endif %} {#- if proxy_name == 'prefer-gzip' #} # As backend is trusting REMOTE_USER header unset it always header_upstream -REMOTE_USER {%- if disable_via_header %} header_downstream -Via -{%- endif %} +{%- endif %} {#- if disable_via_header #} {%- if disable_no_cache_header %} header_upstream -Cache-Control header_upstream -Pragma -{%- endif %} +{%- endif %} {#- if disable_no_cache_header #} transparent timeout 600s {%- if ssl_proxy_verify %} {%- if 'ssl_proxy_ca_crt' in slave_parameter %} -{%- endif %} -{%- else %} +{%- endif %} {#- if 'ssl_proxy_ca_crt' in slave_parameter #} +{%- else %} {#- if ssl_proxy_verify #} insecure_skip_verify -{%- endif %} - } +{%- endif %} {#- if ssl_proxy_verify #} + } {# proxy #} +{%- endfor %} {#- for (proxy_name, proxy_comment) in proxy_append_list #} {%- if 'default-path' in slave_parameter %} redir 301 { if {path} is / / {scheme}://{host}/{{ slave_parameter.get('default-path') }} - } - {%- endif %} + } {# redir #} + {%- endif %} {#- if 'default-path' in slave_parameter #} rewrite { regexp (.*) to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-https-port', '443') }}%2F{{ slave_parameter.get('path', '') }}%2FVirtualHostRoot/{1} - } -{%- elif slave_type == 'redirect' and backend_url %} + } {# rewrite #} +{%- elif slave_type == 'redirect' and backend_url %} {#- if slave_type == 'zope' and backend_url #} # Redirect configuration redir 302 { / {{ backend_url }}{uri} - } -{%- else %} + } {# redir #} +{%- else %} {#- if slave_type == 'zope' and backend_url #} # Default configuration {%- if 'default-path' in slave_parameter %} redir 301 { if {path} is / / {scheme}://{host}/{{ slave_parameter.get('default-path') }} - } -{%- endif %} + } {# redir #} +{%- endif %} {#- if 'default-path' in slave_parameter #} {%- if backend_url %} - proxy / {{ backend_url }} { +{%- for (proxy_name, proxy_comment) in proxy_append_list %} + # {{ proxy_comment }} + proxy /{{ proxy_name }} {{ backend_url }} { +{%- if proxy_name == 'prefer-gzip' %} + without /prefer-gzip + header_upstream Accept-Encoding gzip +{%- endif %} {#- if proxy_name == 'prefer-gzip' #} # As backend is trusting REMOTE_USER header unset it always header_upstream -REMOTE_USER {%- if disable_via_header %} header_downstream -Via -{%- endif %} +{%- endif %} {#- if disable_via_header #} {%- if disable_no_cache_header %} header_upstream -Cache-Control header_upstream -Pragma -{%- endif %} +{%- endif %} {#- if disable_no_cache_header #} transparent timeout 600s {%- if ssl_proxy_verify %} {%- if 'ssl_proxy_ca_crt' in slave_parameter %} -{%- endif %} -{%- else %} +{%- endif %} {#- if 'ssl_proxy_ca_crt' in slave_parameter #} +{%- else %} {#- if ssl_proxy_verify #} insecure_skip_verify -{%- endif %} - } -{%- endif %} -{%- endif %} -} +{%- endif %} {#- if ssl_proxy_verify #} + } {# proxy #} +{%- endfor %} {#- for (proxy_name, proxy_comment) in proxy_append_list #} +{%- endif %} {#- if backend_url #} +{%- endif %} {#- if slave_type == 'zope' and backend_url #} +} {# https_host_list|join(', ') #} # SSL-disabled hosts {{ http_host_list|join(', ') }} { @@ -127,88 +149,106 @@ gzip {%- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %} status 501 / -{%- endif %} +{%- endif %} {#- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter #} log / {{ slave_parameter.get('access_log') }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" errors {{ slave_parameter.get('error_log') }} {%- for disabled_cookie in disabled_cookie_list %} -{%- endfor %} +{%- endfor %} {#- for disabled_cookie in disabled_cookie_list #} {%- if prefer_gzip %} -{%- endif %} + rewrite { + if {>Accept-Encoding} match "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)" + to /prefer-gzip{uri} + } +{%- endif %} {#- if prefer_gzip #} {%- if https_only %} # Enforced redirection to SSL-enabled host redir / https://{host}{uri} -{%- elif slave_type == 'redirect' and slave_parameter.get('url', '') %} +{%- elif slave_type == 'redirect' and slave_parameter.get('url', '') %} {#- if https_only #} # Redirect configuration redir 302 { / {{ slave_parameter.get('url', '') }}{uri} - } -{%- elif slave_type == 'zope' and backend_url %} + } {# redir #} +{%- elif slave_type == 'zope' and backend_url %} {#- if https_only #} # Zope configuration - proxy / {{ backend_url }} { +{%- for (proxy_name, proxy_comment) in proxy_append_list %} + # {{ proxy_comment }} + proxy /{{ proxy_name }} {{ backend_url }} { +{%- if proxy_name == 'prefer-gzip' %} + without /prefer-gzip + header_upstream Accept-Encoding gzip +{%- endif %} {#- if proxy_name == 'prefer-gzip' #} # As backend is trusting REMOTE_USER header unset it always header_upstream -REMOTE_USER {%- if disable_via_header %} header_downstream -Via -{%- endif %} +{%- endif %} {#- if disable_via_header #} {%- if disable_no_cache_header %} header_upstream -Cache-Control header_upstream -Pragma -{%- endif %} +{%- endif %} {#- if disable_no_cache_header #} transparent timeout 600s {%- if ssl_proxy_verify %} {%- if 'ssl_proxy_ca_crt' in slave_parameter %} -{%- endif %} -{%- else %} +{%- endif %} {#- if 'ssl_proxy_ca_crt' in slave_parameter #} +{%- else %} {#- if ssl_proxy_verify #} insecure_skip_verify -{%- endif %} - } +{%- endif %} {#- if ssl_proxy_verify #} + } {# proxy #} +{%- endfor %} {#- for (proxy_name, proxy_comment) in proxy_append_list #} {%- if 'default-path' in slave_parameter %} redir 301 { if {path} is / / {scheme}://{host}/{{ slave_parameter.get('default-path') }} - } -{%- endif %} + } {# redir #} +{%- endif %} {#- if 'default-path' in slave_parameter #} rewrite { regexp (.*) to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-http-port', '80') }}%2F{{ slave_parameter.get('path', '') }}%2FVirtualHostRoot/{1} - } -{%- else %} + } {# rewrite #} +{%- else %} {#- if https_only #} # Default configuration {%- if 'default-path' in slave_parameter %} redir 301 { if {path} is / / {scheme}://{host}/{{ slave_parameter.get('default-path') }} - } -{%- endif %} + } {# redir #} +{%- endif %} {#- if 'default-path' in slave_parameter #} {%- if slave_parameter.get('url', '') %} - proxy / {{ slave_parameter.get('url', '') }} { +{%- for (proxy_name, proxy_comment) in proxy_append_list %} + # {{ proxy_comment }} + proxy /{{ proxy_name }} {{ slave_parameter.get('url', '') }} { +{%- if proxy_name == 'prefer-gzip' %} + without /prefer-gzip + header_upstream Accept-Encoding gzip +{%- endif %} {#- if proxy_name == 'prefer-gzip' #} # As backend is trusting REMOTE_USER header unset it always header_upstream -REMOTE_USER {%- if disable_via_header %} header_downstream -Via -{%- endif %} +{%- endif %} {#- if disable_via_header #} {%- if disable_no_cache_header %} header_upstream -Cache-Control header_upstream -Pragma -{%- endif %} +{%- endif %} {#- if disable_no_cache_header #} transparent timeout 600s {%- if ssl_proxy_verify %} {%- if 'ssl_proxy_ca_crt' in slave_parameter %} -{%- endif %} -{%- else %} +{%- endif %} {#- if 'ssl_proxy_ca_crt' in slave_parameter #} +{%- else %} {#- if ssl_proxy_verify #} insecure_skip_verify -{%- endif %} - } -{%- endif %} -{%- endif %} -} +{%- endif %} {#- if ssl_proxy_verify #} + } {# proxy #} +{%- endfor %} {#- for (proxy_name, proxy_comment) in proxy_append_list #} +{%- endif %} {#- if slave_parameter.get('url', '') #} +{%- endif %} {#- if https_only #} +} {# http_host_list|join(', ') #} diff --git a/software/caddy-frontend/test/test.py b/software/caddy-frontend/test/test.py index b07d53d41faa98ba9c7e9a11d8b60859b038c4e1..a7519dbe394a0daf6a468847dc53dbbca534b8f8 100644 --- a/software/caddy-frontend/test/test.py +++ b/software/caddy-frontend/test/test.py @@ -70,6 +70,35 @@ if IS_CADDY: else: no_backend_response_code = 502 +caddy_custom_https = '''# caddy_custom_https_filled_in_accepted +https://caddycustomhttpsaccepted.example.com:%%(https_port)s { + bind %%(local_ipv4)s + tls %%(ssl_crt)s %%(ssl_key)s + + log / %%(access_log)s {combined} + errors %%(error_log)s + + proxy / %(url)s { + transparent + timeout 600s + insecure_skip_verify + } +} +''' +caddy_custom_http = '''# caddy_custom_http_filled_in_accepted +http://caddycustomhttpsaccepted.example.com:%%(http_port)s { + bind %%(local_ipv4)s + log / %%(access_log)s {combined} + errors %%(error_log)s + + proxy / %(url)s { + transparent + timeout 600s + insecure_skip_verify + } +} +''' + # apache_custom_http[s] difference if IS_CADDY: LOG_REGEXP = '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} SOME_REMOTE_USER ' \ @@ -124,7 +153,7 @@ RewriteEngine On RewriteRule ^/(.*)$ %(url)s/$1 [L,P] ''' - apache_custom_http = '''# apache_custom_http_filled_in_accpeted + apache_custom_http = '''# apache_custom_http_filled_in_accepted ServerName apachecustomhttpsaccepted.example.com ServerAlias apachecustomhttpsaccepted.example.com @@ -221,6 +250,8 @@ class TestDataMixin(object): def test_file_list_log(self): self._test_file_list('log', [ + # no control at all when cron would kick in, ignore it + 'cron.log', # appears late, not needed for assertion 'trafficserver/diags.log', 'trafficserver/squid.blog', @@ -495,13 +526,15 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): 'public-ipv4': utils.LOCAL_IPV4, 'apache-certificate': open('wildcard.example.com.crt').read(), 'apache-key': open('wildcard.example.com.key').read(), - '-frontend-authorized-slave-string': '_apache_custom_http_s-accepted', + '-frontend-authorized-slave-string': + '_apache_custom_http_s-accepted _caddy_custom_http_s-accepted', 'port': HTTPS_PORT, 'plain_http_port': HTTP_PORT, 'nginx_port': NGINX_HTTPS_PORT, 'plain_nginx_port': NGINX_HTTP_PORT, 'monitor-httpd-port': MONITOR_HTTPD_PORT, '-frontend-config-1-monitor-httpd-port': MONITOR_F1_HTTPD_PORT, + 'mpm-graceful-shutdown-timeout': 2, } @classmethod @@ -632,6 +665,16 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): 'apache_custom_https': apache_custom_https % dict(url=cls.backend_url), 'apache_custom_http': apache_custom_http % dict(url=cls.backend_url), }, + 'caddy_custom_http_s-rejected': { + 'url': cls.backend_url, + 'caddy_custom_https': '# caddy_custom_https_filled_in_rejected', + 'caddy_custom_http': '# caddy_custom_http_filled_in_rejected', + }, + 'caddy_custom_http_s-accepted': { + 'url': cls.backend_url, + 'caddy_custom_https': caddy_custom_https % dict(url=cls.backend_url), + 'caddy_custom_http': caddy_custom_http % dict(url=cls.backend_url), + }, 'prefer-gzip-encoding-to-backend': { 'url': cls.backend_url, 'prefer-gzip-encoding-to-backend': 'true', @@ -668,14 +711,27 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): parameter_dict = self.computer_partition.getConnectionParameterDict() self.assertKeyWithPop('monitor-setup-url', parameter_dict) - self.assertEqual( - { + if IS_CADDY: + expected_parameter_dict = { + 'monitor-base-url': None, + 'domain': 'example.com', + 'accepted-slave-amount': '33', + 'rejected-slave-amount': '2', + 'slave-amount': '35', + 'rejected-slave-list': + '["_caddy_custom_http_s-rejected", "_apache_custom_http_s-rejected"]'} + else: + expected_parameter_dict = { 'monitor-base-url': None, 'domain': 'example.com', - 'accepted-slave-amount': '32', + 'accepted-slave-amount': '34', 'rejected-slave-amount': '1', - 'slave-amount': '33', - 'rejected-slave-list': '["_apache_custom_http_s-rejected"]'}, + 'slave-amount': '35', + 'rejected-slave-list': + '["_apache_custom_http_s-rejected"]'} + + self.assertEqual( + expected_parameter_dict, parameter_dict ) @@ -694,6 +750,27 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): set(), set(os.listdir(os.path.join(partition_path, 'etc', 'monitor-promise')))) + # check that monitor cors domains are correctly setup by file presence, as + # we trust monitor stack being tested in proper place and it is too hard + # to have working monitor with local proxy + self.assertTestData( + open( + os.path.join( + partition_path, 'etc', 'httpd-cors.cfg'), 'r').read().strip()) + + @skipIf(not IS_CADDY, 'Will NOT be covered on apache-frontend') + def test_slave_partition_state(self): + partition_path = self.getSlavePartitionPath() + self.assertTrue( + '-grace 2s' in + open(os.path.join(partition_path, 'bin', 'caddy-wrapper'), 'r').read() + ) + + self.assertTrue( + '-grace 2s' in + open(os.path.join(partition_path, 'bin', 'nginx-wrapper'), 'r').read() + ) + def test_empty(self): parameter_dict = self.slave_connection_parameter_dict_dict[ 'empty'] @@ -1423,7 +1500,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): self.assertEqual( headers, - {'Age': '0', 'Content-type': 'text/json', + {'Age': '0', 'Content-type': 'application/json', + 'Vary': 'Accept-Encoding', 'Content-Encoding': 'gzip', 'Set-Cookie': 'secured=value;secure, nonsecured=value'} ) @@ -1446,7 +1524,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): self.assertEqual( headers, - {'Age': '0', 'Content-type': 'text/json', + {'Age': '0', 'Content-type': 'application/json', + 'Vary': 'Accept-Encoding', 'Content-Encoding': 'gzip', 'Set-Cookie': 'secured=value;secure, nonsecured=value'} ) @@ -1744,7 +1823,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): self.assertEqual( headers, - {'Age': '0', 'Content-type': 'text/json', + {'Age': '0', 'Content-type': 'application/json', 'Set-Cookie': 'secured=value;secure, nonsecured=value', 'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding'} ) @@ -1837,7 +1916,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): self.assertEqual( headers, - {'Age': '0', 'Content-type': 'text/json', + {'Age': '0', 'Content-type': 'application/json', 'Set-Cookie': 'secured=value;secure, nonsecured=value', 'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding'} ) @@ -1888,11 +1967,12 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): self.assertEqual( headers, - {'Age': '0', 'Content-type': 'text/json', + {'Age': '0', 'Content-type': 'application/json', 'Set-Cookie': 'secured=value;secure, nonsecured=value', 'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding'} ) + @skipIf(not IS_CADDY, 'Will NOT be fixed for apache-frontend') def test_enable_http2_false(self): parameter_dict = self.slave_connection_parameter_dict_dict[ 'enable-http2-false'] @@ -1934,7 +2014,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): headers, { 'Vary': 'Accept-Encoding', - 'Content-Type': 'text/json', + 'Content-Type': 'application/json', 'Set-Cookie': 'secured=value;secure, nonsecured=value', 'Content-Encoding': 'gzip', } @@ -1984,7 +2064,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): headers, { 'Vary': 'Accept-Encoding', - 'Content-type': 'text/json', + 'Content-type': 'application/json', 'Set-Cookie': 'secured=value;secure, nonsecured=value', 'Content-Encoding': 'gzip', } @@ -1993,7 +2073,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): self.assertTrue( isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4'])) - @skipIf(IS_CADDY, 'Feature postponed') def test_prefer_gzip_encoding_to_backend(self): parameter_dict = self.slave_connection_parameter_dict_dict[ 'prefer-gzip-encoding-to-backend'] @@ -2068,10 +2147,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): self.assertEqual( result.json()['Incoming Headers']['cookie'], 'Coffee=present') - @skip('Feature postponed') - def test_caddy_custom_http_s_rejected(self): - raise NotImplementedError - def test_apache_custom_http_s_rejected(self): parameter_dict = self.slave_connection_parameter_dict_dict[ 'apache_custom_http_s-rejected'] @@ -2120,16 +2195,102 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): headers.pop('Connection', None) headers.pop('Keep-Alive', None) + if IS_CADDY: + self.assertEqual( + headers, + { + 'Content-type': 'application/json', + 'Set-Cookie': 'secured=value;secure, nonsecured=value' + } + ) + else: + self.assertEqual( + headers, + { + 'Vary': 'Accept-Encoding', 'Content-Encoding': 'gzip', + 'Content-type': 'application/json', + 'Set-Cookie': 'secured=value;secure, nonsecured=value' + } + ) + + result_http = self.fakeHTTPResult( + 'apachecustomhttpsaccepted.example.com', + parameter_dict['public-ipv4'], 'test-path') + self.assertEqualResultJson(result_http, 'Path', '/test-path') + + slave_configuration_file_list = glob.glob(os.path.join( + self.instance_path, '*', 'etc', '*slave-conf.d', '*.conf')) + # no configuration file contains provided custom http + configuration_file_with_custom_https_list = [ + q for q in slave_configuration_file_list + if 'apache_custom_https_filled_in_accepted' in open(q).read()] + self.assertEqual(1, len(configuration_file_with_custom_https_list)) + + configuration_file_with_custom_http_list = [ + q for q in slave_configuration_file_list + if 'apache_custom_http_filled_in_accepted' in open(q).read()] + self.assertEqual(1, len(configuration_file_with_custom_http_list)) + + @skipIf(not IS_CADDY, 'Feature not applicable') + def test_caddy_custom_http_s_rejected(self): + parameter_dict = self.slave_connection_parameter_dict_dict[ + 'caddy_custom_http_s-rejected'] + self.assertEqual({}, parameter_dict) + slave_configuration_file_list = glob.glob(os.path.join( + self.instance_path, '*', 'etc', '*slave-conf.d', '*.conf')) + # no configuration file contains provided custom http + configuration_file_with_custom_https_list = [ + q for q in slave_configuration_file_list + if 'caddy_custom_https_filled_in_rejected' in open(q).read()] + self.assertEqual([], configuration_file_with_custom_https_list) + + configuration_file_with_custom_http_list = [ + q for q in slave_configuration_file_list + if 'caddy_custom_http_filled_in_rejected' in open(q).read()] + self.assertEqual([], configuration_file_with_custom_http_list) + + @skipIf(not IS_CADDY, 'Feature not applicable') + def test_caddy_custom_http_s_accepted(self): + parameter_dict = self.slave_connection_parameter_dict_dict[ + 'caddy_custom_http_s-accepted'] + self.assertLogAccessUrlWithPop( + parameter_dict, 'caddy_custom_http_s-accepted') + self.assertEqual( + parameter_dict, + {'replication_number': '1', 'public-ipv4': utils.LOCAL_IPV4} + ) + + result = self.fakeHTTPSResult( + 'caddycustomhttpsaccepted.example.com', + parameter_dict['public-ipv4'], 'test-path') + + self.assertEqual( + utils.der2pem(result.peercert), + open('wildcard.example.com.crt').read()) + + self.assertEqualResultJson(result, 'Path', '/test-path') + + headers = result.headers.copy() + + self.assertKeyWithPop('Server', headers) + self.assertKeyWithPop('Date', headers) + + # drop vary-keys + headers.pop('Content-Length', None) + headers.pop('Transfer-Encoding', None) + headers.pop('Connection', None) + headers.pop('Keep-Alive', None) + self.assertEqual( headers, { - 'Content-type': 'text/json', + 'Content-type': 'application/json', 'Set-Cookie': 'secured=value;secure, nonsecured=value' } ) result_http = self.fakeHTTPResult( - 'apachecustomhttpsaccepted.example.com', + 'caddycustomhttpsaccepted.example.com', parameter_dict['public-ipv4'], 'test-path') self.assertEqualResultJson(result_http, 'Path', '/test-path') @@ -2138,12 +2299,12 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): # no configuration file contains provided custom http configuration_file_with_custom_https_list = [ q for q in slave_configuration_file_list - if 'apache_custom_https_filled_in_accepted' in open(q).read()] + if 'caddy_custom_https_filled_in_accepted' in open(q).read()] self.assertEqual(1, len(configuration_file_with_custom_https_list)) configuration_file_with_custom_http_list = [ q for q in slave_configuration_file_list - if 'apache_custom_https_filled_in_accepted' in open(q).read()] + if 'caddy_custom_http_filled_in_accepted' in open(q).read()] self.assertEqual(1, len(configuration_file_with_custom_http_list)) def test_https_url(self): @@ -2247,6 +2408,7 @@ class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin): 2, len(slave_configuration_file_list), slave_configuration_file_list) +@skipIf(not IS_CADDY, 'Will NOT be fixed for apache-frontend') class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase, TestDataMixin): @classmethod @@ -2340,6 +2502,7 @@ class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase, isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4'])) +@skipIf(not IS_CADDY, 'Will NOT be fixed for apache-frontend') class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase, TestDataMixin): @classmethod diff --git a/software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-APACHE.txt b/software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-APACHE.txt index 101855790d10fb1edd7b0e6e691346fd59f7b1d6..01bcec1f33d9c4e38d7e7d7d02da5452b7f1e209 100644 --- a/software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-APACHE.txt +++ b/software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-APACHE.txt @@ -2,6 +2,10 @@ TestSlave-1/var/log/frontend-apache-access.log TestSlave-1/var/log/frontend-apache-error.log TestSlave-1/var/log/httpd/_apache_custom_http_s-accepted_access_log TestSlave-1/var/log/httpd/_apache_custom_http_s-accepted_error_log +TestSlave-1/var/log/httpd/_caddy_custom_http_s-accepted_access_log +TestSlave-1/var/log/httpd/_caddy_custom_http_s-accepted_error_log +TestSlave-1/var/log/httpd/_caddy_custom_http_s-rejected_access_log +TestSlave-1/var/log/httpd/_caddy_custom_http_s-rejected_error_log TestSlave-1/var/log/httpd/_custom_domain_access_log TestSlave-1/var/log/httpd/_custom_domain_error_log TestSlave-1/var/log/httpd/_custom_domain_ssl_crt_ssl_key_access_log diff --git a/software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt b/software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt index 3a17a209665160755eceaf718e8c20621a3ffafc..3446e747c4d6370f17657730f0c3922ea377b882 100644 --- a/software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt +++ b/software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt @@ -3,6 +3,8 @@ TestSlave-1/var/log/frontend-access.log TestSlave-1/var/log/frontend-error.log TestSlave-1/var/log/httpd/_apache_custom_http_s-accepted_access_log TestSlave-1/var/log/httpd/_apache_custom_http_s-accepted_error_log +TestSlave-1/var/log/httpd/_caddy_custom_http_s-accepted_access_log +TestSlave-1/var/log/httpd/_caddy_custom_http_s-accepted_error_log TestSlave-1/var/log/httpd/_custom_domain_access_log TestSlave-1/var/log/httpd/_custom_domain_error_log TestSlave-1/var/log/httpd/_custom_domain_ssl_crt_ssl_key_access_log diff --git a/software/caddy-frontend/test/test_data/test.TestSlave.test_master_partition_state-APACHE.txt b/software/caddy-frontend/test/test_data/test.TestSlave.test_master_partition_state-APACHE.txt new file mode 100644 index 0000000000000000000000000000000000000000..ef7577190f5c14ea431f71b0528a0f86ad932562 --- /dev/null +++ b/software/caddy-frontend/test/test_data/test.TestSlave.test_master_partition_state-APACHE.txt @@ -0,0 +1,5 @@ +SetEnvIf Origin "^http(s)?://(.+\.)?(monitor\.app\.officejs\.com)$" ORIGIN_DOMAIN=$0 +Header always set Access-Control-Allow-Origin "%{ORIGIN_DOMAIN}e" env=ORIGIN_DOMAIN +Header always set Access-Control-Allow-Credentials "true" env=ORIGIN_DOMAIN +Header always set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST" env=ORIGIN_DOMAIN +Header always set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization" env=ORIGIN_DOMAIN \ No newline at end of file diff --git a/software/caddy-frontend/test/test_data/test.TestSlave.test_master_partition_state-CADDY.txt b/software/caddy-frontend/test/test_data/test.TestSlave.test_master_partition_state-CADDY.txt new file mode 100644 index 0000000000000000000000000000000000000000..ef7577190f5c14ea431f71b0528a0f86ad932562 --- /dev/null +++ b/software/caddy-frontend/test/test_data/test.TestSlave.test_master_partition_state-CADDY.txt @@ -0,0 +1,5 @@ +SetEnvIf Origin "^http(s)?://(.+\.)?(monitor\.app\.officejs\.com)$" ORIGIN_DOMAIN=$0 +Header always set Access-Control-Allow-Origin "%{ORIGIN_DOMAIN}e" env=ORIGIN_DOMAIN +Header always set Access-Control-Allow-Credentials "true" env=ORIGIN_DOMAIN +Header always set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST" env=ORIGIN_DOMAIN +Header always set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization" env=ORIGIN_DOMAIN \ No newline at end of file diff --git a/software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-APACHE.txt b/software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-APACHE.txt index 8ba5c0d9590a814dfec22a48b0003f77bcef00e9..c27ae0a7728d9bffafe0d3ab8d2baab3130aa726 100644 --- a/software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-APACHE.txt +++ b/software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-APACHE.txt @@ -1,5 +1,9 @@ TestSlave-1/etc/monitor-promise/check-_apache_custom_http_s-accepted-error-log-last-day TestSlave-1/etc/monitor-promise/check-_apache_custom_http_s-accepted-error-log-last-hour +TestSlave-1/etc/monitor-promise/check-_caddy_custom_http_s-accepted-error-log-last-day +TestSlave-1/etc/monitor-promise/check-_caddy_custom_http_s-accepted-error-log-last-hour +TestSlave-1/etc/monitor-promise/check-_caddy_custom_http_s-rejected-error-log-last-day +TestSlave-1/etc/monitor-promise/check-_caddy_custom_http_s-rejected-error-log-last-hour TestSlave-1/etc/monitor-promise/check-_custom_domain-error-log-last-day TestSlave-1/etc/monitor-promise/check-_custom_domain-error-log-last-hour TestSlave-1/etc/monitor-promise/check-_custom_domain_ssl_crt_ssl_key-error-log-last-day diff --git a/software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-CADDY.txt b/software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-CADDY.txt index 8ba5c0d9590a814dfec22a48b0003f77bcef00e9..a069f1a1e831cc7e093abaa46e7ffffcffc35c5a 100644 --- a/software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-CADDY.txt +++ b/software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-CADDY.txt @@ -1,5 +1,7 @@ TestSlave-1/etc/monitor-promise/check-_apache_custom_http_s-accepted-error-log-last-day TestSlave-1/etc/monitor-promise/check-_apache_custom_http_s-accepted-error-log-last-hour +TestSlave-1/etc/monitor-promise/check-_caddy_custom_http_s-accepted-error-log-last-day +TestSlave-1/etc/monitor-promise/check-_caddy_custom_http_s-accepted-error-log-last-hour TestSlave-1/etc/monitor-promise/check-_custom_domain-error-log-last-day TestSlave-1/etc/monitor-promise/check-_custom_domain-error-log-last-hour TestSlave-1/etc/monitor-promise/check-_custom_domain_ssl_crt_ssl_key-error-log-last-day diff --git a/software/caddy-frontend/test/utils.py b/software/caddy-frontend/test/utils.py index a58cfac3cd4c6e024b805791481158dee2e93748..f3ead6a1d070a284194ddd80eee4d43ac5e36a01 100644 --- a/software/caddy-frontend/test/utils.py +++ b/software/caddy-frontend/test/utils.py @@ -254,7 +254,7 @@ class SlapOSInstanceTestCase(unittest.TestCase): class TestHandler(BaseHTTPRequestHandler): def do_GET(self): self.send_response(200) - self.send_header("Content-type", "text/json") + self.send_header("Content-type", "application/json") self.send_header('Set-Cookie', 'secured=value;secure') self.send_header('Set-Cookie', 'nonsecured=value') self.end_headers() diff --git a/software/cdn-me/software.cfg b/software/cdn-me/software.cfg index 3f687cf2ab6e7b249e019dea8f04a4880793f35a..8fea6f420030fbf3650176ca3ba8aafc884fdd4b 100644 --- a/software/cdn-me/software.cfg +++ b/software/cdn-me/software.cfg @@ -38,6 +38,6 @@ eggs = [versions] cns.recipe.symlink = 0.2.3 collective.recipe.environment = 0.2.0 -erp5.util = 0.4.49 +erp5.util = 0.4.51 plone.recipe.command = 1.1 slapos.recipe.template = 4.3 diff --git a/software/cloudooo/software-common.cfg b/software/cloudooo/software-common.cfg index cf07220ddfb5e46114f2fe2ed8f64c2f6391cdf5..5097f517438395bbcee6597854f1c0ddeca59810 100644 --- a/software/cloudooo/software-common.cfg +++ b/software/cloudooo/software-common.cfg @@ -83,3 +83,12 @@ template-logrotate-base = ${template-logrotate-base:rendered} recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/${:filename} mode = 640 + +[versions] +# Required by: +# cloudooo==1.2.6.dev0 +argparse = 1.4.0 + +# Required by: +# cloudooo==1.2.6.dev0 +pyPdf = 1.13 diff --git a/software/erp5testnode/software.cfg b/software/erp5testnode/software.cfg index 084b0cbd4a4b48a17c72b1890b24a7ff64ee9e5f..46ab32054c4e01c9bd254ba9879d2436a0261f53 100644 --- a/software/erp5testnode/software.cfg +++ b/software/erp5testnode/software.cfg @@ -58,7 +58,7 @@ mode = 0644 [versions] PyXML = 0.8.5 -erp5.util = 0.4.50 +erp5.util = 0.4.51 slapos.recipe.template = 4.3 ipython = 5.3.0 apache-libcloud = 2.1.0 diff --git a/software/erp5testnode/testsuite/caddy-frontend/software.cfg b/software/erp5testnode/testsuite/caddy-frontend/software.cfg index f01f5f94d964c337c8f2bc12bd2d34485ca39f0f..87938e468c9406d599906dcd7f1de07afb1b30d3 100644 --- a/software/erp5testnode/testsuite/caddy-frontend/software.cfg +++ b/software/erp5testnode/testsuite/caddy-frontend/software.cfg @@ -73,3 +73,6 @@ slapos.test.caddy-frontend = erp5.util = slapos.recipe.template = 4.3 forcediphttpsadapter = 1.0.1 + +# slapos.test.caddy-frontend==0.0.1.dev0 +requests-toolbelt = 0.8.0 diff --git a/software/erp5testnode/testsuite/deploy-test/instance.cfg.in b/software/erp5testnode/testsuite/deploy-test/instance.cfg.in index 0844bd5e063f995f5c573029b08d72dfa44006e5..750740fd9c37f8003055cafbb579ec7481f46593 100644 --- a/software/erp5testnode/testsuite/deploy-test/instance.cfg.in +++ b/software/erp5testnode/testsuite/deploy-test/instance.cfg.in @@ -29,7 +29,7 @@ rendered = $${buildout:directory}/bin/$${:_buildout_section_name_} template = inline: #!/bin/sh export PATH=${python-with-eggs:location}:$PATH - exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory')}}" "$@" + exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory')}}" --python_interpreter=${buildout:bin-directory}/${runTestSuite_py:interpreter} "$@" mode = 0755 context = key slapparameter_dict slap-configuration:configuration diff --git a/software/erp5testnode/testsuite/deploy-test/runTestSuite.py b/software/erp5testnode/testsuite/deploy-test/runTestSuite.py index f2f11546d4b841e4d5104e9fc3cd44f40d235e75..5c80373e81784260f960adb64629e21daf87438f 100644 --- a/software/erp5testnode/testsuite/deploy-test/runTestSuite.py +++ b/software/erp5testnode/testsuite/deploy-test/runTestSuite.py @@ -143,14 +143,21 @@ def main(): '--test_location', help="Location of the tests" ) + parser.add_argument( + '--python_interpreter', + help="Path to python interpreter used to run the test suite" + ) args = parser.parse_args() revision = args.revision test_suite_title = args.test_suite_title or args.test_suite - os.environ['SOURCE_CODE_TO_TEST'] = args.test_location suite = testsuite.EggTestSuite( 1, test_suite=args.test_suite, node_quantity=args.node_quantity, + python_interpreter=args.python_interpreter, + egg_test_path_dict={ + os.path.basename(os.path.normpath(path)): path + for path in args.test_location.split(',')}, revision=revision) access_url_http = None access_url_https = None diff --git a/software/erp5testnode/testsuite/deploy-test/software.cfg b/software/erp5testnode/testsuite/deploy-test/software.cfg index baf2d71f200d9300dc4f1de0f7f913ff1382feb3..b828367471b237f20d165fd022e5ceda996ef946 100644 --- a/software/erp5testnode/testsuite/deploy-test/software.cfg +++ b/software/erp5testnode/testsuite/deploy-test/software.cfg @@ -72,7 +72,8 @@ location = ${:_profile_base_location_}/${:filename} recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg.in output = ${buildout:directory}/template.cfg +md5sum = 616abb7fb4608321e11ade0a43b0ce4b mode = 0644 [versions] -erp5.util = 0.4.50 +erp5.util = 0.4.51 diff --git a/software/htmlvalidatorserver/software.cfg b/software/htmlvalidatorserver/software.cfg index 341356b11b6ea5459a598682ae5f1850350d90b3..d6b62cc1fb972254470f432db67ed367f70d0624 100644 --- a/software/htmlvalidatorserver/software.cfg +++ b/software/htmlvalidatorserver/software.cfg @@ -73,7 +73,7 @@ dnspython = 1.15.0 # Required by: # slapos.toolbox==0.73 -erp5.util = 0.4.50 +erp5.util = 0.4.51 # Required by: # slapos.toolbox==0.73 diff --git a/software/jstestnode/software.cfg b/software/jstestnode/software.cfg index 8815f10f179b2065275437550184694ef2249c2d..f140f8a704d695c16b8c1752a4f057a2fa0c34d3 100644 --- a/software/jstestnode/software.cfg +++ b/software/jstestnode/software.cfg @@ -112,6 +112,6 @@ output = ${buildout:directory}/runTestSuite.in mode = 0644 [versions] -erp5.util = 0.4.50 +erp5.util = 0.4.51 slapos.recipe.template = 4.3 selenium = 3.8.0 diff --git a/software/kvm/common.cfg b/software/kvm/common.cfg deleted file mode 100644 index 8e0b413adabcab97e836a87175e556fb7660740f..0000000000000000000000000000000000000000 --- a/software/kvm/common.cfg +++ /dev/null @@ -1,254 +0,0 @@ -[buildout] - -extends = - ../../component/6tunnel/buildout.cfg - ../../component/curl/buildout.cfg - ../../component/dash/buildout.cfg - ../../component/qemu-kvm/buildout.cfg - ../../component/noVNC/buildout.cfg - ../../component/openssl/buildout.cfg - ../../component/netcat/buildout.cfg - ../../component/pycurl/buildout.cfg - ../../stack/slapos.cfg - ../../component/nodejs/buildout.cfg - ../../stack/resilient/buildout.cfg - -# stacks are listed from most generic to most specific, -# to avoid versioning issues -common-parts = - template - eggs - -# XXX: we have to manually add this for resilience - rdiff-backup - pbs-recipe-egg - - -parts = ${:common-parts} - -#XXX-Cedric : Currently, one can only access to KVM using noVNC. -# Ideally one should be able to access KVM by using either NoVNC or VNC. -# Problem is : no native crypto support in web browsers. So we have to disable ssl -# In qemu builtin vnc server, and make it available only for localhost -# so that only novnc can listen to it. - -#XXX-Cedric: Check status of https://github.com/kanaka/noVNC/issues/13 to see -# When qemu has builtin support for websockets in vnc server to get rid of -# Websockify (socket <-> websocket proxy server) when it is ready. -# May solve previous XXX depending on the implementation. - -#XXX-Cedric : add list of keyboard layouts (azerty/us querty/...) parameter to qemu - -[eggs] -recipe = zc.recipe.egg -interpreter = python.eggs -eggs = - ${python-cffi:egg} - ${python-cryptography:egg} - ${lxml-python:egg} - websockify - slapos.cookbook - slapos.toolbox - erp5.util - cns.recipe.symlink - collective.recipe.template - plone.recipe.command - ${pycurl:egg} - -[http-proxy] -# https://github.com/nodejitsu/node-http-proxy -recipe = slapos.recipe.build:download-unpacked -#XXX-Cedric : use upstream when merged -url = https://nodeload.github.com/desaintmartin/node-http-proxy/zipball/20120621 -md5sum = 20204d0b29c2cef26e1c91e99eedca6b - -[proxy-by-url] -# https://github.com/dominictarr/proxy-by-url -recipe = slapos.recipe.build:download-unpacked -#XXX-Cedric : use upstream when merged -url = https://nodeload.github.com/desaintmartin/proxy-by-url/zipball/20120621 -md5sum = c2609948aa708581f93b981b23880314 - -[npm-modules] -recipe = plone.recipe.command -destination = ${buildout:parts-directory}/${:_buildout_section_name_} -location = ${buildout:parts-directory}/${:_buildout_section_name_} -command = - export HOME=${:location}; - rm -fr ${:destination} && - mkdir -p ${:destination} && - cd ${:destination} && - ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install colors@0.6.0-1 && - ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io@0.8.7 && - ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io-client@0.8.7 && - ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install optimist@0.3.1 && - ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3 - - -# Create all templates that will be used to deploy instances - -[template] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/instance.cfg.in -md5sum = 5a17fc127190bbc19361c5ffb10711b3 -output = ${buildout:directory}/template.cfg -mode = 0644 - -[template-kvm] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2 -mode = 644 -md5sum = e59ea29533d7f989ec676e14b0f29839 -download-only = true -on-update = true - -[template-kvm-cluster] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in -mode = 644 -md5sum = ba3337b3678ed9d3578cc88749c5cd13 -download-only = true -on-update = true - -[template-kvm-resilient] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2 -mode = 644 -md5sum = 93e7143b46c6136b7cafe888fac90aba -download-only = true -on-update = true - -[template-kvm-import] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/instance-kvm-import.cfg.jinja2.in -md5sum = dc3f3ad9ebd8b3b5c3ded57b91cee9c7 -mode = 0644 -download-only = true -on-update = true - -[template-kvm-import-script] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/template/kvm-import.sh.jinja2 -filename = kvm-import.sh.jinja2 -md5sum = cd0008f1689dfca9b77370bc4d275b70 -download-only = true -mode = 0755 - -[template-kvm-export] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2 -mode = 644 -md5sum = fbad91193be6ebde5fc4c05a38a55e7b -download-only = true -on-update = true - -[template-kvm-export-script] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/template/kvm-export.sh.jinja2 -filename = kvm-export.sh.jinja2 -md5sum = b617d64de73de1eed518185f310bbc82 -download-only = true -mode = 0755 - -[template-nbd] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/instance-nbd.cfg.in -md5sum = f634a5249b773658b7a7bc9fa9bb0368 -output = ${buildout:directory}/template-nbd.cfg -mode = 0644 - -[template-frontend] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/instance-frontend.cfg.in -md5sum = cdb690495e9eb007d2b7d2f8e12f5c59 -output = ${buildout:directory}/template-frontend.cfg -mode = 0644 - -[template-ansible-promise] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/template/ansible-promise.in -md5sum = 2036bf145f472f62ef8dee5e729328fd -mode = 0644 -download-only = true -filename = ansible-promise.in - -[template-kvm-run] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/template/template-kvm-run.in -mode = 644 -filename = template-kvm-run.in -md5sum = c6f1536a3502102dadbfb9d82496cc36 -download-only = true -on-update = true - -[template-kvm-controller] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/template/kvm-controller-run.in -mode = 644 -filename = kvm-controller-run.in -md5sum = c86cd67bbdd26b7b14b7449a1bbd959b -download-only = true -on-update = true - -[template-apache-conf] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/template/apache.conf.in -mode = 644 -filename = apache.conf.in -md5sum = ac97f6a52e1c5a19a646242ef85abb8a -download-only = true -on-update = true - -[template-content] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/template/template-content.in -mode = 644 -filename = template-content.in -md5sum = 822737e483864bf255ad1259237bef2a -download-only = true -on-update = true - -[template-qemu-ready] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/template/qemu-is-ready.in -mode = 644 -filename = qemu-is-ready.in -md5sum = b304eec8e2cb71f10ea83cac22f6db12 -download-only = true -on-update = true - -[file-download-script] -recipe = hexagonit.recipe.download -ignore-existing = true -url = ${:_profile_base_location_}/template/download_file.in -mode = 644 -filename = download_file -md5sum = 599dbbbd438fe7801e3f8642ae9e9a78 -download-only = true -on-update = true - -[template-httpd] -recipe = slapos.recipe.template:jinja2 -filename = template-httpd.cfg -template = ${:_profile_base_location_}/instance-kvm-http.cfg.in -rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/instance-kvm-http.cfg -md5sum = 26a181a48046ce88570adb32334747ef -context = - key apache_location apache:location - raw openssl_executable_location ${openssl:location}/bin/openssl - raw template_apache_conf ${template-apache-conf:location}/${template-apache-conf:filename} - - diff --git a/software/kvm/development.cfg b/software/kvm/development.cfg index 400c007b3d9407c5dc89b5ef5d48c716819fe3a2..fc0983bea9e14b87f0c7fcc8dcd11ed953b292b3 100644 --- a/software/kvm/development.cfg +++ b/software/kvm/development.cfg @@ -4,7 +4,7 @@ # allowing to play with bleeding edge environment. [buildout] -extends = common.cfg +extends = software.cfg ../../stack/slapos-dev.cfg parts += diff --git a/software/kvm/software.cfg b/software/kvm/software.cfg index 0191179a36d8b3853b8b1c39b53609cc123f489e..28d4e44fa5bb56c2748963465750a65cabb31d23 100644 --- a/software/kvm/software.cfg +++ b/software/kvm/software.cfg @@ -1,12 +1,262 @@ [buildout] -extends = common.cfg + +extends = + ../../component/6tunnel/buildout.cfg + ../../component/curl/buildout.cfg + ../../component/dash/buildout.cfg + ../../component/qemu-kvm/buildout.cfg + ../../component/noVNC/buildout.cfg + ../../component/openssl/buildout.cfg + ../../component/netcat/buildout.cfg + ../../component/pycurl/buildout.cfg + ../../stack/slapos.cfg + ../../component/nodejs/buildout.cfg + ../../stack/resilient/buildout.cfg + +# stacks are listed from most generic to most specific, +# to avoid versioning issues +common-parts = + template + eggs + +# XXX: we have to manually add this for resilience + rdiff-backup + pbs-recipe-egg + + +parts = ${:common-parts} + +#XXX-Cedric : Currently, one can only access to KVM using noVNC. +# Ideally one should be able to access KVM by using either NoVNC or VNC. +# Problem is : no native crypto support in web browsers. So we have to disable ssl +# In qemu builtin vnc server, and make it available only for localhost +# so that only novnc can listen to it. + +#XXX-Cedric: Check status of https://github.com/kanaka/noVNC/issues/13 to see +# When qemu has builtin support for websockets in vnc server to get rid of +# Websockify (socket <-> websocket proxy server) when it is ready. +# May solve previous XXX depending on the implementation. + +#XXX-Cedric : add list of keyboard layouts (azerty/us querty/...) parameter to qemu + +[eggs] +recipe = zc.recipe.egg +interpreter = python.eggs +eggs = + ${python-cffi:egg} + ${python-cryptography:egg} + ${lxml-python:egg} + websockify + slapos.cookbook + slapos.toolbox + erp5.util + cns.recipe.symlink + collective.recipe.template + plone.recipe.command + ${pycurl:egg} + +[http-proxy] +# https://github.com/nodejitsu/node-http-proxy +recipe = slapos.recipe.build:download-unpacked +#XXX-Cedric : use upstream when merged +url = https://nodeload.github.com/desaintmartin/node-http-proxy/zipball/20120621 +md5sum = 20204d0b29c2cef26e1c91e99eedca6b + +[proxy-by-url] +# https://github.com/dominictarr/proxy-by-url +recipe = slapos.recipe.build:download-unpacked +#XXX-Cedric : use upstream when merged +url = https://nodeload.github.com/desaintmartin/proxy-by-url/zipball/20120621 +md5sum = c2609948aa708581f93b981b23880314 + +[npm-modules] +recipe = plone.recipe.command +destination = ${buildout:parts-directory}/${:_buildout_section_name_} +location = ${buildout:parts-directory}/${:_buildout_section_name_} +command = + export HOME=${:location}; + rm -fr ${:destination} && + mkdir -p ${:destination} && + cd ${:destination} && + ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install colors@0.6.0-1 && + ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io@0.8.7 && + ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io-client@0.8.7 && + ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install optimist@0.3.1 && + ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3 + + +# Create all templates that will be used to deploy instances + +[template] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance.cfg.in +md5sum = 5a17fc127190bbc19361c5ffb10711b3 +output = ${buildout:directory}/template.cfg +mode = 0644 + +[template-kvm] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2 +mode = 644 +md5sum = e59ea29533d7f989ec676e14b0f29839 +download-only = true +on-update = true + +[template-kvm-cluster] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in +mode = 644 +md5sum = ba3337b3678ed9d3578cc88749c5cd13 +download-only = true +on-update = true + +[template-kvm-resilient] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2 +mode = 644 +md5sum = 93e7143b46c6136b7cafe888fac90aba +download-only = true +on-update = true + +[template-kvm-import] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/instance-kvm-import.cfg.jinja2.in +md5sum = dc3f3ad9ebd8b3b5c3ded57b91cee9c7 +mode = 0644 +download-only = true +on-update = true + +[template-kvm-import-script] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/template/kvm-import.sh.jinja2 +filename = kvm-import.sh.jinja2 +md5sum = cd0008f1689dfca9b77370bc4d275b70 +download-only = true +mode = 0755 + +[template-kvm-export] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2 +mode = 644 +md5sum = fbad91193be6ebde5fc4c05a38a55e7b +download-only = true +on-update = true + +[template-kvm-export-script] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/template/kvm-export.sh.jinja2 +filename = kvm-export.sh.jinja2 +md5sum = b617d64de73de1eed518185f310bbc82 +download-only = true +mode = 0755 + +[template-nbd] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance-nbd.cfg.in +md5sum = f634a5249b773658b7a7bc9fa9bb0368 +output = ${buildout:directory}/template-nbd.cfg +mode = 0644 + +[template-frontend] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance-frontend.cfg.in +md5sum = cdb690495e9eb007d2b7d2f8e12f5c59 +output = ${buildout:directory}/template-frontend.cfg +mode = 0644 + +[template-ansible-promise] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/template/ansible-promise.in +md5sum = 2036bf145f472f62ef8dee5e729328fd +mode = 0644 +download-only = true +filename = ansible-promise.in + +[template-kvm-run] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/template/template-kvm-run.in +mode = 644 +filename = template-kvm-run.in +md5sum = c6f1536a3502102dadbfb9d82496cc36 +download-only = true +on-update = true + +[template-kvm-controller] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/template/kvm-controller-run.in +mode = 644 +filename = kvm-controller-run.in +md5sum = c86cd67bbdd26b7b14b7449a1bbd959b +download-only = true +on-update = true + +[template-apache-conf] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/template/apache.conf.in +mode = 644 +filename = apache.conf.in +md5sum = ac97f6a52e1c5a19a646242ef85abb8a +download-only = true +on-update = true + +[template-content] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/template/template-content.in +mode = 644 +filename = template-content.in +md5sum = 822737e483864bf255ad1259237bef2a +download-only = true +on-update = true + +[template-qemu-ready] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/template/qemu-is-ready.in +mode = 644 +filename = qemu-is-ready.in +md5sum = b304eec8e2cb71f10ea83cac22f6db12 +download-only = true +on-update = true + +[file-download-script] +recipe = hexagonit.recipe.download +ignore-existing = true +url = ${:_profile_base_location_}/template/download_file.in +mode = 644 +filename = download_file +md5sum = 599dbbbd438fe7801e3f8642ae9e9a78 +download-only = true +on-update = true + +[template-httpd] +recipe = slapos.recipe.template:jinja2 +filename = template-httpd.cfg +template = ${:_profile_base_location_}/instance-kvm-http.cfg.in +rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/instance-kvm-http.cfg +md5sum = 26a181a48046ce88570adb32334747ef +context = + key apache_location apache:location + raw openssl_executable_location ${openssl:location}/bin/openssl + raw template_apache_conf ${template-apache-conf:location}/${template-apache-conf:filename} [versions] # XXX - use websockify = 0.5.1 for compatibility with kvm frontend websockify = 0.5.1 slapos.toolbox = 0.76 -erp5.util = 0.4.49 +erp5.util = 0.4.51 apache-libcloud = 1.1.0 collective.recipe.environment = 0.2.0 gitdb = 0.6.4 diff --git a/software/monitor/software.cfg b/software/monitor/software.cfg index a0842c092066020df317fd0c4e77f4ef78cbe4c0..89032f9adc2c2cd0f8f7cc12b64a2aaeb88cad77 100644 --- a/software/monitor/software.cfg +++ b/software/monitor/software.cfg @@ -99,7 +99,7 @@ eggs += [versions] slapos.recipe.template = 4.3 dnspython = 1.15.0 -erp5.util = 0.4.50 +erp5.util = 0.4.51 passlib = 1.7.1 GitPython = 2.1.9 lockfile = 0.12.2 diff --git a/software/neoppod/runTestSuite.in b/software/neoppod/runTestSuite.in index 2f6de2d442064286baec7a9a259afdda2dc5875d..4e0a1615cabdb489d56964feea313d08b65de91d 100644 --- a/software/neoppod/runTestSuite.in +++ b/software/neoppod/runTestSuite.in @@ -71,7 +71,7 @@ def main(): test_name_list = 'SQLite', 'MySQL' - tool = taskdistribution.TaskDistributionTool(portal_url = args.master_url) + tool = taskdistribution.TaskDistributor(portal_url = args.master_url) test_result = tool.createTestResult(revision = revision, test_name_list = test_name_list, node_title = args.test_node_title, diff --git a/software/neoppod/software-common.cfg b/software/neoppod/software-common.cfg index ae5a30bf2d1d1d752556eb20219adbb7b0f22c0b..76bd2a98d8541db1befbefdce950e3c7034b08c3 100644 --- a/software/neoppod/software-common.cfg +++ b/software/neoppod/software-common.cfg @@ -135,6 +135,8 @@ transaction = 1.7.0 zodbpickle = 0.6.0 zodbtools = 0.0.0.dev4 cython-zstd = 0.2 +python-dateutil = 2.7.3 +pyasn1 = 0.4.3 # Required by: # slapos.toolbox==0.71 diff --git a/software/neoppod/software.cfg b/software/neoppod/software.cfg index bb4def170bcc5c1ed804c3f6696d1ae92ea9467a..2abd8908940f21056ec02183965db2d784b58b8e 100644 --- a/software/neoppod/software.cfg +++ b/software/neoppod/software.cfg @@ -26,7 +26,7 @@ md5sum = ee8401a4e7d82bf488a57e3399f9ce48 [runTestSuite.in] recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/${:_buildout_section_name_} -md5sum = d8bba690cf950613d5576ff23813df59 +md5sum = b656e805c5dbc7f9c73716398b3e032e [runTestSuite_py] recipe = zc.recipe.egg @@ -34,7 +34,7 @@ eggs = erp5.util interpreter = ${:_buildout_section_name_} [versions] -erp5.util = 0.4.49 +erp5.util = 0.4.51 # To match ERP5 ZConfig = 2.9.3 zc.lockfile = 1.0.2 diff --git a/software/neotest/software.cfg b/software/neotest/software.cfg index 4df99262c74316ed2dc7f0bb816799e549593fba..71f66d52aa2e11645683a246822fd8172efb7cc1 100644 --- a/software/neotest/software.cfg +++ b/software/neotest/software.cfg @@ -110,6 +110,8 @@ eggs = ZEO # for nxd/runTestSuite erp5.util +# for e.g. tcpu.py + pygolang # wendelin.core: latest not yet released @@ -122,7 +124,8 @@ pyasn1 = 0.3.7 ZODB3 = 3.11.0 numpy = 1.14.2 zope.testing = 4.6.2 -erp5.util = 0.4.50 +pygolang = 0.0.0.dev4 +erp5.util = 0.4.51 # Required by: # ZEO==4.3.1 diff --git a/software/nginx-push-stream/software.cfg b/software/nginx-push-stream/software.cfg index 6f31e3098d6cd00ffb4ca035d0ac4a547e9c3601..c7b01e2b608ff89ebe8d0e87f07b85cade1303a2 100644 --- a/software/nginx-push-stream/software.cfg +++ b/software/nginx-push-stream/software.cfg @@ -49,7 +49,7 @@ slapos.recipe.template = 4.3 slapos.toolbox = 0.76 dnspython = 1.15.0 PyRSS2Gen = 1.1 -erp5.util = 0.4.50 +erp5.util = 0.4.51 passlib = 1.7.1 GitPython = 2.1.8 lockfile = 0.12.2 diff --git a/software/re6stnet/software.cfg b/software/re6stnet/software.cfg index 07795a4e4226b9d00c4e2c5fefa51676995f765d..68511cb947dfe9f3b835cb5c2d1bcde90eca873b 100644 --- a/software/re6stnet/software.cfg +++ b/software/re6stnet/software.cfg @@ -121,7 +121,7 @@ slapos.recipe.template = 4.3 slapos.toolbox = 0.76 smmap = 0.9.0 dnspython = 1.15.0 -erp5.util = 0.4.50 +erp5.util = 0.4.51 passlib = 1.7.1 # Required by: diff --git a/software/slapos-testing/buildout.hash.cfg b/software/slapos-testing/buildout.hash.cfg index 418f7cf586bc87e78a5cea39dfcd294842dd994a..78e5e7127a41eb91d9266e87d63d1446b16fa4aa 100644 --- a/software/slapos-testing/buildout.hash.cfg +++ b/software/slapos-testing/buildout.hash.cfg @@ -15,5 +15,5 @@ [template] filename = instance.cfg -md5sum = 9dece9d12dc94bf5c35d307cc8aa4d6b +md5sum = d361db5f94e8c568e2aa44014d0ba91b diff --git a/software/slapos-testing/instance.cfg b/software/slapos-testing/instance.cfg index 5e50aff78e6a14af73dc93c0e12169969c25a8c4..dff552001772412770bd68ddb82886846af960f5 100644 --- a/software/slapos-testing/instance.cfg +++ b/software/slapos-testing/instance.cfg @@ -30,6 +30,10 @@ git-executable = ${git:location}/bin/git <= download-source repository = ${caucase-repository:location} +[slapos.libnetworkcache] +<= download-source +repository = ${slapos.libnetworkcache-repository:location} + [erp5.util] <= download-source repository = ${erp5.util-repository:location} @@ -65,7 +69,7 @@ wrapper-path = $${create-directory:bin}/runTestSuite command-line = ${buildout:bin-directory}/runTestSuite --python_interpreter=${buildout:bin-directory}/${eggs:interpreter} - --source_code_path_list=$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location} + --source_code_path_list=$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location},$${slapos.libnetworkcache:location} # Notes about environment: # * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests @@ -74,7 +78,7 @@ command-line = # /usr/bin and /bin in $PATH # * LOCAL_IPV4 is needed for some slapos.core tests environment = - PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:/usr/bin/:/bin/ + PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:/usr/bin/:/bin/ LOCAL_IPV4=$${slap-configuration:ipv4-random} diff --git a/software/slapos-testing/software.cfg b/software/slapos-testing/software.cfg index beb80b33e696d538f3f710c9c77426bc8b12075b..45f376317a8cd2c4d3b3f0095fc65de1238e0bca 100644 --- a/software/slapos-testing/software.cfg +++ b/software/slapos-testing/software.cfg @@ -10,6 +10,7 @@ extends = ../../component/phantomjs/buildout.cfg ../../component/pycurl/buildout.cfg ../../component/coreutils/buildout.cfg + ../../component/socat/buildout.cfg ../../stack/slapos.cfg ./buildout.hash.cfg @@ -33,6 +34,11 @@ recipe = zc.recipe.egg:develop egg = caucase setup = ${caucase-repository:location} +[slapos.libnetworkcache-setup] +<= setup-develop-egg +egg = slapos.libnetworkcache +setup = ${slapos.libnetworkcache-repository:location} + [erp5.util-setup] <= setup-develop-egg # XXX erp5.util does not have `test` extra require, but has a `testnode` extra require with same dependencies @@ -95,6 +101,7 @@ eggs = ${slapos.recipe.cmmi-setup:egg} ${slapos.recipe.template-setup:egg} ${slapos.toolbox-setup:egg} + ${slapos.libnetworkcache-setup:egg} mock zope.testing httmock @@ -132,6 +139,10 @@ repository = https://lab.nexedi.com/nexedi/slapos.core.git <= git-clone-repository repository = https://lab.nexedi.com/nexedi/slapos.recipe.template.git +[slapos.libnetworkcache-repository] +<= git-clone-repository +repository = https://lab.nexedi.com/nexedi/slapos.libnetworkcache.git + [slapos.recipe.build-repository] <= git-clone-repository repository = https://lab.nexedi.com/nexedi/slapos.recipe.build.git @@ -161,12 +172,47 @@ mode = 640 [versions] Pygments = 2.1.3 -# clear the version of tested eggs, to make sure we installed the developped ones -caucase = -erp5.util = -slapos.cookbook = -slapos.core = -slapos.recipe.build = -slapos.recipe.cmmi = -slapos.recipe.template = -slapos.toolbox = +slapos.recipe.build = 0.36 +slapos.recipe.cmmi = 0.7 +slapos.recipe.template = 4.3 +slapos.toolbox = 0.76 + +# All depencies should be pinned. +apache-libcloud = 2.3.0 +bcrypt = 3.1.4 +dnspython = 1.15.0 +funcsigs = 1.0.2 +gitdb2 = 2.0.4 +httmock = 1.2.6 +manuel = 1.9.0 +mock = 2.0.0 +pem = 18.1.0 +pyasn1 = 0.4.3 +pycurl = 7.43.0.2 +pyflakes = 2.0.0 +smmap2 = 2.0.4 +zope.testing = 4.6.2 + +# Required by: +# slapos.toolbox==0.76 +GitPython = 2.1.11 + +# Required by: +# slapos.toolbox==0.76 +PyRSS2Gen = 1.1 + +# Required by: +# slapos.toolbox==0.76 +atomize = 0.2.0 + +# Required by: +# slapos.toolbox==0.76 +feedparser = 5.2.1 + +# Required by: +# slapos.toolbox==0.76 +lockfile = 0.12.2 + +# Required by: +# slapos.toolbox==0.76 +passlib = 1.7.1 diff --git a/software/slaprunner/buildout.hash.cfg b/software/slaprunner/buildout.hash.cfg index 7589442a359daa797e592be2de27e0ef484a778e..4d8a60be566d2c170aecc90cb8f51ad027c1d0eb 100644 --- a/software/slaprunner/buildout.hash.cfg +++ b/software/slaprunner/buildout.hash.cfg @@ -22,7 +22,7 @@ md5sum = 04e31ac503753f89510dd412b4680c56 [template-runner-import-script] filename = template/runner-import.sh.jinja2 -md5sum = ab5f0ae6febc0d5c247ec5542b5f0519 +md5sum = e033845c9c24e4bb20caeedf19f9628a [instance-runner-import] filename = instance-runner-import.cfg.in diff --git a/software/slaprunner/common.cfg b/software/slaprunner/common.cfg deleted file mode 100644 index f7e633d358e97e37922c0a73ad714d771a41464e..0000000000000000000000000000000000000000 --- a/software/slaprunner/common.cfg +++ /dev/null @@ -1,162 +0,0 @@ -[buildout] -extends = - buildout.hash.cfg - ../../component/bash/buildout.cfg - ../../component/busybox/buildout.cfg - ../../component/curl/buildout.cfg - ../../component/dash/buildout.cfg - ../../component/dcron/buildout.cfg - ../../component/git/buildout.cfg - ../../component/tig/buildout.cfg - ../../component/logrotate/buildout.cfg - ../../component/lxml-python/buildout.cfg - ../../component/nano/buildout.cfg - ../../component/nginx/buildout.cfg - ../../component/openssh/buildout.cfg - ../../component/mosh/buildout.cfg - ../../component/rsync/buildout.cfg - ../../component/pycurl/buildout.cfg - ../../component/python-2.7/buildout.cfg - ../../component/screen/buildout.cfg - ../../component/shellinabox/buildout.cfg - ../../component/vim/buildout.cfg - ../../component/zip/buildout.cfg - ../../stack/slapos.cfg - ../../stack/flask.cfg - ../../stack/resilient/buildout.cfg - ../../stack/monitor/buildout.cfg - -# stacks are listed from most generic to most specific, -# to avoid versioning issues - -common-parts = - template - eggs - instance-runner-import - instance-runner-export - template-slapos-cfg - template-slapuser-script -# XXX: we have to manually add this for resilience - rdiff-backup - pbs-recipe-egg - -parts = - ${:common-parts} - -# Use shellinabox from github with AF_UNIX support -[shellinabox] -<= shellinabox-github - -[template-base] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/${:filename} -mode = 0644 - -[download-base] -recipe = hexagonit.recipe.download -url = ${:_profile_base_location_}/${:filename} -mode = 0644 - -[download-only-base] -< = download-base -ignore-existing = true -download-only = true - -[template-download-base] -# Downloads from template directory into current directory -< = download-only-base -url = ${:_profile_base_location_}/template/${:filename} -location = ${buildout:parts-directory}/${:_buildout_section_name_} - -[template] -< = template-base -output = ${buildout:directory}/template.cfg - -[template-runner] -< = template-base -output = ${buildout:directory}/template-runner.cfg.in - -[template-runner-import-script] -< = template-download-base -filename = runner-import.sh.jinja2 - -[template-runner-export-script] -< = template-download-base -filename = runner-export.sh.jinja2 - -[instance-runner-import] -< = download-base -recipe = slapos.recipe.build:download - -[instance-runner-export] -< = download-base -recipe = slapos.recipe.build:download - -[template-resilient] -< = download-base -recipe = slapos.recipe.build:download - -[template_nginx_conf] -< = download-only-base - -[template_httpd_conf] -< = download-only-base - -[template_launcher] -< = download-base -recipe = slapos.recipe.build:download - -[template-slapos-cfg] -< = template-download-base -filename = slapos.cfg.in - -[template-parameters] -< = download-only-base - -[template-bash-profile] -< = template-download-base -filename = bash_profile.in - -[template-supervisord] -< = template-download-base -filename = supervisord.conf.in - -[template-listener-slapgrid] -< = template-download-base -filename = listener_slapgrid.py.in - -[monitor-check-webrunner-internal-instance] -< = template-download-base -destination = ${:location}/${:filename} -filename = monitor-check-webrunner-internal-instances.py - -[template-resilient-software-release-information] -< = template-download-base -filename = resilient_software_release_information.py.in - -[template-slapuser-script] -< = template-download-base -filename = slapos-slapuser-script.in - -[eggs] -recipe = zc.recipe.egg -eggs = - ${pycurl:egg} - collective.recipe.template - cns.recipe.symlink - erp5.util - lock-file - plone.recipe.command - slapos.recipe.build - slapos.toolbox[flask_auth] - gunicorn==19.7.1 - futures - ${slapos-cookbook:eggs} - slapos.core # listed explicitly for scripts generation - -[extra-eggs] -recipe = zc.recipe.egg -interpreter = pythonwitheggs -eggs += - supervisor - diff --git a/software/slaprunner/development.cfg b/software/slaprunner/development.cfg index 1ab5b4b85acee9add2698f8f3e40d676da37183c..b29f3b20f06a3a07ae493359e1ceae0e74df7478 100644 --- a/software/slaprunner/development.cfg +++ b/software/slaprunner/development.cfg @@ -4,7 +4,7 @@ # allowing to play with bleeding edge environment. [buildout] -extends = common.cfg +extends = software.cfg ../../stack/slapos-dev.cfg parts += @@ -15,4 +15,3 @@ parts += slapos.cookbook = slapos.core = slapos.toolbox = -lockfile = diff --git a/software/slaprunner/software.cfg b/software/slaprunner/software.cfg index bdfc9713ba581a787a966981b8dff87406b759d8..eda4f89d0e2a87fd0b020528e040933cae9d5d58 100644 --- a/software/slaprunner/software.cfg +++ b/software/slaprunner/software.cfg @@ -1,10 +1,164 @@ -# Production profile of slaprunner. -# Exactly the same as common.cfg, but: -# 1/ Use a defined set of Python eggs instead of using the latest available -# ones from Pypi, to ensure stability; - [buildout] -extends = common.cfg +extends = + buildout.hash.cfg + ../../component/bash/buildout.cfg + ../../component/busybox/buildout.cfg + ../../component/curl/buildout.cfg + ../../component/dash/buildout.cfg + ../../component/dcron/buildout.cfg + ../../component/git/buildout.cfg + ../../component/tig/buildout.cfg + ../../component/logrotate/buildout.cfg + ../../component/lxml-python/buildout.cfg + ../../component/nano/buildout.cfg + ../../component/nginx/buildout.cfg + ../../component/openssh/buildout.cfg + ../../component/mosh/buildout.cfg + ../../component/rsync/buildout.cfg + ../../component/pycurl/buildout.cfg + ../../component/python-2.7/buildout.cfg + ../../component/screen/buildout.cfg + ../../component/shellinabox/buildout.cfg + ../../component/vim/buildout.cfg + ../../component/zip/buildout.cfg + ../../stack/slapos.cfg + ../../stack/flask.cfg + ../../stack/resilient/buildout.cfg + ../../stack/monitor/buildout.cfg + +# stacks are listed from most generic to most specific, +# to avoid versioning issues + +common-parts = + template + eggs + instance-runner-import + instance-runner-export + template-slapos-cfg + template-slapuser-script +# XXX: we have to manually add this for resilience + rdiff-backup + pbs-recipe-egg + +parts = + ${:common-parts} + +# Use shellinabox from github with AF_UNIX support +[shellinabox] +<= shellinabox-github + +[template-base] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/${:filename} +mode = 0644 + +[download-base] +recipe = hexagonit.recipe.download +url = ${:_profile_base_location_}/${:filename} +mode = 0644 + +[download-only-base] +< = download-base +ignore-existing = true +download-only = true + +[template-download-base] +# Downloads from template directory into current directory +< = download-only-base +url = ${:_profile_base_location_}/template/${:filename} +location = ${buildout:parts-directory}/${:_buildout_section_name_} + +[template] +< = template-base +output = ${buildout:directory}/template.cfg + +[template-runner] +< = template-base +output = ${buildout:directory}/template-runner.cfg.in + +[template-runner-import-script] +< = template-download-base +filename = runner-import.sh.jinja2 + +[template-runner-export-script] +< = template-download-base +filename = runner-export.sh.jinja2 + +[instance-runner-import] +< = download-base +recipe = slapos.recipe.build:download + +[instance-runner-export] +< = download-base +recipe = slapos.recipe.build:download + +[template-resilient] +< = download-base +recipe = slapos.recipe.build:download + +[template_nginx_conf] +< = download-only-base + +[template_httpd_conf] +< = download-only-base + +[template_launcher] +< = download-base +recipe = slapos.recipe.build:download + +[template-slapos-cfg] +< = template-download-base +filename = slapos.cfg.in + +[template-parameters] +< = download-only-base + +[template-bash-profile] +< = template-download-base +filename = bash_profile.in + +[template-supervisord] +< = template-download-base +filename = supervisord.conf.in + +[template-listener-slapgrid] +< = template-download-base +filename = listener_slapgrid.py.in + +[monitor-check-webrunner-internal-instance] +< = template-download-base +destination = ${:location}/${:filename} +filename = monitor-check-webrunner-internal-instances.py + +[template-resilient-software-release-information] +< = template-download-base +filename = resilient_software_release_information.py.in + +[template-slapuser-script] +< = template-download-base +filename = slapos-slapuser-script.in + +[eggs] +recipe = zc.recipe.egg +eggs = + ${pycurl:egg} + collective.recipe.template + cns.recipe.symlink + erp5.util + lock-file + plone.recipe.command + slapos.recipe.build + slapos.toolbox[flask_auth] + gunicorn==19.7.1 + futures + ${slapos-cookbook:eggs} + slapos.core # listed explicitly for scripts generation + +[extra-eggs] +recipe = zc.recipe.egg +interpreter = pythonwitheggs +eggs += + supervisor [versions] Flask-Auth = 0.85 @@ -19,6 +173,7 @@ slapos.recipe.template = 4.3 collective.recipe.environment = 0.2.0 slapos.toolbox = 0.76 smmap = 0.9.0 +lockfile = 0.12.2 # Required by: # slapos.toolbox==0.71 @@ -38,7 +193,7 @@ dnspython = 1.14.0 # Required by: # slapos.toolbox==0.71 -erp5.util = 0.4.49 +erp5.util = 0.4.51 # Required by: # slapos.toolbox==0.71 diff --git a/software/slaprunner/template/runner-import.sh.jinja2 b/software/slaprunner/template/runner-import.sh.jinja2 index a2399e09f4a4961b1ca95b7c8a37231e2f85941e..93c1ba26a46efdc05151aa406f747f2ef3e3f710 100644 --- a/software/slaprunner/template/runner-import.sh.jinja2 +++ b/software/slaprunner/template/runner-import.sh.jinja2 @@ -157,6 +157,19 @@ SLAPOSCFG='{{ supervisord["slapos-cfg"] }}' SLAPGRIDSRLOG='{{ supervisord["slapgrid-sr-log"] }}' SLAPGRIDCPLOG='{{ supervisord["slapgrid-cp-log"] }}' +contain_software_release=0 + +SOFTWARE_RELEASES_COUNT=$("$SQLITE3" "$DATABASE" 'SELECT count(1) FROM software11 WHERE url != "";') +if [ $SOFTWARE_RELEASES_COUNT -gt 0 ]; then + contain_software_release=1 +fi + +if [ $contain_software_release -eq 0 ]; then + log_message "No Software Release were deployed, so skip to continue..." + echo 0 > $RESTORE_EXIT_CODE_FILE + exit 0 +fi + log_message "Building newest Software Release..." "$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || "$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || diff --git a/stack/caucase/buildout.cfg b/stack/caucase/buildout.cfg index e894d4ce8e95ca097d78db2b64b4e6a582dbab8a..64fcd3b98765d7275dc238c100f59b706f5ab05e 100644 --- a/stack/caucase/buildout.cfg +++ b/stack/caucase/buildout.cfg @@ -157,7 +157,7 @@ dnspython = 1.15.0 # Required by: # slapos.toolbox==0.71 -erp5.util = 0.4.49 +erp5.util = 0.4.51 # Required by: # slapos.toolbox==0.71 diff --git a/stack/cloudooo.cfg b/stack/cloudooo.cfg index d88f96edfc064609115ad787c5f147ce1b3cec5a..4a81e8b64d5cc7a360b2c7e8f4649f89970af6c8 100644 --- a/stack/cloudooo.cfg +++ b/stack/cloudooo.cfg @@ -96,4 +96,4 @@ PasteDeploy = 1.5.2 # Required by: # cloudooo==1.2.5.dev0 -erp5.util = 0.4.49 +erp5.util = 0.4.51 diff --git a/stack/slapos-dev.cfg b/stack/slapos-dev.cfg index f8f389047cc2f8903de56680fc15bdd4129e83a6..f5f4d5eb5c0aa6141324c0073df7feb95a43195e 100644 --- a/stack/slapos-dev.cfg +++ b/stack/slapos-dev.cfg @@ -55,4 +55,3 @@ slapos.cookbook = slapos.core = slapos.toolbox = erp5-util = -lockfile =