Merge remote-tracking branch 'origin/master' into zope4py2

component/golang/buildout.cfg

@@ -48,11 +48,20 @@ md5sum = dbf727a4b0e365bf88d97cbfde590016
 environment-extra =
 # build Go without testing it
 # NOTE go1.4 does not have build cache
-make-targets= cd src && unset GOBIN && ./make.bash && cp -alf .. ${:location}
+setarch =
+make-targets= cd src && unset GOBIN && ${:setarch} ./make.bash && cp -alf .. ${:location}
+
+# go1.4 is used for bootstrap and does not support CGO
+environment-extra =
+  CGO_ENABLED=0
 
 # skip-chown-tests.patch does not apply to go1.4, but we don't run go1.4 tests.
 patches =
 
+# go1.4 does not have support for arm64 - build it in arm32 mode
+[golang14:platform.machine() == 'aarch64']
+setarch = setarch arm
+
 
 [golang1.12]
 <= golang-common

component/jupyter-py2/buildout.cfg

@@ -75,7 +75,6 @@ scikit-learn = 0.18.1
 statsmodels = 0.8.0
 terminado = 0.6
 tornado = 4.4.2
-traitlets = 4.3.3
 
 # nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
 nbconvert = 4.1.0

component/libiconv/buildout.cfg

 [buildout]
-extends =
-  ../patch/buildout.cfg
 parts =
   libiconv
 
 [libiconv]
-patch-binary = ${patch:location}/bin/patch
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/libiconv.gets.patch#8a20d8afe0617fce56f77537d2b84621
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.14.tar.gz
-md5sum = e34509b1623cec449dfeb73d7ce9c6c6
+url = http://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.17.tar.gz
+md5sum = d718cd5a59438be666d1575855be72c3

component/libiconv/libiconv.gets.patch

---- libiconv-1.14.orig/srclib/stdio.in.h        2011-08-07 13:42:06.000000000 +0000
-+++ libiconv-1.14/srclib/stdio.in.h     2013-01-09 19:56:21.115819812 +0000
-@@ -680,22 +680,7 @@
- #endif
- 
- #if @GNULIB_GETS@
--# if @REPLACE_STDIO_READ_FUNCS@ && @GNULIB_STDIO_H_NONBLOCKING@
--#  if !(defined __cplusplus && defined GNULIB_NAMESPACE)
--#   undef gets
--#   define gets rpl_gets
--#  endif
--_GL_FUNCDECL_RPL (gets, char *, (char *s) _GL_ARG_NONNULL ((1)));
--_GL_CXXALIAS_RPL (gets, char *, (char *s));
--# else
--_GL_CXXALIAS_SYS (gets, char *, (char *s));
--#  undef gets
--# endif
--_GL_CXXALIASWARN (gets);
--/* It is very rare that the developer ever has full control of stdin,
--   so any use of gets warrants an unconditional warning.  Assume it is
--   always declared, since it is required by C89.  */
--_GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");
-+#undef gets
- #endif

component/theia/buildout.cfg

@@ -38,6 +38,7 @@ environment =
   PATH=${nodejs:location}/bin:${pkgconfig:location}/bin:${python3:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${libsecret:pkg-config-path}
   LDFLAGS=-Wl,-rpath=${libsecret:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${glib:location}/lib
+  NODE_OPTIONS=--max_old_space_size=4096
 pre-configure =
   mkdir -p $TMPDIR
   echo '${package.json:content}' > %(location)s/package.json

software/erp5/instance-erp5-input-schema.json

@@ -145,7 +145,7 @@
       "type": "object"
     },
     "hosts-dict": {
-      "description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-cloudooo, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)",
+      "description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)",
       "patternProperties": {
         ".*": {
           "description": "An IP or domain name to which current entry will resolve",
@@ -294,11 +294,14 @@
       ],
       "type": "object"
     },
-    "cloudooo-url": {
-      "description": "Format conversion service URL",
-      "pattern": "^https?://",
-      "type": "string",
-      "format": "uri"
+    "cloudooo-url-list": {
+      "description": "Format conversion service URLs",
+      "type": "array",
+      "items": {
+        "pattern": "^https?://",
+        "type": "string",
+        "format": "uri"
+      }
     },
     "cloudooo-retry-count": {
       "description": "Define retry count for cloudooo in network error case in test",

software/erp5/test/test/test_erp5.py

@@ -28,6 +28,7 @@
 
 import contextlib
 import glob
+import http.client
 import json
 import os
 import shutil
@@ -442,17 +443,20 @@ class ZopeSkinsMixin:
   def _setUpClass(cls):
     super()._setUpClass()
     param_dict = cls.getRootPartitionConnectionParameterDict()
-    with cls.getXMLRPCClient() as erp5_xmlrpc_client:
-      # wait for ERP5 to be ready (TODO: this should probably be a promise)
-      for _ in range(120):
-        time.sleep(1)
+    # wait for ERP5 to be ready and have processed all activities
+    # from initial setup
+    for _ in range(120):
+      with cls.getXMLRPCClient() as erp5_xmlrpc_client:
         try:
-          erp5_xmlrpc_client.getTitle()
+          if erp5_xmlrpc_client.portal_activities.countMessage() == 0:
+            break
         except (xmlrpc.client.ProtocolError,
-                xmlrpc.client.Fault):
+                xmlrpc.client.Fault,
+                http.client.HTTPException):
           pass
-        else:
-          break
+      time.sleep(5)
+    else:
+      raise AssertionError("ERP5 is not ready")
 
   @classmethod
   def _getAuthenticatedZopeUrl(cls, path, family_name='default'):
@@ -902,3 +906,65 @@ class TestZopePublisherTimeout(ZopeSkinsMixin, ERP5InstanceTestCase):
         self._getAuthenticatedZopeUrl('ERP5Site_doSlowRequest', family_name='no-timeout'),
         verify=False,
         timeout=6)
+
+
+class TestCloudooo(ZopeSkinsMixin, ERP5InstanceTestCase):
+  """Test ERP5 can be instantiated with cloudooo parameters
+  """
+  __partition_reference__ = 'c'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+        '_':
+        json.dumps({
+            'cloudooo-url-list': [
+              'https://cloudooo1.example.com/',
+              'https://cloudooo2.example.com/',
+            ],
+            'cloudooo-retry-count': 123,
+        })
+    }
+
+  def test_cloudooo_url_list_preference(self):
+    self.assertEqual(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerUrlList'),
+          verify=False).text,
+      "['https://cloudooo1.example.com/', 'https://cloudooo2.example.com/']")
+
+  @unittest.expectedFailure # setting "retry" is not implemented
+  def test_cloudooo_retry_count_preference(self):
+    self.assertEqual(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerRetry'),
+          verify=False).text,
+      "123")
+
+
+class TestCloudoooDefaultParameter(ZopeSkinsMixin, ERP5InstanceTestCase):
+  """Test default ERP5 cloudooo parameters
+  """
+  __partition_reference__ = 'cd'
+
+  def test_cloudooo_url_list_preference(self):
+    self.assertIn(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerUrlList'),
+          verify=False).text,
+      [
+        "['https://cloudooo1.erp5.net/', 'https://cloudooo.erp5.net/']",
+        "['https://cloudooo.erp5.net/', 'https://cloudooo1.erp5.net/']",
+      ])
+
+  @unittest.expectedFailure # default value of "retry" does not match schema
+  def test_cloudooo_retry_count_preference(self):
+    self.assertEqual(
+      requests.get(
+          self._getAuthenticatedZopeUrl(
+            'portal_preferences/getPreferredDocumentConversionServerRetry'),
+          verify=False).text,
+      "2")

software/monitor/software.cfg

@@ -62,7 +62,6 @@ scripts =
 surykatka = 0.8.0
 
 # For surykatka 0.7.1
-click = 8.0.1
 dnspython = 2.1.0
 forcediphttpsadapter = 1.0.1
 miniupnpc = 2.0.2

software/powerdns/software.cfg

@@ -44,12 +44,3 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
 [template-zones-file]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
-
-[versions]
-PyRSS2Gen = 1.1
-cns.recipe.symlink = 0.2.3
-plone.recipe.command = 1.1
-passlib = 1.7.1
-GitPython = 2.1.11
-feedparser = 5.2.1
-atomize = 0.2.0

software/rapid-cdn/software.cfg

@@ -220,10 +220,7 @@ validators = 0.12.2
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 ecdsa = 0.13
-gitdb = 0.6.4
-plone.recipe.command = 1.1
 pycrypto = 2.6.1
-smmap = 0.9.0
 
 furl = 2.1.0
 orderedmultidict = 1.0.1

software/slapos-master/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 6d9b94b50ea46b0b98c9019d74780876
+md5sum = 2491969d49f8bf9b172e89b1c0e9d98e
 
 [template-balancer]
 filename = instance-balancer.cfg.in

software/slapos-master/instance-erp5.cfg.in

@@ -21,6 +21,10 @@
 {%   set test_runner_total_database_count = mariadb_test_database_amount %}
 {%   set test_runner_enabled = mariadb_test_database_amount > 0 %}
 {% endif -%}
+{# Backward compatibility for cloudooo-url #}
+{% if slapparameter_dict.get('cloudooo-url') -%}
+{%   do slapparameter_dict.setdefault('cloudooo-url-list', slapparameter_dict['cloudooo-url'].split(',')) %}
+{% endif -%}
 {% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
 {% set monitor_base_url_dict = {} -%}
 {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
@@ -231,7 +235,7 @@ return =
 {% endif -%}
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
-config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
+config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
 config-caucase-url = {{ dumps(caucase_url) }}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}

software/slaprunner/software.cfg

@@ -146,8 +146,6 @@ scripts =
 [versions]
 Flask-Auth = 0.85
 futures = 3.0.5
-gitdb = 0.6.4
 gunicorn = 19.10.0
 prettytable = 0.7.2
 pycurl = 7.43.0
-smmap = 0.9.0

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = bd79a9e6306b321414b9f83524308e5f
+md5sum = 937f8ebdfa8112aafe11235a23fb85a9
 
 [instance]
 _update_hash_filename_ = instance.cfg.in

software/theia/instance-theia.cfg.jinja.in

@@ -13,7 +13,9 @@ theia-environment-parts =
   settings.json
 
 theia-parts =
-  frontend-reload
+  frontend-instance
+  frontend-instance-rsyslogd
+  python-server
   promises
 
 parts =
@@ -90,11 +92,13 @@ recipe =
 instance-promises =
   $${theia-listen-promise:name}
   $${frontend-listen-promise:name}
+  $${python-server-listen-promise:name}
   $${frontend-authentication-promise:name}
   $${remote-frontend-url-available-promise:name}
   {% if additional_frontend %}
   $${remote-additional-frontend-url-available-promise:name}
   {% endif %}
+  $${frontend-instance-rsyslogd-promise:name}
   $${slapos-standalone-listen-promise:name}
   $${slapos-standalone-ready-promise:name}
   $${slapos-autorun-promise:name}
@@ -116,6 +120,13 @@ name = $${:_buildout_section_name_}.py
 config-host = $${frontend-instance:ip}
 config-port = $${frontend-instance:port}
 
+[python-server-listen-promise]
+<= monitor-promise-base
+promise = check_socket_listening
+name = $${:_buildout_section_name_}.py
+config-host = $${python-server-port:ip}
+config-port = $${python-server-port:port}
+
 [frontend-authentication-promise]
 <= monitor-promise-base
 promise = check_url_available
@@ -142,6 +153,12 @@ config-url = $${remote-additional-frontend:connection-secure_access}
 config-http-code = 401
 {% endif %}
 
+[frontend-instance-rsyslogd-promise]
+<= monitor-promise-base
+promise = check_command_execute
+name = rsyslogd_listen_promise.py
+config-command = test -S $${frontend-instance-rsyslogd-config:log-socket}
+
 [slapos-standalone-listen-promise]
 <= monitor-promise-base
 promise = check_socket_listening
@@ -207,7 +224,7 @@ sla-instance_guid = {{ parameter_dict['additional-frontend-guid'] }}
 {% endif %}
 
 
-# Local Caddy Frontend
+# Local Haproxy Frontend
 # --------------------
 
 [frontend-instance-password]
@@ -215,6 +232,39 @@ recipe = slapos.cookbook:generate.password
 username = admin
 storage-path = $${buildout:parts-directory}/.$${:_buildout_section_name_}
 
+[frontend-instance-rsyslogd-config]
+recipe = slapos.recipe.template
+output = $${directory:etc}/$${:_buildout_section_name_}
+log-file = $${directory:log}/frontend-instance.log
+log-socket = $${directory:run}/rsyslog.sock
+pidfile = $${directory:pidfiles}/rsyslogd.pid
+inline =
+  module(
+    load="imuxsock"
+    SysSock.Name="$${:log-socket}")
+
+  # Just simply output the raw line without any additional information, as
+  # haproxy emits enough information by itself
+  # Also cut out first empty space in msg, which is related to rsyslogd
+  # internal and end up cutting on 8k, as it's default of $MaxMessageSize
+  template(name="rawoutput" type="string" string="%msg:2:8192%\n")
+  $ActionFileDefaultTemplate rawoutput
+
+  $FileCreateMode 0600
+  $DirCreateMode 0700
+  $Umask 0022
+
+  $WorkDirectory $${directory:run}
+
+  *.* $${:log-file};rawoutput
+
+
+[frontend-instance-rsyslogd]
+recipe = slapos.cookbook:wrapper
+command-line = ${rsyslogd:location}/sbin/rsyslogd -i $${frontend-instance-rsyslogd-config:pidfile} -n -f $${frontend-instance-rsyslogd-config:output}
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+hash-files = $${frontend-instance-rsyslogd-config:output}
+
 [frontend-instance-port]
 recipe = slapos.cookbook:free_port
 minimum = 3000
@@ -224,61 +274,83 @@ ip = {{ ipv6_random }}
 [frontend-instance-certificate]
 recipe = plone.recipe.command
 command =
-  if [ ! -e $${:key-file} ]
+  if [ ! -e $${:cert-file} ]
   then
     ${openssl-output:openssl} req -x509 -nodes -days 3650 \
       -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
-      -newkey rsa:1024 -keyout $${:key-file} \
+      -newkey rsa:1024 -keyout $${:cert-file} \
       -out $${:cert-file}
   fi
 update-command = $${:command}
-key-file = $${directory:etc}/$${:_buildout_section_name_}.key
-cert-file = $${directory:etc}/$${:_buildout_section_name_}.crt
+cert-file = $${directory:etc}/$${:_buildout_section_name_}.pem
 common-name = $${frontend-instance-config:ip}
 location =
-  $${:key-file}
   $${:cert-file}
 
 [frontend-instance-config]
 recipe = slapos.recipe.template
 output = $${directory:etc}/$${:_buildout_section_name_}
+blankline =
 inline =
-  :$${:port} {
-    bind $${:ip}
-    tls $${frontend-instance-certificate:cert-file} $${frontend-instance-certificate:key-file}
-    log stdout
-    errors stderr
-    gzip
-    # because caddy does not support upgrade http2 to websocket
-    # https://tools.ietf.org/html/rfc8441
-    tls {
-      alpn http/1.1
-    }
-    root $${directory:frontend-static}
-    browse
-    proxy / $${theia-instance:base-url} {
-      except $${frontend-instance-fonts:folder-name} $${frontend-instance-slapos.css:folder-name} public $${favicon.ico:filename} $${frontend-instance-logo:filename}
-    }
-    proxy /services $${theia-instance:base-url} {
-      websocket
-    }
-    proxy /socket.io $${theia-instance:base-url} {
-      websocket
-    }
-    basicauth $${frontend-instance-password:username} $${frontend-instance-password:passwd} {
-      realm "Theia"
-      /
-    }
-  }
+  global
+    maxconn 4096
+    master-worker
+    pidfile $${frontend-instance:pidfile}
+    log $${frontend-instance-rsyslogd-config:log-socket} local0  info
+
+  defaults
+    log global
+    option httplog
+    mode http
+    retries 1
+    option redispatch
+    maxconn 2000
+    balance roundrobin
+    timeout connect 10s
+    timeout queue 60s
+    timeout server 305s
+    timeout client 305s
+
+    # compress some content types
+    compression algo gzip
+    compression type application/font-woff application/font-woff2 application/hal+json application/javascript application/json application/rss+xml application/wasm application/x-font-opentype application/x-font-ttf application/x-javascript application/xml image/svg+xml text/cache-manifest text/css text/html text/javascript text/plain text/xml
+
+
+  userlist basic-auth-list
+    user $${frontend-instance-password:username} insecure-password $${frontend-instance-password:passwd}
+
+  frontend app
+    log global
+    bind $${:ip}:$${:port} ssl crt $${frontend-instance-certificate:cert-file} alpn h2,http/1.1
+    # writing twice the same ACL is doing OR
+    acl is_public path_beg /public/
+    acl is_public path /$${favicon.ico:filename}
+    acl auth_ok http_auth(basic-auth-list)
+    # No authentication for public folder
+    http-request auth unless auth_ok || is_public
+    use_backend static if { path_beg /$${frontend-instance-fonts:folder-name} } || { path_beg /$${frontend-instance-slapos.css:folder-name} } || { path /$${frontend-instance-logo:filename} } || is_public
+    default_backend nodejs
+
+  backend nodejs
+    log global
+    server nodejs_backend $${theia-instance:ip}:$${theia-instance:port}
+
+  backend static
+    log global
+    server static_backend $${python-server-port:ip}:$${python-server-port:port}
+  $${:blankline}
+
 ip = $${frontend-instance-port:ip}
 hostname = [$${:ip}]
 port = $${frontend-instance-port:port}
+pidfile = $${directory:pidfiles}/haproxy.pid
 
 [frontend-instance]
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
 command-line =
-  ${caddy:output} -conf $${frontend-instance-config:output} -pidfile $${:pidfile}
+  ${haproxy:location}/sbin/haproxy -f $${frontend-instance-config:output}
+hash-files = $${frontend-instance-config:output}
 
 ip = $${frontend-instance-config:ip}
 hostname = $${frontend-instance-config:hostname}
@@ -287,7 +359,7 @@ pidfile = $${directory:pidfiles}/$${:_buildout_section_name_}.pid
 url = https://$${:hostname}:$${:port}/
 
 [frontend-instance-fonts]
-; XXX caddy 1 does not seem to serve different folders at different locations
+; XXX python server only serves one folder
 ; so we link fonts in static folder
 recipe = plone.recipe.command
 location = $${directory:frontend-static}/$${:folder-name}
@@ -314,18 +386,6 @@ folder-name = css
 context =
   key logo_image frontend-instance-logo:filename
 
-[frontend-reload]
-recipe = slapos.cookbook:wrapper
-wrapper-path = $${directory:services}/$${:_buildout_section_name_}
-command-line =
-  ${bash:location}/bin/bash -c
-  "kill -s USR1 $$(${coreutils:location}/bin/cat $${frontend-instance:pidfile}) \
-    && ${coreutils:location}/bin/sleep infinity"
-hash-files =
-  $${frontend-instance-config:output}
-  $${frontend-instance:wrapper-path}
-wait-for-files = $${frontend-instance:pidfile}
-
 [favicon.ico]
 # generate a pseudo random favicon, different for each instance name.
 recipe = slapos.recipe.build
@@ -350,6 +410,20 @@ install =
 location = $${directory:frontend-static}/$${:filename}
 filename = $${:_buildout_section_name_}
 
+# Local Python Server
+# -------------------
+
+[python-server-port]
+recipe = slapos.cookbook:free_port
+minimum = 3000
+maximum = 3100
+ip = {{ ipv4_random }}
+
+[python-server]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line = $${buildout:executable} -m http.server $${python-server-port:port} --bind $${python-server-port:ip} --directory $${directory:frontend-static}
+
 
 # Common Environment
 # ------------------
@@ -406,7 +480,6 @@ hash-existing-files =
 ip =  {{ ipv4_random }}
 hostname = $${:ip}
 port = $${theia-service:port}
-base-url = $${theia-service:base-url}
 
 [theia-shell]
 recipe = slapos.recipe.template:jinja2

software/theia/software.cfg

 [buildout]
 extends =
-  ../../component/caddy/buildout.cfg
+  ../../component/haproxy/buildout.cfg
+  ../../component/rsyslogd/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/bash/buildout.cfg
   ../../component/bash-completion/buildout.cfg

software/theia/test/test.py

@@ -146,17 +146,17 @@ class TestTheia(TheiaTestCase):
         )).geturl()
     self.get(authenticated_url)
 
-    # there's a public folder to serve file
+    # there's a public folder to serve file (no need for authentication)
     with open('{}/srv/frontend-static/public/test_file'.format(
         self.getPath()), 'w') as f:
       f.write("hello")
-    resp = self.get(urljoin(authenticated_url, '/public/'))
+    resp = self.get(urljoin(url, '/public/'))
     self.assertIn('test_file', resp.text)
-    resp = self.get(urljoin(authenticated_url, '/public/test_file'))
+    resp = self.get(urljoin(url, '/public/test_file'))
     self.assertEqual('hello', resp.text)
 
-    # there's a (not empty) favicon
-    resp = self.get(urljoin(authenticated_url, '/favicon.ico'))
+    # there's a (not empty) favicon (no need for authentication)
+    resp = self.get(urljoin(url, '/favicon.ico'))
     self.assertTrue(resp.raw)
 
     # there is a CSS referencing fonts

stack/erp5/buildout.cfg

@@ -176,7 +176,7 @@ context =
     key dash_location dash:location
     key bash_location bash:location
     key dcron_location dcron:location
-    key default_cloudooo_url erp5-defaults:cloudooo-connection-url
+    key default_cloudooo_url_list erp5-defaults:cloudooo-connection-url-list
     key erp5_location erp5:location
     key findutils_location findutils:location
     key gzip_location gzip:location
@@ -333,7 +333,11 @@ repository_id_list = erp5 erp5-bin erp5-doc
 
 # ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin)
 [erp5-defaults]
-cloudooo-connection-url = https://cloudooo.erp5.net/
+# Default cloudooo is https://cloudooo.erp5.net/ and  https://cloudooo1.erp5.net/ in random order.
+# The random order is applied in instance profile
+cloudooo-connection-url-list =
+  https://cloudooo.erp5.net/
+  https://cloudooo1.erp5.net/
 # Jupyter is by default disabled in ERP5
 jupyter-enable-default = false
 # WCFS is by default disabled in ERP5

stack/erp5/buildout.hash.cfg

@@ -70,11 +70,11 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
 
 [template]
 filename = instance.cfg.in
-md5sum = 74979d6743cdc7ce3d2f17680115458d
+md5sum = 3f7b28085ceff321a3cb785db60f7c3e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 52dfc0c9a4547b1e928707ef5caeecdb
+md5sum = 30a1e738a8211887e75a5e75820e5872
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 1d0cfbefbfee93e28c07f7813b94723c
+md5sum = 90c67d5a86a42cb0dd00e45fd099b1c2
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -21,6 +21,10 @@
 {%   set test_runner_total_database_count = mariadb_test_database_amount %}
 {%   set test_runner_enabled = mariadb_test_database_amount > 0 %}
 {% endif -%}
+{# Backward compatibility for cloudooo-url #}
+{% if slapparameter_dict.get('cloudooo-url') -%}
+{%   do slapparameter_dict.setdefault('cloudooo-url-list', slapparameter_dict['cloudooo-url'].split(',')) %}
+{% endif -%}
 {% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
 {% set monitor_base_url_dict = {} -%}
 {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
@@ -234,7 +238,7 @@ return =
 {% endif -%}
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
-config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
+config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
 config-caucase-url = {{ dumps(caucase_url) }}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}

stack/erp5/instance-zope.cfg.in

@@ -33,7 +33,6 @@ partition. No more (undefined result), no less (IndexError).
 {% for alias, url in (
     ('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
     ('erp5-memcached-persistent', slapparameter_dict['kumofs-url']),
-    ('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
     ('erp5-smtp', slapparameter_dict['smtp-url']),
   ) -%}
 {%   set parsed_url = urllib_parse.urlparse(url) -%}
@@ -269,12 +268,7 @@ bt5 = {{ slapparameter_dict['bt5'] }}
 bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }}
 id-store-interval = {{ dumps(slapparameter_dict['id-store-interval']) }}
 home = ${buildout:directory}
-# We only want to change the hostname to 'erp5-cloudooo' if we use the internal
-# cloudooo. We plan to remove the ability to have an internal one, so this
-# heuristic is enough.
-{%  set cloudooo = urllib_parse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
-cloudooo-url = {{ (cloudooo if cloudooo.port == None else
-  cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
+cloudooo-url = {{ ",".join(slapparameter_dict['cloudooo-url-list']) }}
 
 {% endif -%}
 developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
@@ -514,11 +508,8 @@ wrapper-path = ${directory:bin}/${:command-name}.real
 command-line =
   '{{ parameter_dict['bin-directory'] }}/${:command-name}'
   ${:command-line-extra}
-  --conversion_server_url={{ slapparameter_dict['cloudooo-url'] }}
+  --conversion_server_url={{ ",".join(slapparameter_dict['cloudooo-url-list']) }}
   --conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
-{#- BBB: We still have test suites that only accept the following 2 options. #}
-  --conversion_server_hostname=erp5-cloudooo
-  --conversion_server_port={{ port_dict['erp5-cloudooo'] }}
   --volatile_memcached_server_hostname=erp5-memcached-volatile
   --volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
   --persistent_memcached_server_hostname=erp5-memcached-persistent

stack/erp5/instance.cfg.in

@@ -26,8 +26,21 @@ extra-context =
     section parameter_dict dynamic-template-postfix-parameters
     import urllib urllib
 
+[default-cloudooo-url-list]
+recipe = slapos.recipe.build
+default-url-list =
+  {{ default_cloudooo_url_list | indent }}
+init =
+  # Expose the default cloudooo URLs, sorted using a pseudo-random seed,
+  # so that the same instance keep the same order
+  import six, random
+  seed = repr(sorted(six.iteritems(self.buildout['slap-connection'])))
+  default_cloudooo_url_list = [url.strip() for url in self.options['default-url-list'].splitlines()]
+  random.Random(seed).shuffle(default_cloudooo_url_list)
+  self.options['url-list'] = default_cloudooo_url_list
+
 [dynamic-template-erp5-parameters]
-default-cloudooo-url =  {{ dumps(default_cloudooo_url) }}
+default-cloudooo-url-list = ${default-cloudooo-url-list:url-list}
 jupyter-enable-default = {{ jupyter_enable_default }}
 wcfs-enable-default = {{ wcfs_enable_default }}
 local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
@@ -42,7 +55,7 @@ template-zodb-base = {{ template_zodb_base }}
 url = {{ template_erp5 }}
 filename = instance-erp5.cfg
 extra-context =
-    key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
+    key default_cloudooo_url_list dynamic-template-erp5-parameters:default-cloudooo-url-list
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository

stack/resilient/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = e24d0567cda7c72e39f6c8fcb0ede7ab
 
 [pbsready-import]
 filename = pbsready-import.cfg.in
-md5sum = 0aa7feb615fc5ac1a132a6d92c80f954
+md5sum = da1d3229fe6d3664720d75b2e9d0b8f4
 
 [pbsready-export]
 filename = pbsready-export.cfg.in
@@ -46,7 +46,7 @@ md5sum = 2c9828b57a65ee1eda62fcc4e9a92714
 
 [resilient-web-takeover-cgi-script-download]
 filename = resilient-web-takeover-cgi-script.py.in
-md5sum = 4137b7ac6b20a6cd897ad58f51a763b1
+md5sum = 8f5c9830f4865265d3477fe51425e71d
 
 [template-wrapper]
 filename = templates/wrapper.in

stack/resilient/pbsready-import.cfg.in

@@ -105,7 +105,6 @@ takeover-triggered-file-path = $${resiliency-takeover-script:takeover-triggered-
 [resilient-web-takeover-password]
 recipe = slapos.cookbook:generate.password
 storage-path = $${directory:srv}/passwd
-bytes = 8
 
 [resilient-web-takeover-cgi-script]
 recipe = collective.recipe.template

stack/resilient/resilient-web-takeover-cgi-script.py.in

@@ -7,8 +7,8 @@ takeover_script = '${resiliency-takeover-script:wrapper-takeover}'
 
 import atexit
 import cgi
-import cgitb
 import datetime
+import hmac
 try:
   import dbm.gnu as gdbm
 except ImportError:
@@ -25,8 +25,6 @@ if os.path.exists('resilient_software_release_information.py'):
 else:
   resilient_main = lambda: {}
 
-cgitb.enable()
-
 def deleteTemporaryDirectory(path):
   if os.path.exists(path):
     shutil.rmtree(path)
@@ -108,7 +106,7 @@ if "password" not in form:
 </html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted()))
   sys.exit(0)
 
-if form['password'].value != '${:password}':
+if not hmac.compare_digest(form['password'].value.encode(), b'${:password}'):
   print("<H1>Error</H1>")
   print("Password is invalid.")
   sys.exit(1)

stack/slapos.cfg

@@ -152,7 +152,7 @@ backports.shutil-get-terminal-size = 1.0.0
 bcrypt = 3.1.4
 bleach = 5.0.1
 CacheControl = 0.12.6:whl
-certifi = 2022.6.15
+certifi = 2022.12.7
 cffi = 1.15.0
 chardet = 3.0.4
 charset-normalizer = 2.1.1
@@ -184,8 +184,8 @@ funcsigs = 1.0.2
 functools32 = 3.2.3.post2
 gevent = 20.9.0
 geventmp = 0.0.1
-gitdb2 = 2.0.5
-GitPython = 2.1.11
+gitdb = 4.0.10
+GitPython = 3.1.30
 greenlet = 0.4.17
 h5py = 2.7.1
 h11 = 0.14.0
@@ -294,7 +294,7 @@ slapos.recipe.cmmi = 0.19
 slapos.recipe.template = 5.0
 slapos.toolbox = 0.128
 statsmodels = 0.11.1
-smmap2 = 2.0.5
+smmap = 5.0.0
 sniffio = 1.3.0
 sortedcontainers = 2.4.0
 stevedore = 1.21.0:whl
@@ -314,7 +314,7 @@ urllib3 = 1.26.12
 wcwidth = 0.2.5
 webencodings = 0.5.1
 Werkzeug = 2.0.2
-wheel = 0.35.1:whl
+wheel = 0.38.4:whl
 widgetsnbextension = 2.0.0
 wsproto = 1.2.0
 xml-marshaller = 1.0.2
@@ -335,10 +335,16 @@ charset-normalizer = 2.0.12
 click = 6.7
 distro = 1.6.0
 feedparser = 5.2.1
+gitdb2 = 2.0.5
+GitPython = 2.1.11
 pyparsing = 2.2.0
 pyrsistent = 0.16.1
 requests = 2.27.1
+smmap = 0.9.0
+smmap2 = 2.0.5
+traitlets = 4.3.3
 Werkzeug = 1.0.1
+wheel = 0.35.1:whl
 
 [versions:sys.version_info < (3,8)]
 MarkupSafe = 1.0