Update Release Candidate

component/cloudooo/buildout.cfg

@@ -14,7 +14,7 @@ recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/cloudooo.git
 branch = master
 git-executable = ${git:location}/bin/git
-revision = 0b5ff71a2ede76499e81659aed392057ae910917
+revision = 78d15e091df750952998a6a23f49300a5a0faa53
 
 [cloudooo]
 recipe = zc.recipe.egg
@@ -27,7 +27,7 @@ entry-points =
   cloudooo_tester=cloudooo.bin.cloudooo_tester:main
   runCloudoooUnitTest=cloudooo.tests.runHandlerUnitTest:run
   runCloudoooTestSuite=cloudooo.tests.runTestSuite:run
-scripts = 
+scripts =
   paster=cloudooo_paster
   runCloudoooUnitTest
   runCloudoooTestSuite

component/golang/buildout.cfg

@@ -19,7 +19,11 @@ shared = true
 configure-command = :
 location = @@LOCATION@@
 make-binary =
-make-targets= cd src && unset GOBIN && ./all.bash && cp -alf .. ${:location}
+# build and test Go and its standard library
+# clean intermediate cache before installing
+# contrary to build cache in gowork, build cache, that Go uses during its own build, is not reused anywhere:
+# https://github.com/golang/go/blob/go1.18.2-0-g8ed0e51b5e/src/cmd/dist/build.go#L239-L242
+make-targets= cd src && unset GOBIN && ./all.bash && GOCACHE=`pwd`/../pkg/obj/go-build ../bin/go clean -cache && cp -alf .. ${:location}
 # some testdata files have an issue with slapos.extension.strip.
 post-install = ${findutils:location}/bin/find ${:location}/src -type d -name testdata -exec rm -rf {} \; || true
 environment =
@@ -42,6 +46,8 @@ patches =
 url = https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz
 md5sum = dbf727a4b0e365bf88d97cbfde590016
 environment-extra =
+# build Go without testing it
+# NOTE go1.4 does not have build cache
 make-targets= cd src && unset GOBIN && ./make.bash && cp -alf .. ${:location}
 
 # skip-chown-tests.patch does not apply to go1.4, but we don't run go1.4 tests.

component/libpcap/buildout.cfg

+[buildout]
+extends =
+  ../bison/buildout.cfg
+  ../flex/buildout.cfg
+
+[libpcap]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz
+md5sum = 28e17495004036567c2cc884b51eba45
+environment =
+  PATH=${bison:location}/bin:${flex:location}/bin:%(PATH)s

component/python-pim-dm/buildout.cfg

+[buildout]
+extends =
+  ../libpcap/buildout.cfg
+  ../python-PyYAML/buildout.cfg
+
+parts =
+  eggs
+
+[python-pim-dm]
+recipe = zc.recipe.egg:custom
+egg = pim-dm
+include-dirs =
+  ${libpcap:location}/include
+library-dirs =
+  ${libpcap:location}/lib
+rpath =
+  ${libpcap:location}/lib/
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  igmp
+  ipaddress
+  netifaces
+  ${python-pim-dm:egg}
+  prettytable
+  py-mld
+  pyroute2
+  ${python-PyYAML:egg}

component/re6stnet/buildout.cfg

@@ -5,6 +5,8 @@ extends =
   ../geoip2/buildout.cfg
   ../openvpn/buildout.cfg
   ../python-cryptography/buildout.cfg
+  ../python-PyYAML/buildout.cfg
+  ../python-pim-dm/buildout.cfg
 
 parts =
   re6stnet
@@ -18,6 +20,7 @@ eggs =
   ${maxminddb-python:egg}
   ${python-cffi:egg}
   ${python-cryptography:egg}
+  ${python-PyYAML:egg}
   pyOpenSSL
   re6stnet[geoip]
 
@@ -26,4 +29,54 @@ eggs =
 initialization =
   import os
   os.environ['GEOIP2_MMDB'] = "${geolite2-country:location}/GeoLite2-Country.mmdb"
-  os.environ['PATH'] = "${openvpn:location}/sbin:${babeld:location}/bin:${openssl:location}/bin:${python:location}/bin:" + os.environ.get('PATH', '/usr/sbin:/usr/bin:/sbin:/bin')
+  os.environ['PATH'] = "${openvpn:location}/sbin:${babeld:location}/bin:${openssl:location}/bin:${python:location}/bin:${pim-dm-buildout:location}/bin:" + os.environ.get('PATH', '/usr/sbin:/usr/bin:/sbin:/bin')
+
+[pim-dm-buildout]
+recipe = slapos.recipe.build
+base = ${:_profile_base_location_}
+python = ${python3:executable}
+install =
+  import os
+  from zc.buildout.buildout import _isurl, urljoin
+  from slapos.rebootstrap.bootstrap import setup_script
+  base = options['base']
+  if _isurl(base):
+    extends = lambda path: urljoin(base + '/', path)
+  else:
+    extends = lambda path: os.path.realpath(os.path.join(base, path))
+
+  buildout_extra = []
+  buildout_section = self.buildout['buildout']
+  for option_name in ('download-cache', 'newest', 'shared-part-list'):
+    option_value = buildout_section.get(option_name)
+    if option_value:
+      buildout_extra.append('%s = %s' % (option_name, option_value))
+  os.mkdir(location)
+  with open(os.path.join(location, 'buildout.cfg'), 'w') as buildout_conf:
+    buildout_conf.write(
+      options['template'] % {
+        'extends': '\n  '.join(map(extends, (
+          '../../stack/slapos.cfg',
+          '../python-pim-dm/buildout.cfg',
+          ))),
+        'buildout': '\n'.join(buildout_extra),
+      }
+    )
+  buildout = os.path.join(location, 'bin', 'buildout')
+  os.mkdir(os.path.dirname(buildout))
+  setup_script(buildout, options['python'])
+  call((buildout, '-U'), cwd=location)
+template =
+  [buildout]
+  extends = %(extends)s
+  extends-cache = ${buildout:extends-cache}
+  extensions -= slapos.rebootstrap
+  %(buildout)s
+  [libpcap]
+  recipe =
+  location = ${libpcap:location}
+  environment =
+  [libyaml]
+  recipe =
+  location = ${libyaml:location}
+  pre-configure =

component/replication-manager/buildout.cfg

@@ -30,4 +30,4 @@ go.importpath = github.com/signal18/replication-manager
 repository =  https://github.com/signal18/replication-manager
 #branch = 2.1
 branch = develop
-revision = 838ffeb205ea4477f1c1bda607940fa07d8bcf51
\ No newline at end of file
+revision = 774fde2ce67942a31b99200fea6301189344ecd9
\ No newline at end of file

software/beremiz-ide/software.cfg

@@ -86,9 +86,9 @@ eggs +=
 
 [beremiz]
 recipe = slapos.recipe.build:download-unpacked
-# download beremiz at revision bc72db1aa49fb5deabc950ed587772d027733a26
-url = https://github.com/beremiz/beremiz/archive/bc72db1aa49fb5deabc950ed587772d027733a26.tar.gz
-md5sum = 9ae7a55d4f2e673cbc8d2c4528e069b9
+# download beremiz at revision 86b02aa32d413437ddcb9ab6cf4cc72aad394b3e
+url = https://github.com/beremiz/beremiz/archive/86b02aa32d413437ddcb9ab6cf4cc72aad394b3e.tar.gz
+md5sum = 0b46be8c8e849bd612373dc999427912
 
 [beremiz-setup]
 recipe  = zc.recipe.egg:develop

software/erp5testnode/testsuite/deploy-test/instance-deploy-test.cfg.jinja2

@@ -55,8 +55,8 @@ return =
   url
   backend-url
   ipv6
-  nat-rule-port-443
-  nat-rule-port-80
+  nat-rule-port-tcp-443
+  nat-rule-port-tcp-80
 
 [directory]
 recipe = slapos.cookbook:mkdirectory

software/erp5testnode/testsuite/deploy-test/software.cfg

@@ -70,3 +70,6 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/template.cfg
 mode = 0644
+
+[versions]
+erp5.util = 0.4.69

software/erp5testnode/testsuite/dummy/software.cfg

@@ -11,4 +11,4 @@ parts =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = 9fccb9600a085691ab0f9a20cd615c01
+md5sum = 0385b31519c3489fb8d0919621e02ac7

software/gitlab/buildout.hash.cfg

@@ -54,11 +54,11 @@ md5sum = 0f1ec4077dab586cc003ae13f689eda2
 
 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = c115c88d5798dcb5e16b9d50432f2551
+md5sum = 0b023c7efd027f65b14e752484be2ec7
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in
-md5sum = da2e3d6f67c92233d370dae55fb3b35f
+md5sum = 9ed8220bb3ad71ff7e8638354127412c
 
 [instance-gitlab-test.cfg.in]
 _update_hash_filename_ = instance-gitlab-test.cfg.in

software/gitlab/instance-gitlab-export.cfg.in

@@ -74,7 +74,7 @@ context =
   raw     unicorn_script          ${service-unicorn:wrapper-path}
   raw     sidekiq_script          ${service-sidekiq:wrapper-path}
   raw     gitlab_backup_dir       ${gitlab-backup-directory:backup-gitlab.git}
-  raw     redis_pid_file          ${service-redis:pid_file}
+  raw     redis_pid_file          ${service-redis:pid-file}
   raw     postgres_pid_file       ${service-postgresql:pgdata-directory}/postmaster.pid
   raw     gitlab_work_location    ${gitlab-work:location}
   raw     promise_lab_location    ${directory:promise.slow}

software/gitlab/instance-gitlab.cfg.in

@@ -497,19 +497,20 @@ log     = ${directory:log}/redis
 [service-redis]
 recipe  = slapos.cookbook:redis.server
 wrapper = ${directory:service}/redis
-promise_wrapper = ${directory:bin}/redis-promise
-
-server_dir  = ${redis:srv}
-config_file = ${directory:etc}/redis.conf
-log_file    = ${redis:log}/redis.log
-pid_file    = ${directory:run}/redis.pid
-use_passwd  = false
-unixsocket  = ${:server_dir}/redis.socket
+promise-wrapper = ${directory:bin}/redis-promise
+
+server-dir  = ${redis:srv}
+config-file = ${directory:etc}/redis.conf
+log-file    = ${redis:log}/redis.log
+pid-file    = ${directory:run}/redis.pid
+use-passwd  = false
+unixsocket  = ${:server-dir}/redis.socket
 # port = 0 means "don't listen on TCP at all" - listen only on unix socket
 ipv6    = ::1
 port    = 0
 
-server_bin  = {{ redis_binprefix }}/redis-server
+server-bin  = {{ redis_binprefix }}/redis-server
+cli-bin  = {{ redis_binprefix }}/redis-cli
 depend  =
     ${logrotate-entry-redis:recipe}
 
@@ -517,9 +518,7 @@ depend  =
 <= monitor-promise-base
 promise = check_command_execute
 name = promise-redis.py
-config-command = ${service-redis:promise_wrapper}
-
-# NOTE slapos.cookbook:redis.server setups promise automatically
+config-command = ${service-redis:promise-wrapper}
 
 [logrotate-entry-redis]
 <= logrotate-entry-base

software/matomo/software.cfg

@@ -17,10 +17,6 @@ parts =
 # to create file instance.cfg of all instances
   instance
 
-#to make sure we use python3
-[python]
-part = python3
-
 # download bas
 # inherited by modules that need to download files
 [matomo-download]

software/nextcloud/buildout.hash.cfg

@@ -26,4 +26,4 @@ md5sum = 6f42f0a8c5e5c0c657541a65c4d9ee57
 
 [template-nextcloud-instance]
 filename = nextcloud-instance.cfg.in
-md5sum = 495b1693e451f6c5044c67ce5f1194fe
+md5sum = 4426b5e62c61efec2e68e7f548bc6463

software/nextcloud/nextcloud-input-schema.json

@@ -129,6 +129,11 @@
       "title": "Third authorized domain on nextcloud",
       "description": "Trusted domain used to connect to Nextcloud instance.",
       "type": "string"
+    },
+    "instance.trusted-proxy-list": {
+      "title": "Trusted proxies",
+      "description": "IP adresses of trusted proxies, space separated.",
+      "type": "string"
     }
   }
 }

software/nextcloud/nextcloud-instance.cfg.in

@@ -12,17 +12,18 @@ backup   = ${directory:backup}/nextcloud
 [service-redis]
 recipe  = slapos.cookbook:redis.server
 wrapper = ${directory:services}/redis
-promise_wrapper = ${directory:bin}/redis-promise
-server_dir  = ${nc-directory:redis}
-config_file = ${directory:etc}/redis.conf
-log_file    = ${nc-directory:redis-log}/redis.log
-pid_file    = ${directory:run}/redis.pid
-use_passwd  = false
-unixsocket  = ${:server_dir}/redis.socket
+promise-wrapper = ${directory:bin}/redis-promise
+server-dir  = ${nc-directory:redis}
+config-file = ${directory:etc}/redis.conf
+log-file    = ${nc-directory:redis-log}/redis.log
+pid-file    = ${directory:run}/redis.pid
+use-passwd  = false
+unixsocket  = ${:server-dir}/redis.socket
 # port = 0 means "don't listen on TCP at all" - listen only on unix socket
 ipv6    = ::1
 port    = 0
-server_bin  = {{ redis_bin }}
+server-bin  = {{ redis_bin }}
+cli-bin  = {{ redis_cli }}
 depend  =
     ${logrotate-entry-redis:recipe}
 
@@ -30,7 +31,7 @@ depend  =
 <= monitor-promise-base
 promise = check_command_execute
 name = redis.py
-config-command = ${service-redis:promise_wrapper}
+config-command = ${service-redis:promise-wrapper}
 
 [logrotate-entry-redis]
 <= logrotate-entry-base

software/nextcloud/test/test.py

@@ -272,7 +272,7 @@ class TestServices(NextCloudTestCase):
     news_config_file = os.path.join(self.partition_dir, 'srv/data/news/config/config.ini')
     with open(news_config_file) as f:
       config = f.read()
-    self.assertRegexpMatches(config, r"(useCronUpdates\s+=\s+false)")
+    self.assertRegex(config, r"(useCronUpdates\s+=\s+false)")
 
 
 class TestNextCloudParameters(NextCloudTestCase):

software/slapos-sr-testing/software-py3.cfg

@@ -21,6 +21,7 @@ extra =
   jupyter ${slapos.test.jupyter-setup:setup}
   matomo ${slapos.test.matomo-setup:setup}
   monitor ${slapos.test.monitor-setup:setup}
+  nextcloud ${slapos.test.nextcloud-setup:setup}
   nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
   plantuml ${slapos.test.plantuml-setup:setup}
   powerdns ${slapos.test.powerdns-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -371,7 +371,6 @@ extra =
   htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
   jscrawler ${slapos.test.jscrawler-setup:setup}
   jstestnode ${slapos.test.jstestnode-setup:setup}
-  nextcloud ${slapos.test.nextcloud-setup:setup}
   re6stnet ${slapos.test.re6stnet-setup:setup}
   seleniumserver ${slapos.test.seleniumserver-setup:setup}
   slapos-master ${slapos.test.slapos-master-setup:setup}

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 850dedff7af48a70208361cabf688724
+md5sum = 3112bf86fa543df52548a551f5413f99

software/slapos-testing/instance.cfg

@@ -47,6 +47,10 @@ output = $${caucase:location}/host_setting.sh
 <= download-source
 repository = ${slapos.libnetworkcache-repository:location}
 
+[slapcache]
+<= download-source
+repository = ${slapcache-repository:location}
+
 [erp5.util]
 <= download-source
 repository = ${erp5.util-repository:location}
@@ -177,6 +181,12 @@ inline =
       cwd="""$${slapos.libnetworkcache:location}""",
       summaryf=UnitTest.summary,
   )
+  TestCase(
+      "slapcache",
+      ['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapcache/test'],
+      cwd="""$${slapcache:location}""",
+      summaryf=UnitTest.summary,
+  )
   TestCase(
       "slapos.rebootstrap",
       ['python', '-m', 'unittest', '-v', 'slapos.rebootstrap.tests.test_suite'],

software/slapos-testing/software.cfg

@@ -55,6 +55,11 @@ depends = ${caucase-setup:egg}
 egg = slapos.libnetworkcache
 setup = ${slapos.libnetworkcache-repository:location}
 
+[slapcache-setup]
+<= setup-develop-egg
+egg = slapcache
+setup = ${slapcache-repository:location}
+
 [erp5.util-setup]
 <= setup-develop-egg
 # XXX erp5.util does not have `test` extra require, but has a `testnode` extra require with same dependencies
@@ -132,6 +137,7 @@ eggs +=
   ${slapos.recipe.template-setup:egg}
   ${slapos.toolbox-setup:egg}
   ${slapos.libnetworkcache-setup:egg}
+  ${slapcache-setup:egg}
   ${slapos.rebootstrap-setup:egg}
   ${rubygemsrecipe-setup:egg}
   zope.testing
@@ -178,6 +184,10 @@ repository = https://lab.nexedi.com/nexedi/slapos.recipe.template.git
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.libnetworkcache.git
 
+[slapcache-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapcache.git
+
 [slapos.recipe.build-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.recipe.build.git
@@ -249,3 +259,4 @@ zope.testing = 4.6.2
 urllib3 = 1.24.1
 pathlib = 1.0.1
 psycopg2 = 2.8.6
+iniparse = 0.5

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 7d146ca31d7e44e909386e21e0a562ec
+md5sum = bdf824a8e09899d6da27b26706d177c5
 
 [instance]
 _update_hash_filename_ = instance.cfg.in

software/theia/instance-theia.cfg.jinja.in

@@ -377,12 +377,10 @@ recipe = slapos.recipe.template
 output = $${directory:bin}/$${:_buildout_section_name_}
 inline =
   #!/bin/sh
-  {{ "{% raw %}" }}
-  {% raw %}
+  {% raw -%}
   export THEIA_WEBVIEW_EXTERNAL_ENDPOINT='{{hostname}}'
   export THEIA_MINI_BROWSER_HOST_PATTERN='{{hostname}}'
-  {% endraw %}
-  {{ "{% endraw %}" }}
+  {% endraw -%}
   export THEIA_OPEN_EDITOR_TOKEN=$(${openssl:location}/bin/openssl rand -hex 32)
   export THEIA_URL=$${:base-url}
   export THEIA_SHELL=$${theia-shell:output}

stack/lamp/buildout.cfg

@@ -38,6 +38,9 @@ url = ${:_profile_base_location_}/${:filename}
 [application]
 recipe = slapos.recipe.build:download-unpacked
 
+[python]
+part = python3
+
 #----------------
 #--
 #-- Optional part allowing applications using this stack to run a custom

stack/slapos.cfg

@@ -148,7 +148,7 @@ atomicwrites = 1.4.0
 backports.functools-lru-cache = 1.6.1:whl
 backports.lzma = 0.0.14
 cffi = 1.14.0
-click = 6.7
+click = 8.1.3
 cliff = 2.8.3:whl
 cmd2 = 0.7.0
 collective.recipe.shelloutput = 0.1
@@ -163,6 +163,7 @@ gevent = 20.9.0
 geventmp = 0.0.1
 greenlet = 0.4.17
 idna = 2.9
+igmp = 1.0.4
 importlib-metadata = 1.7.0:whl
 inotify-simple = 1.1.1
 itsdangerous = 0.24
@@ -174,6 +175,7 @@ more-itertools = 5.0.0
 netaddr = 0.7.19
 pathlib2 = 2.3.5
 pbr = 2.0.0
+pim-dm = 1.4.0nxd001
 plone.recipe.command = 1.1
 prettytable = 0.7.2
 psutil = 5.8.0
@@ -181,15 +183,18 @@ pluggy = 0.13.1:whl
 py = 1.9.0:whl
 pyOpenSSL = 19.1.0
 pyparsing = 2.2.0
+py-mld = 1.0.3
+pyroute2 = 0.6.9
 pytz = 2016.10
 regex = 2020.9.27
-requests = 2.24.0
+requests = 2.27.1
+charset-normalizer = 2.0.12
 scandir = 1.10.0
 setproctitle = 1.1.10
 setuptools-dso = 1.7
 rubygemsrecipe  = 0.4.3
 six = 1.12.0
-slapos.cookbook = 1.0.237
+slapos.cookbook = 1.0.244
 slapos.core = 1.7.6
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
@@ -223,7 +228,7 @@ atomize = 0.2.0
 croniter = 0.3.25
 dnspython = 1.16.0
 enum34 = 1.1.10
-erp5.util = 0.4.73
+erp5.util = 0.4.74
 feedparser = 5.2.1
 functools32 = 3.2.3.post2
 attrs = 18.2.0
@@ -247,14 +252,15 @@ uritemplate = 3.0.0
 zope.interface = 4.3.3
 certifi = 2020.6.20
 chardet = 3.0.4
-urllib3 = 1.25.9
+urllib3 = 1.26.9
 pkgconfig = 1.5.1
 distro = 1.7.0
 
 
 [versions:python2]
-Werkzeug = 1.0.1
+click = 6.7
 distro = 1.6.0
+Werkzeug = 1.0.1
 
 
 [networkcache]