Update release candidate

component/qt/buildout.cfg

@@ -43,7 +43,7 @@ post-install =
 # qmake binary can be reached directly from ${qt:location}/bin/qmake if [qt] is fully built
 recipe = slapos.recipe.cmmi
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-url = http://download.qt.io/official_releases/qt/4.8/4.8.7/qt-everywhere-opensource-src-4.8.7.tar.gz
+url = http://download.qt.io/archive/qt/4.8/4.8.7/qt-everywhere-opensource-src-4.8.7.tar.gz
 md5sum = d990ee66bf7ab0c785589776f35ba6ad
 # see https://github.com/NixOS/nixpkgs/blob/3e387c3e005c87566b5403d24c86f71f4945a79b/pkgs/development/libraries/qt-4.x/4.8/default.nix#L101
 pre-configure =

component/tomcat/buildout.cfg

@@ -20,8 +20,8 @@ md5sum = 3dde098fd0b3a08d3f2867e4a95591ba
 recipe = hexagonit.recipe.download
 ignore-existing = true
 strip-top-level-dir = true
-url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.99/bin/apache-tomcat-7.0.99.tar.gz
-md5sum = ab39c15461f2a99493528b4a5819bc56
+url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.100/bin/apache-tomcat-7.0.100.tar.gz
+md5sum = 79be4ba5a6e770730a4be3d5cb3c7862
 
 [tomcat9]
 recipe = hexagonit.recipe.download

slapos/recipe/simplehttpserver/__init__.py

@@ -27,6 +27,7 @@
 from slapos.recipe.librecipe import GenericBaseRecipe
 import string, random
 import os
+from six.moves import range
 
 class Recipe(GenericBaseRecipe):
 
@@ -35,7 +36,7 @@ class Recipe(GenericBaseRecipe):
     base_path = options['base-path']
     if options.get('use-hash-url', 'True') in ['true', 'True']:
       pool = string.letters + string.digits
-      hash_string = ''.join(random.choice(pool) for i in xrange(64))
+      hash_string = ''.join(random.choice(pool) for i in range(64))
       path = os.path.join(base_path, hash_string)
   
       if os.path.exists(base_path):

slapos/recipe/simplehttpserver/simplehttpserver.py

 # -*- coding: utf-8 -*-
-from SimpleHTTPServer import SimpleHTTPRequestHandler
-from BaseHTTPServer import HTTPServer
+from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
+from six.moves.BaseHTTPServer import HTTPServer
 import ssl
 import os
 import logging
@@ -8,6 +8,9 @@ from netaddr import valid_ipv4, valid_ipv6
 import socket
 import cgi, errno
 
+from slapos.util import str2bytes
+
+
 class ServerHandler(SimpleHTTPRequestHandler):
 
   document_path = ''
@@ -22,7 +25,7 @@ class ServerHandler(SimpleHTTPRequestHandler):
     if self.restrict_root_folder and self.path and self.path == '/':
       # no access to root path
       self.respond(403)
-      self.wfile.write("Forbidden")
+      self.wfile.write(b"Forbidden")
       return True
     return False
 
@@ -46,11 +49,11 @@ class ServerHandler(SimpleHTTPRequestHandler):
     name = form['path'].value
     content = form['content'].value
     method = 'a'
-    if form.has_key('clear') and form['clear'].value == '1':
+    if 'clear' in form and form['clear'].value == '1':
       method = 'w'
     self.writeFile(name, content, method)
     self.respond(200, type=self.headers['Content-Type'])
-    self.wfile.write("Content written to %s" % name)
+    self.wfile.write(b"Content written to %s" % str2bytes(name))
 
   def writeFile(self, filename, content, method='a'):
     file_path = os.path.join(self.document_path, filename)
@@ -97,7 +100,7 @@ def run(args):
   
   httpd = server((host, port), Handler)
   scheme = 'http'
-  if args.has_key('cert-file') and args.has_key('key-file') and \
+  if 'cert-file' in args and 'key-file' in args and \
       os.path.exists(args['cert-file']) and os.path.exists(args['key-file']):
     scheme = 'https'
     httpd.socket = ssl.wrap_socket (httpd.socket, 

software/backupserver/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 5c94d952305552dcbeaeaeb27dd28f3b
 
 [template-nginx-configuration]
 filename = template-nginx.cfg.in
-md5sum = c54d36f55ba71c897505ed61213e104a
+md5sum = a7012aac9b47ddd4e3aa3197aee1ffac
 
 [template-dcron-service]
 filename = template-dcron-service.sh.in

software/backupserver/template-nginx.cfg.in

@@ -48,7 +48,7 @@ http {
   gzip_comp_level 6;
   gzip_buffers 16 8k;
   gzip_http_version 1.1;
-  gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+  gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript;
 
 server {
   listen [$${nginx-configuration:ip}]:$${nginx-configuration:port};

software/erp5testnode/testsuite/deploy-test/runTestSuite.py

@@ -10,7 +10,7 @@ import tempfile
 import json
 
 
-SLEEP_TIME = 10
+SLEEP_TIME = 15
 TRY_AMOUNT = 3600
 
 

software/erp5testnode/testsuite/instance.cfg.in

@@ -6,7 +6,7 @@ default = $${:test}
 RootSoftwareInstance = $${:test}
 
 # Used for the test of resiliency. The system wants a "test" software_type.
-test = $${dynamic-template-resilient-test:rendered}
+test = dynamic-template-resilient-test:rendered
 
 [dynamic-template-resilient-test]
 recipe = slapos.recipe.template:jinja2

software/erp5testnode/testsuite/testsuite.cfg

 [template-erp5testnode]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = e5adcd511bca060bfeccec48b57c635c 
+md5sum = d864a32edab3c4f7207a7d4fe6bb7e30
 output = ${buildout:directory}/template.cfg
 mode = 0644
 

software/jstestnode/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 1d6bb3263f642a115982ddf245cb1cf0
+md5sum = e6203cba0084289edec2176b185d3427
 
 [template-nginx-service]
 filename = template-nginx-service.sh.in
@@ -23,7 +23,7 @@ md5sum = 529532e1240a66bdf39e3cbbef90ba87
 
 [template-nginx-configuration]
 filename = template-nginx.cfg.in
-md5sum = 9f22db89a2679534aa8fd37dbca86782
+md5sum = 98faa5ad8cfb23a11d97a459078a1d05
 
 [template-runTestSuite]
 filename = runTestSuite.in

software/jstestnode/instance.cfg.in

@@ -95,3 +95,4 @@ partition = $${slap_connection:partition_id}
 url = $${slap_connection:server_url}
 key = $${slap_connection:key_file}
 cert = $${slap_connection:cert_file}
+configuration._ = {}
\ No newline at end of file

software/jstestnode/template-nginx.cfg.in

@@ -48,7 +48,7 @@ http {
   gzip_comp_level 6;
   gzip_buffers 16 8k;
   gzip_http_version 1.1;
-  gzip_types text/html text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+  gzip_types text/html text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript;
 
 server {
   listen [$${nginx-configuration:ip}]:$${nginx-configuration:port};

software/jstestnode/test/README.md

+Tests for jstestnode software release

software/jstestnode/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.jstestnode'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' jstestnode",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'requests',
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/jstestnode/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+
+import httplib
+import json
+import os
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestJSTestNode(InstanceTestCase):
+
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertTrue('_' in parameter_dict)
+    try:
+      connection_dict = json.loads(parameter_dict['_'])
+    except Exception as e:
+      self.fail("Can't parse json in %s, error %s" % (parameter_dict['_'], e))
+
+    self.assertEqual(
+      {
+        'nginx': 'http://[%s]:9443/' % (self._ipv6_address, )
+      },
+      connection_dict
+    )
+
+    # jio tests
+    result = requests.get(
+      '%sjio/test/tests.html' % (connection_dict['nginx'], ), allow_redirects=False)
+    self.assertEqual(
+      [httplib.OK, False],
+      [result.status_code, result.is_redirect]
+    )
+
+    # rjs tests
+    result = requests.get(
+      '%srenderjs/test/' % (connection_dict['nginx'], ), allow_redirects=False)
+    self.assertEqual(
+      [httplib.OK, False],
+      [result.status_code, result.is_redirect]
+    )
+
+    # rsvp tests
+    result = requests.get(
+      '%srsvp/test/index.html' % (connection_dict['nginx'], ), allow_redirects=False)
+    self.assertEqual(
+      [httplib.OK, False],
+      [result.status_code, result.is_redirect]
+    )
+
+    # Default access
+    result = requests.get(
+      'http://[%s]:9443' % (self._ipv6_address, ), allow_redirects=False)
+    self.assertEqual(
+      [httplib.FORBIDDEN, False],
+      [result.status_code, result.is_redirect]
+    )

software/kvm/buildout.hash.cfg

@@ -15,15 +15,15 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = a236b719aaac61ac342ada0ce569151a
+md5sum = c9d4356b5148ed8ff8c4f3da63c137ce
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = fd0b26e5ae200ce2f7ee2a169731998c
+md5sum = 0db4ed796808b892a2e8a0aea704b13e
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = 2e743132ba4e001f784791311df9ba6a
+md5sum = 6f4c60f4366728021a6e438ad3dc6956
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
@@ -31,7 +31,7 @@ md5sum = 7de5756f59ef7d823cd8ed33e6d15230
 
 [template-kvm-import]
 filename = instance-kvm-import.cfg.jinja2.in
-md5sum = 97a8ff8a5891678274b14481dfc5214c
+md5sum = bd7e5db872b0dbe7716ec49c3907c401
 
 [template-kvm-import-script]
 filename = template/kvm-import.sh.jinja2
@@ -39,7 +39,7 @@ md5sum = cd0008f1689dfca9b77370bc4d275b70
 
 [template-kvm-export]
 filename = instance-kvm-export.cfg.jinja2
-md5sum = 00ce5e6da3c833d9d9d1825311f11a81
+md5sum = f12df4256eb5bd31a01c0ddc4b3897bb
 
 [template-kvm-export-script]
 filename = template/kvm-export.sh.jinja2
@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82
 
 [template-nbd]
 filename = instance-nbd.cfg.jinja2
-md5sum = 003112c44ef8d536861a46e3dfc67a37
+md5sum = 6ea26f88252bf899c966d0f5675e7176
 
 [template-ansible-promise]
 filename = template/ansible-promise.in
@@ -55,11 +55,11 @@ md5sum = 2036bf145f472f62ef8dee5e729328fd
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = 08af4ed0e2a53e76c844e3d7325aac09
+md5sum = c319ca536b6bac5425245fae1684ca49
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in
-md5sum = b96cba47c97f277c857176d69e086a12
+md5sum = 3827b4f7624de190cf5f5d37e3b72e86
 
 [template-apache-conf]
 filename = template/apache.conf.in

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -19,6 +19,14 @@ config-{{ name }} = {{ dumps(value) }}
 {% endif -%}
 {% endmacro -%}
 
+[slap-network-information]
+global-ipv6 = {{ ipv6 }}
+
+[slap-parameter]
+{% for k, v in slapparameter_dict.items() -%}
+{{ k }} = {{ v }}
+{% endfor -%}
+
 [request-common]
 recipe = slapos.cookbook:request
 software-url = ${slap-connection:software-release-url}

software/kvm/instance-kvm-export.cfg.jinja2

@@ -20,6 +20,11 @@ parts +=
 # monitor parts
   monitor-base
 
+[slap-parameter]
+{% for k, v in slapparameter_dict.items() -%}
+{{ k }} = {{ v }}
+{% endfor -%}
+
 # Create the exporter executable, which is a simple shell script
 [exporter]
 recipe = slapos.recipe.template:jinja2

software/kvm/instance-kvm-import.cfg.jinja2.in

@@ -12,6 +12,11 @@ parts +=
 extends = 
   {{ pbsready_import_template }}
 
+[slap-parameter]
+{% for k, v in slapparameter_dict.items() -%}
+{{ k }} = {{ v }}
+{% endfor -%}
+
 [resilient-publish-connection-parameter]
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 monitor-url = ${monitor-publish-parameters:monitor-url}

software/kvm/instance-kvm.cfg.jinja2

@@ -31,6 +31,10 @@
 {% do extends_list.append(template_monitor) -%}
 {% do extends_list.append(logrotate_cfg) -%}
 
+[slap-network-information]
+local-ipv4 = {{ slap_configuration['ipv4-random'] }}
+global-ipv6 = {{ slap_configuration['ipv6-random'] }}
+
 [directory]
 recipe = slapos.cookbook:mkdirectory
 etc = ${buildout:directory}/etc
@@ -454,38 +458,6 @@ interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://moni
 [helper]
 blank-line =
 
-[frontend-port-execute-base]
-recipe = plone.recipe.command
-command =
-  set -e
-  port=$(echo '${request-slave-frontend:connection-secure_access}' | cut -d ':' -f 3 | cut -d '/' -f 1)
-  [ -z $port ] && port=443
-  echo $port > ${:output}
-update-command = ${:command}
-stop-on-error = True
-
-[frontend-port-execute]
-<= frontend-port-execute-base
-secure_access = ${request-slave-frontend:connection-secure_access}
-output = ${directory:var}/frontend_port.txt
-
-[frontend-port]
-recipe = collective.recipe.shelloutput
-filename = ${frontend-port-execute:output}
-commands =
-  port = [ -f '${:filename}' ] && cat '${:filename}' || echo "NotReady"
-
-{% if additional_frontend %}
-[frontend-additional-port-execute]
-<= frontend-port-execute-base
-secure_access = ${request-slave-frontend-additional:connection-secure_access}
-output = ${directory:var}/frontend_additional_port.txt
-
-[frontend-additional-port]
-<= frontend-port
-filename = ${frontend-additional-port-execute:output}
-{% endif %}
-
 [publish-connection-information]
 <= monitor-publish
 recipe = slapos.cookbook:publish
@@ -493,7 +465,7 @@ ipv6 = ${slap-network-information:global-ipv6}
 backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
 url = ${request-slave-frontend:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
 {% if additional_frontend %}
-url-additional = ${request-slave-frontend-additional:connection-secure_access}/vnc.html?host=${request-slave-frontend-additional:connection-domain}&port=${frontend-additional-port:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
+url-additional = ${request-slave-frontend-additional:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
 {% endif %}
 {% set disk_number = len(storage_dict) -%}
 maximum-extra-disk-amount = {{ disk_number }}
@@ -781,6 +753,10 @@ data-to-vm =
 # Change keyboard layout language (Change to en-us if you face some bad bihaviors)
 keyboard-layout-language = fr
 
+{% for k, v in slapparameter_dict.items() -%}
+{{ k }} = {{ v }}
+{% endfor -%}
+
 #############################
 #
 # Instanciate kvm (Buildout Section)

software/kvm/instance-nbd.cfg.jinja2

@@ -9,9 +9,8 @@ parts =
   onetimeupload-promise
   publish-connection-information
 extends = {{ template_monitor }}
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
+
+{% set ipv6 = slap_configuration['ipv6-random'] -%}
 
 [rootdirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -27,7 +26,7 @@ watched-services = ${rootdirectory:etc}/service
 
 [nbd-instance]
 recipe = slapos.cookbook:nbdserver
-ip = ${slap-network-information:global-ipv6}
+ip = {{ ipv6 }}
 port = 1024
 image-path = ${onetimeupload-instance:image-path}
 qemu-path =  {{ qemu_nbd_executable_location }}
@@ -55,7 +54,7 @@ bytes = 24
 
 [onetimeupload-instance]
 recipe = slapos.cookbook:onetimeupload
-ip = ${slap-network-information:global-ipv6}
+ip = {{ ipv6 }}
 port = {{ slapparameter_dict.get('otu-port', 8080) }}
 image-path = ${rootdirectory:srv}/cdrom.iso
 log-path = ${rootdirectory:log}/onetimeupload.log
@@ -80,6 +79,10 @@ upload_key = ${onetimeupload-instance:key}
 status_message = ${detect-if-cdrom-present:status}
 
 [detect-if-cdrom-present]
-recipe = collective.recipe.shelloutput
-commands =
-  status = [ -f ${onetimeupload-instance:image-path} ] && echo "image already uploaded, you can't upload it again" || echo "WARNING: no image yet, the NBD server doesn't work"
+recipe = slapos.recipe.build
+init =
+  import os
+  options['status'] = (
+    "image already uploaded, you can't upload it again"
+    if os.path.isfile("${onetimeupload-instance:image-path}")
+    else "WARNING: no image yet, the NBD server doesn't work")

software/kvm/instance.cfg.in

@@ -8,18 +8,21 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 extends = ${template-resilient-templates:output}
 
 [switch_softwaretype]
-recipe = slapos.cookbook:softwaretype
+recipe = slapos.cookbook:switch-softwaretype
 default = $${:kvm}
-kvm-cluster = $${dynamic-template-kvm-cluster:rendered}
-kvm = $${dynamic-template-kvm:rendered}
-nbd = $${dynamic-template-nbd:rendered}
+kvm-cluster = dynamic-template-kvm-cluster:rendered
+kvm = dynamic-template-kvm:rendered
+nbd = dynamic-template-nbd:rendered
 
-kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
-kvm-import = $${dynamic-template-kvm-import:rendered}
-kvm-export = $${dynamic-template-kvm-export:rendered}
+kvm-resilient = dynamic-template-kvm-resilient:rendered
+kvm-import = dynamic-template-kvm-import:rendered
+kvm-export = dynamic-template-kvm-export:rendered
 
-frozen = ${instance-frozen:output}
-pull-backup = ${template-pull-backup:output}
+frozen = instance-frozen:rendered
+pull-backup = template-pull-backup:rendered
+
+# BBB
+RootSoftwareInstance = $${:default}
 
 # XXX - If this configuration is not generated by slapgrid, use empty values
 [storage-configuration]

software/kvm/software-py3.cfg

+[buildout]
+extends =
+  ../../component/python3/buildout.cfg
+  software.cfg
+
+python = python3
+
+# Ignore these for now
+common-parts -=
+  rdiff-backup

software/kvm/software.cfg

@@ -192,8 +192,7 @@ context =
     raw template_apache_conf ${template-apache-conf:location}/${template-apache-conf:filename}
 
 [versions]
-# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
-websockify = 0.5.1
+websockify = 0.9.0
 
 collective.recipe.environment = 0.2.0
 gitdb = 0.6.4

software/kvm/template/kvm-controller-run.in

@@ -57,7 +57,7 @@ def update():
         'nslot': 128,
         'canreboot': 1
       })
-  except Exception, e:
+  except Exception as e:
     write(str(e))
     raise
 

software/kvm/template/template-kvm-run.in

@@ -6,7 +6,10 @@ import hashlib
 import os
 import socket
 import subprocess
-import urllib
+try:
+  from urllib.request import FancyURLopener
+except ImportError:
+  from urllib import FancyURLopener
 import gzip
 import shutil
 from random import shuffle
@@ -96,9 +99,9 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe
 logfile = '{{ parameter_dict.get("log-file") }}'
 
 if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
-  opener = urllib.FancyURLopener(context=ssl._create_unverified_context())
+  opener = FancyURLopener(context=ssl._create_unverified_context())
 else:
-  opener = urllib.FancyURLopener({})
+  opener = FancyURLopener({})
 
 def md5Checksum(file_path):
     with open(file_path, 'rb') as fh:
@@ -112,22 +115,16 @@ def md5Checksum(file_path):
 
 def getSocketStatus(host, port):
   s = None
-  for res in socket.getaddrinfo(host, port,
-      socket.AF_UNSPEC, socket.SOCK_STREAM):
-    af, socktype, proto, canonname, sa = res
+  for af, socktype, proto, canonname, sa in socket.getaddrinfo(
+      host, port, socket.AF_UNSPEC, socket.SOCK_STREAM):
     try:
       s = socket.socket(af, socktype, proto)
-    except socket.error, msg:
-      s = None
-      continue
-    try:
       s.connect(sa)
-    except socket.error, msg:
-      s.close()
-      s = None
-      continue
-    break
-  return s
+      return s
+    except socket.error:
+      if s:
+        s.close()
+        s = None
 
 def getMapStorageList(disk_storage_dict, external_disk_number):
   map_disk_file = os.path.join(etc_directory, '.data-disk-ids')
@@ -319,7 +316,7 @@ for numa in numa_list:
 kvm_argument_list += numa_parameter
 
 if tap_network_parameter == [] and nat_network_parameter == []:
-  print 'Warning : No network interface defined.'
+  print('Warning : No network interface defined.')
 else:
   kvm_argument_list += nat_network_parameter + tap_network_parameter
 if language in language_list:
@@ -361,7 +358,7 @@ for nbd_ip, nbd_port in nbd_list:
     s = getSocketStatus(nbd_ip, nbd_port)
     if s is None:
       # NBD is not available : launch kvm without it
-      print 'Warning : Nbd is not available.'
+      print('Warning : Nbd is not available.')
     else:
       # NBD is available
       # We close the NBD socket else qemu won't be able to use it apparently
@@ -375,5 +372,5 @@ else:
       '-drive', 'file=%s,media=cdrom' % default_cdrom_iso
   ])
 
-print 'Starting KVM: \n %s' % ' '.join(kvm_argument_list)
+print('Starting KVM: \n %s' % ' '.join(kvm_argument_list))
 os.execv(qemu_path, kvm_argument_list)

software/kvm/test/setup.py

@@ -47,6 +47,7 @@ setup(name=name,
           'erp5.util',
           'supervisor',
           'psutil',
+          'six',
       ],
       zip_safe=True,
       test_suite='test',

software/kvm/test/test.py

@@ -25,13 +25,14 @@
 #
 ##############################################################################
 
-import httplib
+import six.moves.http_client as httplib
 import json
 import os
 import requests
+import six
 import slapos.util
 import sqlite3
-import urlparse
+from six.moves.urllib.parse import parse_qs, urlparse
 import unittest
 
 from slapos.recipe.librecipe import generateHashFromFiles
@@ -43,7 +44,8 @@ skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
 if has_kvm:
   setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
-        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+      os.path.join(os.path.dirname(__file__), '..',
+                   'software%s.cfg' % ("-py3" if six.PY3 else ""))))
 else:
   setUpModule, InstanceTestCase = None, unittest.TestCase
 
@@ -118,7 +120,7 @@ class MonitorAccessMixin(object):
     monitor_setup_url = connection_parameter_dict['monitor-setup-url']
     monitor_url_with_auth = 'https' + monitor_setup_url.split('https')[2]
 
-    auth = urlparse.parse_qs(urlparse.urlparse(monitor_url_with_auth).path)
+    auth = parse_qs(urlparse(monitor_url_with_auth).path)
 
     # check that monitor-base-url for all partitions in the tree are accessible
     # with published username and password
@@ -128,7 +130,7 @@ class MonitorAccessMixin(object):
       if not connection_xml:
         continue
       connection_dict = slapos.util.xml2dict(
-        partition_information['connection_xml'].encode('utf-8'))
+        connection_xml if six.PY3 else connection_xml.encode('utf-8'))
       monitor_base_url = connection_dict.get('monitor-base-url')
       if not monitor_base_url:
         continue
@@ -162,8 +164,8 @@ class TestAccessDefault(MonitorAccessMixin, InstanceTestCase):
       httplib.OK,
       result.status_code
     )
-    self.assertTrue('<title>noVNC</title>' in result.text)
-    self.assertFalse('url-additional' in connection_parameter_dict)
+    self.assertIn('<title>noVNC</title>', result.text)
+    self.assertNotIn('url-additional', connection_parameter_dict)
 
 
 @skipUnlessKvm
@@ -186,7 +188,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
       httplib.OK,
       result.status_code
     )
-    self.assertTrue('<title>noVNC</title>' in result.text)
+    self.assertIn('<title>noVNC</title>', result.text)
 
     result = requests.get(
       connection_parameter_dict['url-additional'], verify=False)
@@ -194,7 +196,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
       httplib.OK,
       result.status_code
     )
-    self.assertTrue('<title>noVNC</title>' in result.text)
+    self.assertIn('<title>noVNC</title>', result.text)
 
 
 @skipUnlessKvm
@@ -219,13 +221,13 @@ class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase):
   def test(self):
     connection_parameter_dict = self.computer_partition\
       .getConnectionParameterDict()
-    result = requests.get(connection_parameter_dict['kvm0-url'], verify=False)
+    result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
     self.assertEqual(
       httplib.OK,
       result.status_code
     )
-    self.assertTrue('<title>noVNC</title>' in result.text)
-    self.assertFalse('kvm0-url-additional' in connection_parameter_dict)
+    self.assertIn('<title>noVNC</title>', result.text)
+    self.assertNotIn('KVM0-url-additional', connection_parameter_dict)
 
 
 @skipUnlessKvm
@@ -253,20 +255,20 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
   def test(self):
     connection_parameter_dict = self.computer_partition\
       .getConnectionParameterDict()
-    result = requests.get(connection_parameter_dict['kvm0-url'], verify=False)
+    result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
     self.assertEqual(
       httplib.OK,
       result.status_code
     )
-    self.assertTrue('<title>noVNC</title>' in result.text)
+    self.assertIn('<title>noVNC</title>', result.text)
 
     result = requests.get(
-      connection_parameter_dict['kvm0-url-additional'], verify=False)
+      connection_parameter_dict['KVM0-url-additional'], verify=False)
     self.assertEqual(
       httplib.OK,
       result.status_code
     )
-    self.assertTrue('<title>noVNC</title>' in result.text)
+    self.assertIn('<title>noVNC</title>', result.text)
 
 
 @skipUnlessKvm

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 3ad1b06673000d9f424a1e7187c6a1fa
+md5sum = b21b2a9ac7f027a044a897c6eacbba56

software/slapos-sr-testing/instance.cfg

@@ -2,44 +2,41 @@
 parts =
   slapos-test-runner
 
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
 offline = true
 
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
 
 [download-source]
 recipe = slapos.recipe.build:gitclone
-git-executable = ${git:location}/bin/git
+git-executable = {{ git_location }}/bin/git
 
 [slapos]
 <= download-source
-repository = ${slapos-repository:location}
+repository = {{ slapos_location }}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
-bin = $${buildout:directory}/bin
-working-dir = $${buildout:directory}/tmp
-
-[test-list]
-path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup},${slapos.test.grafana-setup:setup},${slapos.test.gitlab-setup:setup}
+bin = ${buildout:directory}/bin
+working-dir = ${buildout:directory}/tmp
 
 [slapos-test-runner]
 recipe = slapos.cookbook:wrapper
-wrapper-path = $${directory:bin}/runTestSuite
+wrapper-path = ${directory:bin}/runTestSuite
 command-line =
-  ${buildout:bin-directory}/runTestSuite
-  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
-  --source_code_path_list=$${test-list:path_list}
+  {{ buildout['bin-directory'] }}/runTestSuite
+  --python_interpreter={{ buildout['bin-directory'] }}/{{ interpreter }}
+  --source_code_path_list={{ ','.join(tests.splitlines()) }}
 
 environment =
-  PATH=${buildout:bin-directory}:${quic_client-bin:location}:${curl:location}/bin/:/usr/bin/:/bin
-  SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
-  SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
-  SLAPOS_TEST_WORKING_DIR=$${directory:working-dir}
+  PATH={{ buildout['bin-directory'] }}:{{ quic_client_location }}:{{ curl_location }}/bin/:/usr/bin/:/bin
+  SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random}
+  SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random}
+  SLAPOS_TEST_WORKING_DIR=${directory:working-dir}

software/slapos-sr-testing/software-py3.cfg

+[buildout]
+
+extends =
+  ../../component/python3/buildout.cfg
+  software.cfg
+
+python = python3
+
+[eggs]
+eggs -=
+# plantuml is not Py3-compatible
+  ${slapos.test.plantuml-setup:egg}
+
+[template]
+extra =

software/slapos-sr-testing/software.cfg

@@ -27,6 +27,11 @@ recipe = zc.recipe.egg:develop
 egg = slapos.cookbook[test]
 setup = ${slapos-repository:location}/
 
+[slapos.test.backupserver-setup]
+<= setup-develop-egg
+egg = slapos.test.backupserver
+setup = ${slapos-repository:location}/software/backupserver/test/
+
 [slapos.test.caddy-frontend-setup]
 <= setup-develop-egg
 egg = slapos.test.caddy-frontend
@@ -37,11 +42,21 @@ setup = ${slapos-repository:location}/software/caddy-frontend/test/
 egg = slapos.test.erp5
 setup = ${slapos-repository:location}/software/erp5/test/
 
+[slapos.test.htmlvalidatorserver-setup]
+<= setup-develop-egg
+egg = slapos.test.htmlvalidatorserver
+setup = ${slapos-repository:location}/software/htmlvalidatorserver/test/
+
 [slapos.test.slapos-master-setup]
 <= setup-develop-egg
 egg = slapos.test.slapos-master
 setup = ${slapos-repository:location}/software/slapos-master/test/
 
+[slapos.test.jstestnode-setup]
+<= setup-develop-egg
+egg = slapos.test.jstestnode
+setup = ${slapos-repository:location}/software/jstestnode/test/
+
 [slapos.test.kvm-setup]
 <= setup-develop-egg
 egg = slapos.test.kvm
@@ -140,9 +155,12 @@ eggs =
   slapos.libnetworkcache
   supervisor
   ${slapos.cookbook-setup:egg}
+  ${slapos.test.backupserver-setup:egg}
   ${slapos.test.caddy-frontend-setup:egg}
   ${slapos.test.erp5-setup:egg}
+  ${slapos.test.htmlvalidatorserver-setup:egg}
   ${slapos.test.slapos-master-setup:egg}
+  ${slapos.test.jstestnode-setup:egg}
   ${slapos.test.kvm-setup:egg}
   ${slapos.test.monitor-setup:egg}
   ${slapos.test.plantuml-setup:egg}
@@ -176,10 +194,43 @@ repository = https://lab.nexedi.com/nexedi/slapos.git
 branch = master
 
 [template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/${:filename}
-output = ${buildout:directory}/template.cfg
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+rendered = ${buildout:directory}/template.cfg
 mode = 640
+context =
+  section buildout buildout
+  key git_location git:location
+  key slapos_location slapos-repository:location
+  key interpreter eggs:interpreter
+  key quic_client_location quic_client-bin:location
+  key curl_location curl:location
+  key tests :tests
+tests =
+  ${slapos.test.kvm-setup:setup}
+  ${slapos.test.slaprunner-setup:setup}
+  ${:extra}
+extra =
+  ${slapos.cookbook-setup:setup}
+  ${slapos.test.backupserver-setup:setup}
+  ${slapos.test.caddy-frontend-setup:setup}
+  ${slapos.test.erp5-setup:setup}
+  ${slapos.test.htmlvalidatorserver-setup:setup}
+  ${slapos.test.slapos-master-setup:setup}
+  ${slapos.test.monitor-setup:setup}
+  ${slapos.test.plantuml-setup:setup}
+  ${slapos.test.powerdns-setup:setup}
+  ${slapos.test.proftpd-setup:setup}
+  ${slapos.test.re6stnet-setup:setup}
+  ${slapos.test.seleniumserver-setup:setup}
+  ${slapos.test.helloworld-setup:setup}
+  ${slapos.test.jstestnode-setup:setup}
+  ${slapos.test.jupyter-setup:setup}
+  ${slapos.test.nextcloud-setup:setup}
+  ${slapos.test.turnserver-setup:setup}
+  ${slapos.test.theia-setup:setup}
+  ${slapos.test.grafana-setup:setup}
+  ${slapos.test.gitlab-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always

software/slaprunner/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 8b78e32b877d591400746ec7fd68ed4c
+md5sum = ec70348dd71b319590a5c5837f3d2e45
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 84b81c18c77af08d7e97b0b6da4ee105
+md5sum = b20894f378530e79c847ddfb61782cc5
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
@@ -34,7 +34,7 @@ md5sum = b992bb3391de9d6d422bfa8011d8ffc4
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2
-md5sum = 2271c829b94542b7b2d9c589376ae538
+md5sum = 105ed7f54f251c64a2d34559360a5604
 
 [template_nginx_conf]
 filename = nginx_conf.in
@@ -62,7 +62,7 @@ md5sum = 7645048216fcf957f7773534cd0408dc
 
 [template-supervisord]
 filename = template/supervisord.conf.in
-md5sum = d294d0dafd265048399de6da8c96345f
+md5sum = 28f69b57c2835bddfcfbe312563eef51
 
 [template-listener-slapgrid]
 filename = template/listener_slapgrid.py.in

software/slaprunner/instance-resilient.cfg.jinja2

@@ -19,7 +19,7 @@
 {% set monitor_interface_url = slapparameter_dict.pop('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
 
 {% import 'parts' as parts %}
-{% import 'replicated' as replicated %}
+{% import 'replicated' as replicated with context %}
 
 [buildout]
 eggs-directory = {{ eggs_directory }}

software/slaprunner/instance-runner.cfg

@@ -51,9 +51,9 @@ extends =
   ${monitor2-template:rendered}
   ${template-logrotate-base:rendered}
 
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
+[slap-network-information]
+local-ipv4 = $${slap-configuration:ipv4-random}
+global-ipv6 = $${slap-configuration:ipv6-random}
 
 {% if slapparameter_dict.get('custom-frontend-backend-url') -%}
 [request-custom-frontend]
@@ -287,35 +287,29 @@ extra-args=-t rsa
 <=runner-sshd-ssh-keygen-base
 extra-args=-t ecdsa -b 521
 
-[runner-sshd-publickey-fingerprint-shelloutput]
-recipe = collective.recipe.shelloutput
-# XXX because collective.recipe.shelloutput ignore errors, we run the same
-# command in a plone.recipe.command so that if fails if something goes wrong.
-commands =
-  fingerprint = bash -o pipefail -c "${openssh-output:keygen} -lf $${runner-sshd-ssh-host-ecdsa-key:output} | cut  -f 2 -d\ | sed 's/+/%2B/g' | sed 's/\//%2F/g' | sed 's/SHA256://'"
-
 [runner-sshd-publickey-fingerprint]
 # fingerprint for ssh url, see
 # https://tools.ietf.org/id/draft-salowey-secsh-uri-00.html#connparam
 # https://winscp.net/eng/docs/session_url#hostkey
-
-_fingerprint = $${runner-sshd-publickey-fingerprint-shelloutput:fingerprint}
-
-# format is host-key-alg-fingerprint, but we know that
-# $${runner-sshkeys-sshd:public-key} is rsa so for host-key-alg
-# we just use use rsa.
-fingerprint = ssh-rsa-$${:_fingerprint}
-
-# XXX because collective.recipe.shelloutput ignore errors and capture output
-# "Error ...", we use a plone.recipe.command to check that this command did
-# not fail.
-# This command will always fail on first buildout run, because
-# collective.recipe.shelloutput is evaluated at buildout recipes __init__ step,
-# but the key file is created later at install step.
-recipe = plone.recipe.command
-stop-on-error = true
-command = echo "$${:_fingerprint}" | ( grep ^Error || exit 0 && exit 1 )
-
+recipe = slapos.recipe.build
+init =
+  import os
+  import subprocess
+  from six.moves.urllib.parse import quote
+  keyfile = self.buildout['runner-sshd-ssh-host-ecdsa-key']['output']
+  if os.path.isfile(keyfile):
+    x = subprocess.check_output(('${openssh-output:keygen}', '-lf', keyfile))
+    x = x.split()[1]
+    assert x.startswith(b'SHA256:'), x
+    # format is host-key-alg-fingerprint, but we know that
+    # $${runner-sshd-ssh-host-ecdsa-key:output} is rsa so for host-key-alg
+    # we just use use rsa.
+    options['fingerprint'] = "ssh-rsa-" + quote(x[7:], safe='')
+  else:
+    # This command will always fail on first buildout run, because it is
+    # evaluated at buildout recipes __init__ step, but the key file is created
+    # later at install step.
+    options['fingerprint'] = "NotReady"
 
 #---------------------------
 #--
@@ -665,6 +659,9 @@ monitor-cors-domains =
 monitor-interface-url =
 monitor-httpd-port = 8386
 buildout-shared-folder = $${runnerdirectory:home}/shared
+{% for k, v in slapparameter_dict.items() -%}
+{{ k }} = {{ v }}
+{% endfor -%}
 
 [slapos-cfg]
 recipe = slapos.recipe.template:jinja2
@@ -745,15 +742,6 @@ recipe = plone.recipe.command
 stop-on-error = true
 command = SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
 
-[slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
-
-
 [minishell-cwd]
 recipe = plone.recipe.command
 command = if [ ! -f $${slaprunner:minishell_cwd_file} ]; then echo $${runnerdirectory:home} > $${slaprunner:minishell_cwd_file}; fi
@@ -820,7 +808,6 @@ template = ${template-supervisord:location}/${template-supervisord:filename}
 rendered = $${directory:etc}/supervisord.conf
 context =
     import multiprocessing multiprocessing
-    import builtin __builtin__
     section supervisord supervisord
     key slapparameter_dict slap-configuration:configuration
     key listener_slapgrid listener-slapgrid-bin:rendered

software/slaprunner/instance.cfg

@@ -9,15 +9,18 @@ extends =
   ${template-resilient-templates:output}
 
 [switch_softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = $${instance-base-runner:rendered}
-resilient = $${instance-resilient:rendered}
-runner = $${instance-base-runner:rendered}
-runner-import = $${template-runner-import:rendered}
-runner-export = $${template-runner-export:rendered}
+recipe = slapos.cookbook:switch-softwaretype
+default = $${:runner}
+resilient = instance-resilient:rendered
+runner = instance-base-runner:rendered
+runner-import = template-runner-import:rendered
+runner-export = template-runner-export:rendered
 
-frozen = ${instance-frozen:output}
-pull-backup = ${template-pull-backup:output}
+frozen = instance-frozen:rendered
+pull-backup = template-pull-backup:rendered
+
+# BBB
+RootSoftwareInstance = $${:default}
 
 [instance-base-runner]
 recipe = slapos.recipe.template:jinja2

software/slaprunner/software-py3.cfg

+[buildout]
+extends =
+  ../../component/python3/buildout.cfg
+  software.cfg
+
+python = python3
+
+# Ignore these for now
+common-parts -=
+  rdiff-backup
+
+[eggs]
+eggs -=
+# futures is a backport of Py3's concurrent.futures module
+  futures

software/slaprunner/software.cfg

@@ -150,10 +150,10 @@ eggs =
   erp5.util
   lock-file
   plone.recipe.command
-  collective.recipe.shelloutput
   slapos.recipe.build
   slapos.toolbox[flask_auth]
-  gunicorn==19.7.1
+  gunicorn
+# for gunicorn[gthread]
   futures
   ${slapos-cookbook:eggs}
   slapos.core # listed explicitly for scripts generation
@@ -169,7 +169,7 @@ Flask-Auth = 0.85
 cns.recipe.symlink = 0.2.3
 futures = 3.0.5
 gitdb = 0.6.4
-gunicorn = 19.7.1
+gunicorn = 19.10.0
 prettytable = 0.7.2
 pycurl = 7.43.0
 slapos.recipe.template = 4.4

software/slaprunner/template/supervisord.conf.in

@@ -27,7 +27,7 @@ stdout_logfile = {{ supervisord['no_logfile'] }}
 stderr_logfile = {{ supervisord['no_logfile'] }}
 directory = {{ supervisord['directory'] }}
 {# how many parallel build jobs to spawn when compiling software -#}
-{% set njobs = builtin.max(1, (multiprocessing.cpu_count() // builtin.int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%}
+{% set njobs = max(1, (multiprocessing.cpu_count() // int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%}
 environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{ njobs }}",NPY_NUM_BUILD_JOBS="{{ njobs }}",BUNDLE_JOBS="{{ njobs }}"
 
 [program:{{- supervisord['slapgrid-cp'] -}}]

software/slaprunner/test/test.py

@@ -38,13 +38,16 @@ from six.moves.urllib.parse import quote
 from six.moves.urllib.parse import urljoin
 from six.moves.configparser import ConfigParser
 import requests
+import six
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.util import bytes2str
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
-        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+        os.path.join(os.path.dirname(__file__), '..',
+                     'software%s.cfg' % ("-py3" if six.PY3 else ""))))
 
 
 class SlaprunnerTestCase(SlapOSInstanceTestCase):
@@ -185,7 +188,7 @@ class TestSSH(SlaprunnerTestCase):
       channel.settimeout(30)
       received = ''
       while True:
-        r = channel.recv(1024)
+        r = bytes2str(channel.recv(1024))
         self.logger.debug("received >%s<", r)
         if not r:
           break
@@ -197,7 +200,7 @@ class TestSSH(SlaprunnerTestCase):
       # simple commands can also be executed ( this would be like `ssh bash -c 'pwd'` )
       self.assertEqual(
           self.computer_partition_root_path,
-          client.exec_command("pwd")[1].read(1000).strip())
+          bytes2str(client.exec_command("pwd")[1].read(1000)).strip())
 
 
 class TestSlapOS(SlaprunnerTestCase):
@@ -211,7 +214,7 @@ class TestSlapOS(SlaprunnerTestCase):
             'show',
         ),
         env={})
-    self.assertIn('slaprunner', proxy_show_output)
+    self.assertIn(b'slaprunner', proxy_show_output)
 
   def test_shared_part_list(self):
     # this slapos used shared_part_list

software/theia/buildout.hash.cfg

@@ -23,4 +23,4 @@ md5sum = b63a993a13e7c3b16b66c25fac5ac8b9
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt
-md5sum = ce87c9818d64a16fce49c84ed5dba8fc
+md5sum = 6c940b7015f45de6e679fc44807e5220

software/theia/python-language-server-requirements.txt

@@ -19,5 +19,4 @@ six==1.12.0
 snowballstemmer==1.2.1
 toml==0.10.0
 typed-ast==1.2.0
-typeshed==0.0.1
 yapf==0.28.0

stack/resilient/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [pbsready]
 filename = pbsready.cfg.in
-md5sum = 5e0dcd4c290f0b46cb2d316dc1c9c011
+md5sum = 66331047b7dbf2513c5726d5d1647320
 
 [pbsready-import]
 filename = pbsready-import.cfg.in
@@ -22,15 +22,15 @@ md5sum = d813c43ed00eff868fb13bc75b045336
 
 [pbsready-export]
 filename = pbsready-export.cfg.in
-md5sum = 2e804e06b5203c3f127c31a1704c48bd
+md5sum = 2b0c71b085cfe8017f28098c160b1f49
 
 [template-pull-backup]
 filename = instance-pull-backup.cfg.in
-md5sum = 0bbe16f3d805afd880a251a4f40ecaf1
+md5sum = 555d528b198564f0ce1e94db1160ebf3
 
 [template-replicated]
 filename = template-replicated.cfg.in
-md5sum = 290b380fe3da8736642bc10a8b1163d1
+md5sum = 815fd8f7c42b9cf59b286b0fe77fa76d
 
 [template-parts]
 filename = template-parts.cfg.in
@@ -38,7 +38,7 @@ md5sum = 071b1034ee8f5cc14f79b16fdeba2813
 
 [template-resilient-templates]
 filename = template-resilient-templates.cfg.in
-md5sum = 41e82859dc6b65e94a300a006d51536e
+md5sum = 097a14371efde11465ab4bd08ef3131b
 
 [instance-frozen]
 filename = instance-frozen.cfg.in
@@ -46,7 +46,7 @@ md5sum = d21472f0e58f928fb827f2cbf22c4d4a
 
 [resilient-web-takeover-cgi-script-download]
 filename = resilient-web-takeover-cgi-script.py.in
-md5sum = 60d4912fdf5e8dafaba9d9f333aa9e36
+md5sum = 675ac9e1cf49ccc8f8eddb541a62d899
 
 [template-wrapper]
 filename = templates/wrapper.in
@@ -54,9 +54,9 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 
 [notifier-feed-promise-template]
 filename = templates/notifier-feed-promise.py.in
-md5sum = d75346911dbc4cfcdb39a21e56cd5016
+md5sum = fa6521daaa02fef4dd2ce06d29ef90be
 
 [template-monitor-check-resilient-feed]
 filename = templates/monitor-check-resilient-feed.in
-md5sum = 19ee9055de961acf402e2dfe5b9581d2
+md5sum = af9787f8440fef19924b2e765372b20f
 

stack/resilient/instance-pull-backup.cfg.in

@@ -19,10 +19,6 @@ parts =
 extends =
   ${monitor2-template:rendered}
   ${template-logrotate-base:rendered}
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
 
 #----------------
 #--
@@ -90,7 +86,7 @@ rendered = $${:wrapper}
 wrapper = $${basedirectory:services}/notifier
 mode = 0700
 command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port}
-host = $${slap-network-information:global-ipv6}
+host = {{ ipv6 }}
 port = $${notifier-port:port}
 context =
   key content notifier:command
@@ -128,8 +124,8 @@ run-directory = $${basedirectory:run}
 pull-push-maximum-run = 5
 # XXX: this should be named "notifier-host"
 notifier-url = http://[$${notifier:host}]:$${notifier:port}
-slave-instance-list = $${slap-parameter:slave_instance_list}
-ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file}
+slave-instance-list = {{ slapparameter_dict.get('slave_instance_list', '[]') }}
+ignore-known-hosts-file = {{ slapparameter_dict.get('ignore-known-hosts-file', 'false') }}
 # To get a verbose feed about PBS state
 instance-root-name = $${monitor-instance-parameter:root-instance-title}
 log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/
@@ -194,20 +190,6 @@ identity-file = $${:home}/id_rsa
 command-line = ${openssh:location}/bin/ssh -T -o "UserKnownHostsFile $${pbs:known-hosts}" -i $${:identity-file}
 wrapper-path = $${rootdirectory:bin}/ssh
 
-
-#----------------
-#--
-#-- Slave instance list (empty default).
-
-[slap-parameter]
-slave_instance_list = []
-ignore-known-hosts-file = false
-monitor-cors-domains = 
-monitor-httpd-port = 8070
-monitor-title = PBS Instance
-monitor-password = $${monitor-htpasswd:passwd}
-monitor-username = admin
-
 #----------------
 #--
 #-- Resiliency promises.
@@ -269,11 +251,11 @@ monitor-password = $${monitor-publish-parameters:monitor-password}
 #-- Monitor
 
 [monitor-instance-parameter]
-monitor-httpd-port = $${slap-parameter:monitor-httpd-port}
-monitor-title = $${slap-parameter:monitor-title}
-cors-domains = $${slap-parameter:monitor-cors-domains}
-username = $${slap-parameter:monitor-username}
-password = $${slap-parameter:monitor-password}
+monitor-httpd-port = {{ slapparameter_dict.get('monitor-httpd-port', 8070) }}
+monitor-title = {{ slapparameter_dict.get('monitor-title', 'PBS Instance') }}
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', '') }}
+username = {{ slapparameter_dict.get('monitor-username', 'admin') }}
+password = {{ slapparameter_dict.get('monitor-password', '$${monitor-htpasswd:passwd}') }}
 
 [monitor-conf-parameters]
 private-path-list += 

stack/resilient/pbsready-export.cfg.in

@@ -39,7 +39,7 @@ name = exporter
 title = Dumping ${slap-parameter:namebase}
 executable = ${exporter:wrapper}
 wrapper = ${rootdirectory:bin}/exporter
-notify = ${slap-parameter:notify}
+notify = {{ slapparameter_dict.get('notify', '') }}
 pidfile = ${resilient-directory:pid}/${:name}.pid
 max-run = 3
 
@@ -85,8 +85,3 @@ recipe = slapos.cookbook:random.time
 recipe = slapos.cookbook:publish-early
 -init =
   resiliency-backup-periodicity gen-resiliency-backup-periodicity:time
-
-[slap-parameter]
-# In cron.d format (i.e things like */15 * * * * are accepted).
-resiliency-backup-periodicity =
-notify =
\ No newline at end of file

stack/resilient/pbsready.cfg.in

@@ -22,6 +22,10 @@ parts +=
 extends =
   ${monitor2-template:rendered}
 
+[slap-network-information]
+local-ipv4 = $${slap-configuration:ipv4-random}
+global-ipv6 = $${slap-configuration:ipv6-random}
+
 #----------------
 #--
 #-- Creation of all needed directories.

stack/resilient/resilient-web-takeover-cgi-script.py.in

 #!${buildout:executable}
 
+from __future__ import print_function
 equeue_database = '${equeue:database}'
 equeue_lockfile = '${equeue:lockfile}'
 takeover_script = '${resiliency-takeover-script:wrapper-takeover}'
@@ -8,7 +9,11 @@ import atexit
 import cgi
 import cgitb
 import datetime
-import gdbm
+try:
+  import dbm.gnu as gdbm
+except ImportError:
+  import gdbm
+
 import os
 import shutil
 import subprocess
@@ -39,11 +44,12 @@ def getLatestBackupDate():
   # Usually, there is only one callback (so only one key
   # in the db), but if there are several:
   # Take the "oldest" one (oldest value).
-  if not db.keys():
+  db_keys = db.keys()
+  if not db_keys:
     result = False
   else:
-    last_backup = db[db.keys()[-1]]
-    for callback in db.keys():
+    last_backup = db[db_keys[-1]]
+    for callback in db_keys:
       timestamp = float(db[callback])
       if timestamp < last_backup:
         last_backup = timestamp
@@ -79,12 +85,11 @@ if latest_backup_date == False:
 else:
   latest_backup_message = latest_backup_date.strftime('%Y-%m-%d %H:%M:%S')
 
-print "Content-Type: text/html"
-print
+print("Content-Type: text/html\n")
 
 form = cgi.FieldStorage()
 if "password" not in form:
-  print """<html>
+  print("""<html>
 <body>
   <h1>This is takeover web interface.</h1>
   <p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p>
@@ -100,15 +105,15 @@ if "password" not in form:
     <input type="submit" value="Take over" style="background: red;">
   </form>
 </body>
-</html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted())
+</html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted()))
   sys.exit(0)
 
 if form['password'].value != '${:password}':
-  print "<H1>Error</H1>"
-  print "Password is invalid."
+  print("<H1>Error</H1>")
+  print("Password is invalid.")
   sys.exit(1)
 
 # XXX hardcoded location
 result = subprocess.check_output([takeover_script], stderr=subprocess.STDOUT)
-print 'Success.'
-print '<pre>%s</pre>' % result
+print('Success.')
+print('<pre>%s</pre>' % result)

stack/resilient/template-replicated.cfg.in

@@ -41,7 +41,7 @@ config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}
 config-name = {{namebase}}0
 # Bubble up all the instance parameters to the requested export instance.
 {% if slapparameter_dict is defined %}
-{% for parameter_name, parameter_value in slapparameter_dict.items() %}
+{% for parameter_name, parameter_value in six.iteritems(slapparameter_dict) %}
 {% if parameter_value is string %}
 config-{{parameter_name}} = {{ parameter_value.split('\n') | join('\n  ') }}
 {% else %}
@@ -49,7 +49,7 @@ config-{{parameter_name}} = {{ parameter_value }}
 {% endif %}
 {% endfor %}
 {% endif %}
-{% for key, value in monitor_dict.iteritems() -%}
+{% for key, value in six.iteritems(monitor_dict) -%}
 config-{{ key }} = {{ value }}
 {% endfor -%}
 {% if sla_parameter_dict == {} -%}
@@ -67,7 +67,7 @@ sla-mode = unique_by_network
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}
-{%   for key, value in sla_dict.iteritems() -%}
+{%   for key, value in six.iteritems(sla_dict) -%}
 sla-{{ key }} = {{ value }}
 {%   endfor -%}
 {% endif -%}
@@ -101,7 +101,7 @@ config-number = {{id}}
 config-name = {{namebase}}{{id}}
 config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
 config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
-{% for key, value in monitor_dict.iteritems() -%}
+{% for key, value in six.iteritems(monitor_dict) -%}
 config-{{ key }} = {{ value }}
 {% endfor -%}
 {% if sla_parameter_dict == {} -%}
@@ -119,7 +119,7 @@ sla-mode = unique_by_network
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}
-{%   for key, value in sla_dict.iteritems() -%}
+{%   for key, value in six.iteritems(sla_dict) -%}
 sla-{{ key }} = {{ value }}
 {%   endfor -%}
 {% endif %}
@@ -202,7 +202,7 @@ software-type = pull-backup
 name = PBS ({{namebase}} / {{id}})
 config-ignore-known-hosts-file = ${slap-parameter:ignore-known-hosts-file}
 config-monitor-title = PBS ${slap-connection:computer-id}-{{namebase}}-{{id}}
-{% for key, value in monitor_dict.iteritems() -%}
+{% for key, value in six.iteritems(monitor_dict) -%}
 config-{{ key }} = {{ value }}
 {% endfor -%}
 return = ssh-key notification-url feeds-url  {{ monitor_return }}
@@ -222,7 +222,7 @@ sla-mode = unique_by_network
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}
-{%   for key, value in sla_dict.iteritems() -%}
+{%   for key, value in six.iteritems(sla_dict) -%}
 sla-{{ key }} = {{ value }}
 {%   endfor %}
 {% endif %}

stack/resilient/template-resilient-templates.cfg.in

@@ -8,3 +8,20 @@ context =
   raw pbsready_template_path ${pbsready:output}
   raw bash_executable_location ${bash:location}/bin/bash
   raw logrotate_executable_location ${logrotate:location}/usr/sbin/logrotate
+
+[instance-frozen]
+recipe = slapos.recipe.template:jinja2
+template = ${instance-frozen:output}
+rendered = $${buildout:directory}/template-frozen.cfg
+extensions = jinja2.ext.do
+context =
+  key slapparameter_dict slap-configuration:configuration
+
+[template-pull-backup]
+recipe = slapos.recipe.template:jinja2
+template = ${template-pull-backup:output}
+rendered = $${buildout:directory}/template-pull-backup.cfg
+extensions = jinja2.ext.do
+context =
+  key slapparameter_dict slap-configuration:configuration
+  key ipv6 slap-configuration:ipv6-random

stack/resilient/templates/monitor-check-resilient-feed.in

 #!{{ python_executable }}
 
+from __future__ import print_function
 import os
-import urllib2
+try:
+  from urllib2 import HTTPError, urlopen
+except ImportError:
+  from urllib.error import HTTPError
+  from urllib.request import urlopen
 import sys
 
 input_feed_directory = '{{ input_feed_directory }}'
@@ -12,12 +17,14 @@ feed_file_list = os.listdir(input_feed_directory)
 rss_ok = True
 
 for feed_file_name in feed_file_list:
-  print "Getting %s" % feed_file_name
+  print("Getting", feed_file_name)
   url = base_url + feed_file_name
   try:
-    feed = urllib2.urlopen(url)
+    feed = urlopen(url)
     body = feed.read()
-    open(os.path.join(monitor_feed_directory, feed_file_name + '.rss'), 'w').write(body)
-    print "FEED is ok"
-  except urllib2.HTTPError as e:
+    with open(os.path.join(monitor_feed_directory, feed_file_name + '.rss'),
+              'wb') as f:
+      f.write(body)
+    print("FEED is ok")
+  except HTTPError as e:
     sys.exit("%s is unvailable: %s" % (feed_file_name, e))

stack/resilient/templates/notifier-feed-promise.py.in

@@ -2,7 +2,11 @@
 import csv
 import os
 import sys
-import urllib2
+try:
+  from urllib2 import HTTPError, urlopen
+except ImportError:
+  from urllib.error import HTTPError
+  from urllib.request import urlopen
 
 csv.field_size_limit(sys.maxsize)
 
@@ -15,9 +19,9 @@ for feed_file_name in feed_file_list:
   url = base_url + feed_file_name
   # Try feed consistency
   try:
-    feed = urllib2.urlopen(url)
+    feed = urlopen(url)
     body = feed.read()
-  except urllib2.HTTPError as e:
+  except HTTPError as e:
     sys.exit("%s is unavailable: %s" % (feed_file_name, e))
   with open(os.path.join(notifier_feed_directory, feed_file_name)) as feed_file:
     reader = csv.reader(feed_file)

stack/slapos.cfg

@@ -145,7 +145,7 @@ slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.4
 slapos.recipe.build = 0.43
 slapos.recipe.cmmi = 0.12
-slapos.toolbox = 0.108
+slapos.toolbox = 0.109
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1