slapos-master|wendelin: Add Python3 Software Release version

  This is a PoC to compile wendelin and slapos-master with python3, and
  specially to run the tests.

software/slapos-master/base.cfg

+[erp5-defaults]
+wcfs-enable-default = true
+
+[local-bt5-repository]
+# Same as bt5-repository, but only local repository.
+# Used to generate bt5lists.
+list += ${vifib:location}/master/bt5
+
+[default-bt5]
+list =
+  erp5_full_text_mroonga_catalog
+  slapos_configurator
+
+[erp5_repository_list]
+repository_id_list += vifib/master
+
+[erp5]
+branch = erp5-vifib
+
+[vifib]
+<= erp5
+repository = https://lab.nexedi.com/nexedi/slapos.core.git
+branch = master
+
+[slapos-bin]
+<= erp5
+repository = https://lab.nexedi.com/nexedi/slapos-bin.git
+branch = master
+
+[eggs]
+eggs +=
+  slapos.core
+
+dummy +=
+  ${vifib:location}
+  ${slapos-bin:location}
+
+extra-paths +=
+  ${vifib:location}/master
+  ${slapos-bin:location}
+
+
+### Overwrite recipes to introduce customized changes
+[download-base-part]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+
+[template-erp5]
+< = download-base-part
+filename = instance-erp5.cfg.in
+
+[template-balancer]
+< = download-base-part
+filename = instance-balancer.cfg.in
+
+[template-apache-backend-conf]
+url = ${:_profile_base_location_}/${:filename}
+
+[template-haproxy-cfg]
+url = ${:_profile_base_location_}/${:filename}

software/slapos-master/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = cf5fbee0b5e104015eb9eac53ba2d241
+md5sum = 9f1245e834fb0861db59e7c5ec2d8805
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 8af2ed33ef2a57cf5c5df4dd5d834d69
+md5sum = 87516dad789d74456d120866089d7efd
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in
@@ -26,4 +26,4 @@ md5sum = aa11283ca8b80a91b508732cd3443f7b
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = fec6a312e4ef84b02837742992aaf495
+md5sum = 3848f3b78811d72b7633973fad9c54e9

software/slapos-master/haproxy.cfg.in

@@ -32,7 +32,7 @@ defaults
   # to render a page
   option httpclose
 
-{% for name, (port, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
+{% for name, (port, backend_list) in sorted(six.iteritems(parameter_dict['backend-dict'])) -%}
 listen {{ name }}
   bind {{ parameter_dict['ip'] }}:{{ port }}
   http-request set-header X-Balancer-Current-Cookie SERVERID

software/slapos-master/instance-balancer.cfg.in

@@ -39,9 +39,9 @@ recipe = slapos.recipe.template:jinja2
 {% set apache_dict = {} -%}
 {% set zope_virtualhost_monster_backend_dict = {} %}
 {% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
-{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
+{% set next_port = functools.partial(next, itertools.count(slapparameter_dict['tcpv4-port'])) -%}
 {% for family_name, parameter_id_list in sorted(
-  slapparameter_dict['zope-family-dict'].iteritems()) -%}
+  six.iteritems(slapparameter_dict['zope-family-dict'])) -%}
 {%   set zope_family_address_list = [] -%}
 {%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
 {%   set has_webdav = [] -%}
@@ -154,7 +154,7 @@ cert = ${apache-conf-ssl:cert}
 {%- endif %}
 
 [apache-conf-parameter-dict]
-backend-list = {{ dumps(apache_dict.values()) }}
+backend-list = {{ dumps(list(apache_dict.values())) }}
 zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
 ip-list = {{ dumps(apache_ip_list) }}
 pid-file = ${directory:run}/apache.pid
@@ -207,7 +207,7 @@ input = inline:
 promise = check_socket_listening
 name = apache.py
 config-host = {{ ipv4 }}
-config-port = {{ apache_dict.values()[0][0] }}
+config-port = {{ list(apache_dict.values())[0][0] }}
 
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised

software/slapos-master/instance-erp5.cfg.in

@@ -11,6 +11,10 @@
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
 {% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
 {% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
+{% set with_max_rlimit_nofile_enable = slapparameter_dict.get(
+    'with-max-rlimit-nofile',
+    with_max_rlimit_nofile_enable_default.lower() in ('true', 'yes'))
+%}
 {% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
 {% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
 {% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
@@ -32,7 +36,7 @@
 {% set partition_thread_count_list = [] -%}
 {% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
 {% set zope_family_override_dict = slapparameter_dict.get('family-override', {}) -%}
-{% for zope_parameter_dict in zope_partition_dict.values() -%}
+{% for zope_parameter_dict in six.itervalues(zope_partition_dict) -%}
 {#   Apply some zope_parameter_dict default values, to avoid duplication. -#}
 {%   do zope_parameter_dict.setdefault('thread-amount', 4) -%}
 {%   do zope_parameter_dict.setdefault('instance-count', 1) -%}
@@ -53,7 +57,7 @@ config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-m
 name = {{ name }}
 software-type = {{ software_type }}
 return = {{ ' '.join(ret) }}
-{% for ret, publish in ret.iteritems() -%}
+{% for ret, publish in six.iteritems(ret) -%}
 {%   if publish -%}
 {%     do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}') %}
 {%   endif -%}
@@ -62,10 +66,10 @@ return = {{ ' '.join(ret) }}
 {%   endif -%}
 {% endfor -%}
 {{ root_common.sla(name) }}
-{% for k, v in config.iteritems() -%}
+{% for k, v in six.iteritems(config) -%}
 config-{{ k }} = {{ dumps(v) }}
 {% endfor -%}
-{% for k, v in key_config.iteritems() -%}
+{% for k, v in six.iteritems(key_config) -%}
 config-{{ k }} = {{ '${' ~ v ~ '}' }}
 {% endfor -%}
 config-name = {{ name }}
@@ -76,6 +80,7 @@ recipe = slapos.cookbook:mkdirectory
 bin = ${buildout:directory}/bin
 service-on-watch = ${buildout:directory}/etc/service
 srv = ${buildout:directory}/srv
+tmp = ${buildout:directory}/tmp
 backup-caucased = ${:srv}/backup/caucased
 
 {% if not caucase_url -%}
@@ -180,7 +185,7 @@ connection-url = smtp://127.0.0.2:0/
 {% endfor -%}
 {% do assert(len(mountpoints) == len(zodb_dict)) -%}
 {% set neo = [] -%}
-{% for server_type, server_dict in storage_dict.iteritems() -%}
+{% for server_type, server_dict in six.iteritems(storage_dict) -%}
 {%   if server_type == 'neo' -%}
 {%     set ((name, server_dict),) = server_dict.items() -%}
 {%     do neo.append(server_dict.get('cluster')) -%}
@@ -199,10 +204,10 @@ connection-url = smtp://127.0.0.2:0/
 
 [request-zodb-base]
 config-zodb-dict = {{ dumps(zodb_dict) }}
-{% for server_type, server_dict in storage_dict.iteritems() -%}
+{% for server_type, server_dict in six.iteritems(storage_dict) -%}
 {%   if server_type == 'neo' -%}
 config-neo-cluster = ${publish-early:neo-cluster}
-config-neo-name = {{ server_dict.keys()[0] }}
+config-neo-name = {{ list(server_dict.keys())[0] }}
 config-neo-masters = ${publish-early:neo-masters}
 {%   else -%}
 config-zodb-zeo = ${request-zodb:connection-storage-dict}
@@ -266,14 +271,14 @@ config-wcfs_enable = {{ dumps(wcfs_enable) }}
 config-test-runner-configuration = {{ dumps(slapparameter_dict.get('test-runner', {})) }}
 software-type = zope
 
-{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
+{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout', 300) -%}
 {% set global_activity_timeout = slapparameter_dict.get('activity-timeout') -%}
 {% set zope_family_dict = {} -%}
 {% set zope_family_name_list = [] -%}
 {% set zope_backend_path_dict = {} -%}
 {% set ssl_authentication_dict = {} -%}
 {% set jupyter_zope_family_default = [] -%}
-{% for custom_name, zope_parameter_dict in zope_partition_dict.items() -%}
+{% for custom_name, zope_parameter_dict in six.iteritems(zope_partition_dict) -%}
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
 {%   set promise_software_url_section_name = 'promise-software-url' ~ partition_name -%}
@@ -298,12 +303,12 @@ config-name = {{ dumps(custom_name) }}
 config-instance-count = {{ dumps(zope_parameter_dict['instance-count']) }}
 config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
 config-thread-amount = {{ dumps(zope_parameter_dict['thread-amount']) }}
-config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
+config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 1)) }}
 config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
 config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
-config-with-max-rlimit-nofile = {{ dumps(slapparameter_dict.get('with-max-rlimit-nofile', false)) }}
+config-with-max-rlimit-nofile = {{ dumps(with_max_rlimit_nofile_enable) }}
 {# BBB: zope_parameter_dict used to contain 'webdav', so fallback to it -#}
 config-webdav = {{ dumps(current_zope_family_override_dict.get('webdav', zope_parameter_dict.get('webdav', False))) }}
 config-xml-rpc = {{ dumps(current_zope_family_override_dict.get('xml-rpc', False)) }}
@@ -503,7 +508,7 @@ recipe = slapos.cookbook:publish.serialised
 Pick any published hosts-dict, they are expected to be identical - and there is
 no way to check here.
 -#}
-hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-dict}' }}
+hosts-dict = {{ '${' ~ next(iter(zope_address_list_id_dict)) ~ ':connection-hosts-dict}' }}
 {% endif -%}
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}

software/slapos-master/software-py3.cfg

+[buildout]
+extends = ../wendelin/software-py3.cfg
+  base.cfg
+  buildout.hash.cfg
+
+[erp5-defaults]
+# wcfs dont work in python3 for now.
+wcfs-enable-default = false
+
+[erp5]
+branch = erp5-vifib-py3
+
+[vifib]
+branch = zope4py3
+

software/slapos-master/software-py3.cfg.json

+{
+  "name": "SlapOS Master",
+  "description": "SlapOS Master",
+  "serialisation": "json-in-xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "software-type": "default",
+      "request": "../erp5/instance-erp5-input-schema.json",
+      "response": "../erp5/instance-erp5-output-schema.json",
+      "index": 0
+    }
+  }
+}

software/slapos-master/software.cfg

@@ -2,64 +2,4 @@
 extends =
   ../../software/wendelin/software.cfg
   buildout.hash.cfg
-
-[erp5-defaults]
-wcfs-enable-default = true
-
-[local-bt5-repository]
-# Same as bt5-repository, but only local repository.
-# Used to generate bt5lists.
-list += ${vifib:location}/master/bt5
-
-[default-bt5]
-list =
-  erp5_full_text_mroonga_catalog
-  slapos_configurator
-
-[erp5_repository_list]
-repository_id_list += vifib/master
-
-[erp5]
-branch = erp5-vifib
-
-[vifib]
-<= erp5
-repository = https://lab.nexedi.com/nexedi/slapos.core.git
-branch = master
-
-[slapos-bin]
-<= erp5
-repository = https://lab.nexedi.com/nexedi/slapos-bin.git
-branch = master
-
-[eggs]
-eggs +=
-  slapos.core
-
-dummy +=
-  ${vifib:location}
-  ${slapos-bin:location}
-
-extra-paths +=
-  ${vifib:location}/master
-  ${slapos-bin:location}
-
-
-### Overwrite recipes to introduce customized changes
-[download-base-part]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/${:filename}
-
-[template-erp5]
-< = download-base-part
-filename = instance-erp5.cfg.in
-
-[template-balancer]
-< = download-base-part
-filename = instance-balancer.cfg.in
-
-[template-apache-backend-conf]
-url = ${:_profile_base_location_}/${:filename}
-
-[template-haproxy-cfg]
-url = ${:_profile_base_location_}/${:filename}
+  base.cfg

software/wendelin/base.cfg

+[eggs]
+initialization =
+  import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
+extra-paths +=
+  ${wendelin:location}
+eggs +=
+  astor
+  ${scipy:egg}
+  ${msgpack-python:egg}
+  ${msgpack-numpy:egg}
+  ${ipython:egg}
+
+[generic_testrunner_init]
+recipe = slapos.recipe.build
+init =
+  import struct
+  gcc = self.buildout['gcc']
+  options['initialization'] %= (
+    gcc['prefix'] + '/lib%s/libstdc++.so' % (8 * struct.calcsize("P"))
+    ) if 'part' in gcc else ''
+initialization =
+  # The 4 lines below will replace the process with another one, with the proper
+  # LD_PRELOAD environment variable. This is necessary because LD_PRELOAD is only
+  # taken into account when the process starts. Modifying it in runtime doesn't
+  # work.
+  import os
+  if not os.getenv('LD_PRELOAD'):
+    os.environ['LD_PRELOAD'] = %r
+    os.execve(os.path.realpath(__file__), sys.argv, os.environ)
+
+[test_suite_runner]
+# we need to override the test suite runner to add our custom libstdc++ dynamic
+# library to the path
+initialization +=
+  ${generic_testrunner_init:initialization}
+
+[testrunner]
+# we need to override the test suite runner to add our custom libstdc++ dynamic
+# library to the path
+initialization +=
+  ${generic_testrunner_init:initialization}
+
+[erp5_repository_list]
+repository_id_list += wendelin
+
+[default-bt5]
+list =
+  erp5_full_text_mroonga_catalog
+  erp5_wendelin_configurator
+
+[local-bt5-repository]
+list += ${wendelin:location}/bt5
+
+# Jupyter is by default enabled in Wendelin
+# and also the soft limit of open file descriptors is set
+# to the hard limit.
+[erp5-defaults]
+jupyter-enable-default = true
+with-max-rlimit-nofile-enable-default = true
+
+[wendelin]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+repository = https://lab.nexedi.com/nexedi/wendelin.git
+branch = master
+

software/wendelin/software-py3.cfg

+[buildout]
+versions = versions
+extends =
+  ../../component/msgpack-python/buildout.cfg
+  ../../component/msgpack-numpy/buildout.cfg
+  ../../component/scipy/buildout.cfg
+  ../../software/erp5/software-py3.cfg
+  base.cfg
+
+parts +=
+  wendelin
+  scipy
+  msgpack-python
+  msgpack-numpy
+  ipython
+  jupyter
+  jupyter-notebook-initialized-scripts
+
+[wendelin]
+# Use specific branch until the changes
+# are merged.
+branch = zope4py3
+
+[versions]
+msgpack-numpy = 0.4.4.3

software/wendelin/software-py3.cfg.json

+{
+  "name": "Wendelin",
+  "description": "Wendelin",
+  "serialisation": "json-in-xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "software-type": "default",
+      "request": "../erp5/instance-erp5-input-schema.json",
+      "response": "../erp5/instance-erp5-output-schema.json",
+      "index": 0
+    }
+  }
+}

software/wendelin/software.cfg

@@ -5,6 +5,8 @@ extends =
   ../../component/msgpack-numpy/buildout.cfg
   ../../component/scipy/buildout.cfg
   ../../software/erp5/software.cfg
+  base.cfg
+
 parts +=
   wendelin
   scipy
@@ -14,71 +16,5 @@ parts +=
   jupyter
   jupyter-notebook-initialized-scripts
 
-[eggs]
-initialization =
-  import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
-extra-paths +=
-  ${wendelin:location}
-eggs +=
-  astor
-  ${scipy:egg}
-  ${msgpack-python:egg}
-  ${msgpack-numpy:egg}
-  ${ipython:egg}
-
-[generic_testrunner_init]
-recipe = slapos.recipe.build
-init =
-  import struct
-  gcc = self.buildout['gcc']
-  options['initialization'] %= (
-    gcc['prefix'] + '/lib%s/libstdc++.so' % (8 * struct.calcsize("P"))
-    ) if 'part' in gcc else ''
-initialization =
-  # The 4 lines below will replace the process with another one, with the proper
-  # LD_PRELOAD environment variable. This is necessary because LD_PRELOAD is only
-  # taken into account when the process starts. Modifying it in runtime doesn't
-  # work.
-  import os
-  if not os.getenv('LD_PRELOAD'):
-    os.environ['LD_PRELOAD'] = %r
-    os.execve(os.path.realpath(__file__), sys.argv, os.environ)
-
-[test_suite_runner]
-# we need to override the test suite runner to add our custom libstdc++ dynamic
-# library to the path
-initialization +=
-  ${generic_testrunner_init:initialization}
-
-[testrunner]
-# we need to override the test suite runner to add our custom libstdc++ dynamic
-# library to the path
-initialization +=
-  ${generic_testrunner_init:initialization}
-
-[erp5_repository_list]
-repository_id_list += wendelin
-
-[default-bt5]
-list =
-  erp5_full_text_mroonga_catalog
-  erp5_wendelin_configurator
-
-[local-bt5-repository]
-list += ${wendelin:location}/bt5
-
-# Jupyter is by default enabled in Wendelin
-# and also the soft limit of open file descriptors is set
-# to the hard limit.
-[erp5-defaults]
-jupyter-enable-default = true
-with-max-rlimit-nofile-enable-default = true
-
-[wendelin]
-recipe = slapos.recipe.build:gitclone
-git-executable = ${git:location}/bin/git
-repository = https://lab.nexedi.com/nexedi/wendelin.git
-branch = master
-
 [versions]
 msgpack-numpy = 0.4.4.3