ERP5: add support for Python 3

software/erp5/test/test/test_erp5.py

@@ -28,7 +28,7 @@
 import os
 import json
 import glob
-import urlparse
+from six.moves.urllib.parse import urljoin, urlparse
 import socket
 import time
 
@@ -70,14 +70,14 @@ class TestPublishedURLIsReachableMixin(object):
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
+      urljoin(param_dict['family-default-v6'], param_dict['site-id']))
 
   def test_published_family_default_v4_is_reachable(self):
     """Tests the IPv4 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
+      urljoin(param_dict['family-default'], param_dict['site-id']))
 
 
 class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
@@ -122,7 +122,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
     }
 
   def checkValidHTTPSURL(self, url):
-    parsed = urlparse.urlparse(url)
+    parsed = urlparse(url)
     self.assertEqual(parsed.scheme, 'https')
     self.assertTrue(parsed.hostname)
     self.assertTrue(parsed.port)

software/neoppod/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 80599fcc6e5d07270d7900aebfd62139
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = c03fbfc9df9edc1ef60be970e0627c5e
+md5sum = 5d609b0384bf6698b09871c2954c05e6
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
@@ -38,7 +38,7 @@ md5sum = 9f6f8f2b5f4cb0d97d50ffc1d3837e2f
 
 [template-neo]
 filename = instance.cfg.in
-md5sum = 9e63e16eda75e73ad4ffb50afde0505d
+md5sum = 3c7770c03c51ee12d5a94f0924e98990
 
 [cluster]
 filename = cluster.cfg.in

software/neoppod/instance.cfg.in

@@ -9,7 +9,7 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}.cfg
 template = {{ cluster }}
 extra-context =
     import re re
-    import urlparse urlparse
+    import urllib_parse six.moves.urllib.parse
 import-list =
     rawfile root_common {{ root_common }}
 

software/neoppod/root-common.cfg.in

@@ -2,13 +2,13 @@
 {% set part_list = [] -%}
 
 {% set sla_dict = {} -%}
-{% for sla, ref_list in slapparameter_dict.get('sla-dict', {}).iteritems() -%}
+{% for sla, ref_list in six.iteritems(slapparameter_dict.get('sla-dict', {})) -%}
 {%   do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
 {% endfor -%}
 
 {% macro sla(name, required=False) -%}
 {% if required or name in sla_dict -%}
-{%  for k, (v,) in urlparse.parse_qs(sla_dict.pop(name), strict_parsing=1).iteritems() -%}
+{%  for k, (v,) in six.iteritems(urllib_parse.parse_qs(sla_dict.pop(name), strict_parsing=1)) -%}
 sla-{{ k }} = {{ v }}
 {%  endfor -%}
 {% else -%}
@@ -112,7 +112,7 @@ return =
 config-monitor-passwd = ${monitor-htpasswd:passwd}
 config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
 {%-   endif %}
-{%-   for k, v in node.iteritems() %}
+{%-   for k, v in six.iteritems(node) %}
 config-{{ k }} = {{ dumps(v) }}
 {%-   endfor %}
 {{ sla(section_id) }}

stack/erp5/buildout.hash.cfg

@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c
 
 [template-zope-conf]
 filename = zope.conf.in
-md5sum = 762897486b1e7e28b614224a9a577125
+md5sum = 1e0b8429a985a0f945496840d8320b17
 
 [site-zcml]
 filename = site.zcml
@@ -66,11 +66,11 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template-run-zelenium]
 filename = run-zelenium-test.py.in
-md5sum = cc19560b9400cecbd23064d55c501eec
+md5sum = 6afc237458fcc0538dd7971c2e90aa96
 
 [template]
 filename = instance.cfg.in
-md5sum = 520b6bf3461dddc9c8b862e50b14465d
+md5sum = 4caf189efc2a71947486a345a9dfeb78
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -78,19 +78,19 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 939522bb26bf2109d7db818585a46f7a
+md5sum = 62c145894710749f804f6f9eb76e3faa
 
 [template-zeo]
 filename = instance-zeo.cfg.in
-md5sum = 0648e38bd5d3a15bb9f93264932740b9
+md5sum = 0ae838b3a9e709788cc6c13a02bb1763
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 8b4a15dca7e30ba5a792f1a9622216b0
+md5sum = 91caceba71514ad5794780e52b4d8933
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = cfc9c1ad40bf215771f4902b2ea3fe5b
+md5sum = b0bc3048907d0a0483ecc5c57808a675
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -43,7 +43,7 @@ mode = 644
 {% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #}
 {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
 {% for family_name, parameter_id_list in sorted(
-  slapparameter_dict['zope-family-dict'].iteritems()) -%}
+  six.iteritems(slapparameter_dict['zope-family-dict'])) -%}
 {%   set zope_family_address_list = [] -%}
 {%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
 {%   set has_webdav = [] -%}
@@ -139,7 +139,7 @@ cert = ${apache-conf-ssl:cert}
 {%- endif %}
 
 [apache-conf-parameter-dict]
-backend-list = {{ dumps(apache_dict.values()) }}
+backend-list = {{ dumps(list(apache_dict.values())) }}
 zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
 ip-list = {{ dumps(apache_ip_list) }}
 pid-file = ${directory:run}/apache.pid
@@ -185,7 +185,7 @@ input = inline:
 module = check_port_listening
 name = apache.py
 config-hostname = {{ ipv4 }}
-config-port = {{ apache_dict.values()[0][0] }}
+config-port = {{ list(apache_dict.values())[0][0] }}
 
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised

stack/erp5/instance-erp5.cfg.in

@@ -24,7 +24,7 @@
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
 {% set partition_thread_count_list = [] -%}
 {% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
-{% for zope_parameter_dict in zope_partition_dict.values() -%}
+{% for zope_parameter_dict in list(zope_partition_dict.values()) -%}
 {#   Apply some zope_parameter_dict default values, to avoid duplication. -#}
 {%   do zope_parameter_dict.setdefault('thread-amount', 4) -%}
 {%   do zope_parameter_dict.setdefault('instance-count', 1) -%}
@@ -44,7 +44,7 @@ config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-m
 name = {{ name }}
 software-type = {{ software_type }}
 return = {{ ' '.join(ret) }}
-{% for ret, publish in ret.iteritems() -%}
+{% for ret, publish in six.iteritems(ret) -%}
 {%   if publish -%}
 {%     do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}') %}
 {%   endif -%}
@@ -53,10 +53,10 @@ return = {{ ' '.join(ret) }}
 {%   endif -%}
 {% endfor -%}
 {{ root_common.sla(name) }}
-{% for k, v in config.iteritems() -%}
+{% for k, v in six.iteritems(config) -%}
 config-{{ k }} = {{ dumps(v) }}
 {% endfor -%}
-{% for k, v in key_config.iteritems() -%}
+{% for k, v in six.iteritems(key_config) -%}
 config-{{ k }} = {{ '${' ~ v ~ '}' }}
 {% endfor -%}
 config-name = {{ name }}
@@ -146,7 +146,7 @@ connection-url = smtp://127.0.0.2:0/
 {% endfor -%}
 {% do assert(len(mountpoints) == len(zodb_dict)) -%}
 {% set neo = [] -%}
-{% for server_type, server_dict in storage_dict.iteritems() -%}
+{% for server_type, server_dict in six.iteritems(storage_dict) -%}
 {%   if server_type == 'neo' -%}
 {%     set ((name, server_dict),) = server_dict.items() -%}
 {%     do neo.append(server_dict.get('cluster')) -%}
@@ -207,10 +207,10 @@ config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-z
 config-zodb-dict = {{ dumps(zodb_dict) }}
 config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
 config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
-{% for server_type, server_dict in storage_dict.iteritems() -%}
+{% for server_type, server_dict in six.iteritems(storage_dict) -%}
 {%   if server_type == 'neo' -%}
 config-neo-cluster = ${publish-early:neo-cluster}
-config-neo-name = {{ server_dict.keys()[0] }}
+config-neo-name = {{ list(server_dict)[0] }}
 config-neo-masters = ${publish-early:neo-masters}
 {%   else -%}
 config-zodb-zeo = ${request-zodb:connection-storage-dict}
@@ -372,7 +372,7 @@ recipe = slapos.cookbook:publish.serialised
 Pick any published hosts-dict, they are expected to be identical - and there is
 no way to check here.
 -#}
-hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-dict}' }}
+hosts-dict = {{ '${' ~ list(zope_address_list_id_dict)[0] ~ ':connection-hosts-dict}' }}
 {% endif -%}
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}

stack/erp5/instance-zeo.cfg.in

@@ -19,11 +19,11 @@ ip = {{ ipv4 }}
 
 {% set known_tid_storage_identifier_dict = {} -%}
 {% set zodb_dict = {} -%}
-{% for name, zodb in slapparameter_dict['zodb-dict'].iteritems() -%}
+{% for name, zodb in six.iteritems(slapparameter_dict['zodb-dict']) -%}
 {%   do zodb_dict.setdefault(zodb.get('family', 'default').lower(), []).append((name, zodb)) -%}
 {% endfor -%}
 {% set tidstorage_port = slapparameter_dict['tcpv4-port'] + len(zodb_dict) -%}
-{% for family, zodb in zodb_dict.iteritems() -%}
+{% for family, zodb in six.iteritems(zodb_dict) -%}
 {%   set storage_list = [] -%}
 {%   set current_port = next_port() -%}
 {%   set known_tid_storage_identifier_host = (ipv4, current_port), -%}
@@ -140,7 +140,7 @@ input = inline: #!/bin/sh
   zodb_backup_directory="{{ default_backup_path }}"
   repozo="${tidstorage:repozo-binary}"
   EXIT_CODE=0
-  {% for family, zodb in zodb_dict.iteritems() -%}
+  {% for family, zodb in six.iteritems(zodb_dict) -%}
   {%   for name, zodb in zodb -%}
   {%   set zeo_section_name = 'zeo-' ~ family %}
   storage_name="{{ name }}"

stack/erp5/instance-zope.cfg.in

@@ -30,14 +30,14 @@ partition. No more (undefined result), no less (IndexError).
     ('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
     ('erp5-smtp', slapparameter_dict['smtp-url']),
   ) -%}
-{%   set parsed_url = urlparse.urlparse(url) -%}
+{%   set parsed_url = urllib_parse.urlparse(url) -%}
 {%   do port_dict.__setitem__(alias, parsed_url.port)  -%}
 {%   do hosts_dict.__setitem__(alias, parsed_url.hostname)  -%}
 {%- endfor %}
 {% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%}
 {%   do hosts_dict.__setitem__(
        'erp5-catalog-' ~ i,
-       urlparse.urlparse(url).hostname,
+       urllib_parse.urlparse(url).hostname,
      ) -%}
 {%- endfor %}
 {% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
@@ -161,7 +161,7 @@ environment =
 command-line = '{{ parameter_dict['userhosts'] }}' ${:wrapped-command-line}
 
 {# Hack to deploy SSL certs via instance parameters -#}
-{% for zodb in zodb_dict.itervalues() -%}
+{% for zodb in six.itervalues(zodb_dict) -%}
 {%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
 {%   if zodb['type'] == 'neo' and storage_dict.get('ssl', 1) -%}
 {%     for k, v in (('_ca', 'ca.crt'),
@@ -209,7 +209,7 @@ site-id = {{ site_id }}
 {% if site_id -%}
 mysql-url = {{ slapparameter_dict['mysql-url-list'][0] }}
 inituser = {{ slapparameter_dict['inituser-login'] }}
-{%  set mysql = urlparse.urlsplit(slapparameter_dict['mysql-url-list'][0]) -%}
+{%  set mysql = urllib_parse.urlsplit(slapparameter_dict['mysql-url-list'][0]) -%}
 {%  set mysql_db = mysql.path.split('/')[1] -%}
 sql-connection-string = {{ '%s@erp5-catalog-0:%s %s %s' % (
     mysql_db, mysql.port, mysql.username, mysql.password) }}
@@ -220,13 +220,13 @@ home = ${buildout:directory}
 # We only want to change the hostname to 'erp5-cloudooo' if we use the internal
 # cloudooo. We plan to remove the ability to have an internal one, so this
 # heuristic is enough.
-{%  set cloudooo = urlparse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
+{%  set cloudooo = urllib_parse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
 cloudooo-url = {{ (cloudooo if cloudooo.port == None else
   cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
 
 {% endif -%}
 {% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
-{% for name, zodb in zodb_dict.iteritems() -%}
+{% for name, zodb in six.iteritems(zodb_dict) -%}
 {%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
 {%   if zodb['type'] == 'zeo' -%}
 {%     do storage_dict.update(zeo_dict.get(name, ())) -%}
@@ -294,7 +294,7 @@ z2-log = ${directory:log}/{{ name }}-Z2.log
 node-id = {{ dumps(node_id_base ~ (node_id_index_format % index)) }}
 {% set log_list = [] -%}
 {% set import_set = set() -%}
-{% for db_name, zodb in zodb_dict.iteritems() -%}
+{% for db_name, zodb in six.iteritems(zodb_dict) -%}
 {%   do zodb.setdefault('pool-size', thread_amount) -%}
 {%   if zodb['type'] == 'neo' -%}
 {%     do import_set.add('neo.client') -%}
@@ -440,7 +440,7 @@ wrapper-path = ${buildout:bin-directory}/runTestSuite
 
 {%   set connection_string_list = [] -%}
 {%   for url in slapparameter_dict['mysql-test-url-list'] -%}
-{%     set parsed_url = urlparse.urlparse(url) -%}
+{%     set parsed_url = urllib_parse.urlparse(url) -%}
 {%     do connection_string_list.append(
          '%s@%s:%s %s %s' % (
            parsed_url.path.lstrip('/'),

stack/erp5/instance.cfg.in

@@ -47,7 +47,7 @@ extra-context =
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key openssl_location :openssl-location
     import re re
-    import urlparse urlparse
+    import urllib_parse six.moves.urllib.parse
 import-list =
     file root_common context:root-common
     file caucase context:caucase-jinja2-library
@@ -114,7 +114,7 @@ extra-context =
     key buildout_directory buildout:directory
     key root_common context:root-common
     section parameter_dict dynamic-template-zope-parameters
-    import urlparse urlparse
+    import urllib_parse six.moves.urllib.parse
     import hashlib hashlib
     import itertools itertools
 

stack/erp5/run-zelenium-test.py.in

@@ -181,7 +181,7 @@ def main():
 
   try:
     test_result = tool.createTestResult(revision = revision,
-                                        test_name_list = test_line_dict.keys(),
+                                        test_name_list = list(test_line_dict),
                                         node_title = args.test_node_title,
                                         test_title = test_suite_title,
                                         project_title = args.project_title)

stack/erp5/zope.conf.in

@@ -144,17 +144,17 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
 %import {{ m }}
 {% endfor -%}
 {% set type_dict = {'neo': 'NEOStorage', 'zeo': 'zeoclient'} %}
-{% for name, zodb_dict in parameter_dict['zodb-dict'].iteritems() %}
+{% for name, zodb_dict in six.iteritems(parameter_dict['zodb-dict']) %}
 <zodb_db {{ name }}>
 {%- set storage_type = type_dict[zodb_dict.pop('type')] %}
 {%- set storage_dict = zodb_dict.pop('storage-dict') %}
 {%- do root_common.apply_overrides(zodb_dict, node_id) %}
-{%- for key, value in zodb_dict.iteritems() %}
+{%- for key, value in six.iteritems(zodb_dict) %}
   {{ key }} {{ value }}
 {%- endfor %}
   <{{ storage_type }}>
 {%- do root_common.apply_overrides(storage_dict, node_id) %}
-{%- for key, value in storage_dict.iteritems() %}
+{%- for key, value in six.iteritems(storage_dict) %}
     {{ key }} {{ value }}
 {%- endfor %}
   </{{ storage_type }}>