ERP5 py3: WIP changes

component/pylint/buildout.cfg

@@ -12,6 +12,8 @@ patches =
   ${:_profile_base_location_}/astroid-six_moves_import_error.patch#377beb0c50f52b9608bb6be7bf93096e
 patch-options = -p1
 patch-binary = ${patch:location}/bin/patch
+
+[astroid:python3]
 setup-eggs = ${lazy-object-proxy:egg}
 
 [lazy-object-proxy]
@@ -40,3 +42,15 @@ patches =
   ${:_profile_base_location_}/pylint-redefining-builtins-modules.patch#043defc6e9002ac48b40e078797d4d17
 patch-options = -p1
 patch-binary = ${patch:location}/bin/patch
+
+[pylint:python3]
+recipe = zc.recipe.egg
+patches =
+
+[mccabe:python3]
+recipe = zc.recipe.egg
+patches =
+setup-eggs =
+
+[astroid:python3]
+recipe = zc.recipe.egg

software/neoppod/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = c91b5540f94ce76af31f84584df7a3ef
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
-md5sum = b6e1ccb1d90160110202e5111eec2afa
+md5sum = a0ec1dce4c7a237fbeef3f8aee62e55a
 
 [instance-neo-master]
 filename = instance-neo-master.cfg.in
@@ -34,7 +34,7 @@ md5sum = fda911d5ef9efee365f1b0ff9843a50b
 
 [template-neo-my-cnf]
 filename = my.cnf.in
-md5sum = 56ea8f452d9e1526157ab9d03e631e1a
+md5sum = 3ae93702f3890a504cc8a93eb5ad52bc
 
 [template-neo]
 filename = instance.cfg.in

software/neoppod/instance-neo-admin.cfg.in

@@ -45,7 +45,7 @@ ssl = {{ dumps(bool(slapparameter_dict['ssl'])) }}
 cluster = {{ dumps(slapparameter_dict['cluster']) }}
 masters = {{ dumps(slapparameter_dict['masters']) }}
 extra-options =
-{%- for k, v in monitor_dict.iteritems() %}
+{%- for k, v in six.iteritems(monitor_dict) %}
 {%-   if k == 'backup' %}
 {%-     set k = 'monitor-backup' %}
 {%-   endif %}

software/neoppod/my.cnf.in

@@ -45,7 +45,7 @@ innodb_locks_unsafe_for_binlog = 1
 {{x}}sync_frm = 0
 
 # Extra parameters.
-{%- for k, v in extra_dict.iteritems() %}
+{%- for k, v in six.iteritems(extra_dict) %}
 {%- do assert('-' not in k) %}
 {{ k }} = {{ v }}
 {%- endfor %}

software/neoppod/software-common.cfg

@@ -130,8 +130,11 @@ inline =
 [versions]
 coverage = 5.5
 ecdsa = 0.13
-mysqlclient = 1.3.12
+mysqlclient = 2.0.1
 PyMySQL = 0.10.1
 pycrypto = 2.6.1
 cython-zstd = 0.2
 funcsigs = 1.0.2
+
+[versions:python2]
+mysqlclient = 1.3.12

software/slapos-sr-testing/software.cfg

@@ -496,7 +496,6 @@ pysftp = 0.2.9
 requests-toolbelt = 0.8.0
 testfixtures = 6.11.0
 mysqlclient = 2.1.1
-pexpect = 4.8.0
 ptyprocess = 0.6.0
 paho-mqtt = 1.5.0
 pcpp = 1.30

stack/erp5/buildout-py3.cfg

@@ -4,12 +4,6 @@ extends =
   ../../component/cython/buildout.cfg
   buildout.cfg
 
-parts +=
-# Development eggs
-  slapos-cookbook-develop
-  slapos-cookbook
-  PyStemmer
-  cython
 
 parts -=
 # XXX: The following parts have no support for Python 3. Ignore for now.
@@ -24,7 +18,6 @@ parts -=
   jupyter
   matplotlibrc
   h5py
-  custom-js
 
 find-links +=
   http://www.nexedi.org/static/packages/source/
@@ -59,6 +52,7 @@ recipe =
 recipe =
 [ocropy]
 recipe =
+
 [hookbox]
 recipe =
 [scipy]
@@ -83,30 +77,10 @@ include-dirs +=
   ${cyrus-sasl:location}/include
 
 [eggs]
-eggs +=
-## Python 3 variant
-  SOAPpy-py3
-  suds-py3
+
 eggs -=
-## Python 2 only
-  ZServer
-  Products.DCWorkflowGraph
-  uuid
-# Probably not worth the effort (used for backuping multi-ZODB instances and has
-# been discouraged for many years over NEO)
-  Products.TIDStorage
-## Python < 3.5
-  typing
-## tracemalloc is now part of Python 3.4!
-  pytracemalloc
-## XXX: The following eggs have no support for Python 3. Ignore for now.
-  ocropy
   ${ocropy:egg}
-  PyXML
-  threadframe
-  suds
-  SOAPpy
-  fpconst
+#  fpconst
   ${scipy:egg}
   ${scikit-learn:egg}
   ${scikit-image:egg}
@@ -116,7 +90,6 @@ eggs -=
   ${statsmodels:egg}
   astor
   jupyter
-# explicit eggs to generate scripts from them
   jupyter_client
   jupyter_console
   jupyter_core
@@ -125,86 +98,57 @@ eggs -=
   notebook
 # additional eggs
   astor
-  ${matplotlib:egg}
-  ${scipy:egg}
-  ${scikit-learn:egg}
   ${seaborn:egg}
   ${statsmodels:egg}
-  patsy
-  ${h5py:egg}
-  openpyxl
-  sympy
-  xlrd
 # additional eggs for jupyterlab
   jupyterlab
   jupyterlab-launcher
   pytesseract
 
-[astroid]
-patches =
-
-[pylint]
-patches=
-setup-eggs = ${mccabe:egg}
 
 
 [versions]
-# zc.recipe.egg 2.0.7 (in versions.cfg) requires zc.buildout>=2.12.0, so let's
-# revert to the version contained in stack/slapos.cfg
-zc.recipe.egg = 2.0.3+slapos003
-
 pyasn1-modules = 0.2.8
 pyasn1 = 0.4.8
 
-## Version up to support Python 3.8
-numpy = 1.16.6
-pandas = 0.24.2
-google-api-python-client = 2.41.0
-zope.app.appsetup = 4.2.0
-zope.app.publication = 4.5
-# google-api-python-client version bump
-uritemplate = 3.0.1
+google-api-python-client = 2.81.0
 google-api-core = 2.7.1
 google-auth-httplib2 = 0.1.0
 google-auth = 2.6.0
 httplib2 = 0.15.0
 protobuf = 3.19.4
 googleapis-common-protos = 1.55.0
-cachetools = 5.0.0
-## XXX: No version of scipy is both compatible with Python 2.7 and 3.7
-scipy = 1.5.2
+scipy = 1.8.1
 #scikit-learn = 0.20.4
 statsmodels = 0.10.2
 pyzmq = 19.0.2
-PyStemmer = 2.0.1
 python-ldap = 3.1.0
-ipython = 7.31.1
-mccabe = 0.6
-setuptools-scm = 7.0.5
-packaging = 20.4
-typing-extensions = 4.3.0:whl
-tomli = 2.0.1:whl
-lazy-object-proxy = 1.4.3
-isort = 4.2.5
-astroid = 2.4.0
-wrapt = 1.11
-typed-ast = 1.4.0
-jedi = 0.16.0
-parso = 0.5.2
-mysqlclient = 2.0.1
-polib = 1.1.0
-# testrunner
-pexpect = 4.8.0
+#mysqlclient = 2.0.1
+#polib = 1.2.0
+
 matplotlib-inline = 0.1.3
-backcall = 0.2.0
-prompt-toolkit = 3.0.27
-# 2to3 fixers to port code to python3
-zope.fixers = 1.1.2
-SOAPpy-py3 = 0.52.26
+#prompt-toolkit = 3.0.27
+
 defusedxml = 0.7.1
 suds-py3 = 1.4.5.0
 wstools-py3 = 0.54.4
-toml = 0.10.2
 soupsieve = 2.3.2.post1:whl
 
-pylint = 2.6.0
+# WIP pylint
+[versions]
+pylint = 2.17.0:whl
+mccabe = 0.7.0:whl
+tomli = 2.0.1:whl
+lazy-object-proxy = 1.4.3
+isort = 4.2.5
+astroid = 2.15.0:whl
+wrapt = 1.11
+typed-ast = 1.4.0
+tomlkit = 0.11.6
+platformdirs = 3.1.1:whl
+dill = 0.3.6:whl
+
+[erp5]
+repository = https://lab.nexedi.com/nexedi/erp5.git
+branch = zope4py3
+develop = true

stack/erp5/buildout.cfg

@@ -347,7 +347,10 @@ with-max-rlimit-nofile-enable-default = false
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/erp5.git
 branch = master
+develop = true
 git-executable = ${git:location}/bin/git
+[erp5:python3]
+branch = zope4py3
 
 [testrunner]
 # XXX: Workaround for fact ERP5Type is not an distribution and does not
@@ -505,7 +508,6 @@ eggs +=
   ipython_genutils
   ipykernel
   ipywidgets
-  requests
 
 
 [egg-with-zope-proxy]
@@ -522,13 +524,14 @@ setup-eggs +=
 
 [eggs]
 <= neoppod
-eggs = ${neoppod:eggs}
+eggs =
+  ${erp5-eggs-python-version-dependent:eggs}
+  ${neoppod:eggs}
   ${caucase-eggs:eggs}
   ${wendelin.core:egg}
   ${numpy:egg}
   ${matplotlib:egg}
   ${lxml-python:egg}
-  ${ocropy:egg}
   ${pandas:egg}
   ${pillow-python:egg}
   ${python-ldap-python:egg}
@@ -550,14 +553,12 @@ eggs = ${neoppod:eggs}
   astor
   APacheDEX
   Pympler
-  SOAPpy
   chardet
   collective.recipe.template
   erp5diff
   interval
   ipdb
   Jinja2
-  jsonschema
   mechanize
   oauthlib
   objgraph
@@ -573,7 +574,6 @@ eggs = ${neoppod:eggs}
   requests
   responses
   urlnorm
-  uuid
   xml_marshaller
   xupdate_processor
   feedparser
@@ -584,24 +584,17 @@ eggs = ${neoppod:eggs}
   qrcode
   spyne
   httplib2
-  suds
   pprofile
   pycountry
   xfw
-  jsonschema
   ${selenium:egg}
   pytesseract
   decorator
   networkx
-# Needed for checking ZODB Components source code
   ${astroid:egg}
   ${pylint:egg}
   jedi
   yapf
-  typing
-# Used for Python 2 only
-  ${pytracemalloc:egg}
-  xlrd
   pydot
 
 # Zope
@@ -614,16 +607,10 @@ eggs = ${neoppod:eggs}
 # for runzeo
   ${ZEO:egg}
 
-# Other Zope 2 packages
-  Products.PluggableAuthService
   Products.DCWorkflow
-
-# Other products
+  Products.PluggableAuthService
   Products.MimetypesRegistry
-  Products.TIDStorage
 
-# Currently forked in our repository
-#    Products.PortalTransforms
 # Dependency for our fork of PortalTransforms
   StructuredText
 
@@ -658,11 +645,9 @@ eggs = ${neoppod:eggs}
   docutils
   zLOG
   Products.ZSQLMethods
-  ZServer
   Products.ExternalMethod
   Products.SiteErrorLog
   tempstorage
-  Products.DCWorkflow
   Products.Sessions
   Products.ZODBMountPoint
   Record
@@ -708,6 +693,12 @@ python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/pyt
 python-magic-patch-options = -p1
 urlnorm-patches = ${:_profile_base_location_}/../../component/egg-patch/urlnorm/urlnorm-1.1.4-py3.patch#5ef268fb44cbc005b62140099c33b641
 urlnorm-patch-options = -p1
+Zope-patches =
+  ${:_profile_base_location_}/../../component/egg-patch/Zope/0001-WSGIPublisher-set-REMOTE_USER-even-in-case-of-error-.patch#a437f4da28975f94dd07db0b02954111
+  ${:_profile_base_location_}/../../component/egg-patch/Zope/0001-Fix-redirections-to-URLS-with-host-given-as-IP-litte.patch#093ad5755094d537c6a4deadc959ade0
+Zope-patch-options = -p1
+
+[eggs:python2]
 # backported security patches for waitress-1.4.4 from Debian 1.4.4-1.1+deb11u1 package.
 waitress-patches =
   ${:_profile_base_location_}/../../component/egg-patch/waitress/CVE-2022-24761-1.patch#a0508880f24662e48a20ce3bcbf440c2
@@ -717,10 +708,26 @@ waitress-patches =
   ${:_profile_base_location_}/../../component/egg-patch/waitress/CVE-2022-24761-5.patch#ad2765822397cd1e28e02a68a52d7768
   ${:_profile_base_location_}/../../component/egg-patch/waitress/CVE-2022-24761-6.patch#85fc9c4105eabee3ff71c800b2ddf63b
 waitress-patch-options = -p1
-Zope-patches =
-  ${:_profile_base_location_}/../../component/egg-patch/Zope/0001-WSGIPublisher-set-REMOTE_USER-even-in-case-of-error-.patch#a437f4da28975f94dd07db0b02954111
-  ${:_profile_base_location_}/../../component/egg-patch/Zope/0001-Fix-redirections-to-URLS-with-host-given-as-IP-litte.patch#093ad5755094d537c6a4deadc959ade0
-Zope-patch-options = -p1
+
+# python version dependent eggs
+[erp5-eggs-python-version-dependent:python2]
+eggs =
+  ${ocropy:egg}
+  ${pytracemalloc:egg}
+  Products.TIDStorage
+  pytracemalloc
+  SOAPpy
+  suds
+  typing
+  uuid
+  xlrd
+  ZServer
+
+[erp5-eggs-python-version-dependent:python3]
+eggs =
+  SOAPpy-py3
+  suds-py3
+
 
 # neoppod installs bin/coverage so we inject erp5 plugin here so that coverage script can use it in report
 [neoppod]
@@ -755,25 +762,22 @@ depends =
 Acquisition = 4.7+SlapOSPatched001
 DateTime = 4.9+SlapOSPatched004
 Products.DCWorkflow = 2.4.1+SlapOSPatched001
-ocropy = 1.0+SlapOSPatched001
 PyPDF2 = 1.26.0+SlapOSPatched002
 pysvn = 1.9.15+SlapOSPatched001
 python-ldap = 2.4.32+SlapOSPatched001
 python-magic = 0.4.12+SlapOSPatched001
-waitress = 1.4.4+SlapOSPatched006
 Zope = 4.8.9+SlapOSPatched002
+
 ## https://lab.nexedi.com/nexedi/slapos/merge_requests/648
 pylint = 1.4.4+SlapOSPatched002
 # astroid 1.4.1 breaks testDynamicClassGeneration
 astroid = 1.3.8+SlapOSPatched001
 
-# modified version that works fine for buildout installation
-SOAPpy = 0.12.0nxd001
-
 
 # Pinned versions
 alabaster = 0.7.12
 APacheDEX = 1.8
+arrow = 1.2.3
 Beaker = 1.11.0
 cloudpickle = 0.5.3
 cookies = 2.2.1
@@ -785,7 +789,7 @@ erp5diff = 0.8.1.9
 facebook-sdk = 2.0.0
 five.formlib = 1.0.4
 fpconst = 0.7.2
-future = 0.18.2
+fqdn = 1.5.1
 google-api-python-client = 1.6.1
 graphviz = 0.5.2
 haufe.requestmonitoring = 0.6.0
@@ -793,8 +797,8 @@ html5lib = 1.1
 huBarcode = 1.0.0
 interval = 1.0.0
 ipdb = 0.10.2
+isoduration = 20.11.0
 jdcal = 1.3
-jedi = 0.15.1
 jsonpickle = 0.9.6
 jsonpointer = 2.2
 logilab-common = 1.3.0
@@ -804,16 +808,13 @@ mpmath = 0.19
 munnel = 0.3
 networkx = 2.1
 nt-svcutils = 2.13.0
-numpy = 1.13.1
 oauth2client = 4.0.0
 oauthlib = 3.1.0
 objgraph = 3.1.0
 oic = 0.15.1
 olefile = 0.44
-openpyxl = 2.4.8
-parso = 0.5.1
 Pillow = 6.2.2
-polib = 1.0.8
+polib = 1.2.0
 pprofile = 2.0.4
 Products.BTreeFolder2 = 4.4+SlapOSPatched001
 Products.CMFCore = 2.7.0+SlapOSPatched001
@@ -832,7 +833,6 @@ Products.ZODBMountPoint = 1.3
 Products.ZSQLMethods = 3.16
 pyasn1-modules = 0.0.8
 pycountry = 17.1.8
-pycrypto = 2.6.1
 pycryptodomex = 3.10.1
 pydot = 1.4.2
 pyflakes = 1.5.0
@@ -848,25 +848,26 @@ pytracemalloc = 1.2
 PyWavelets = 0.5.2
 qrcode = 5.3
 responses = 0.10.6
+rfc3339-validator = 0.1.4
 rfc3987 = 1.3.8
 rsa = 3.4.2
 scikit-image = 0.14.0
 scipy = 0.19.0
-spyne = 2.12.14
+SOAPpy-py3 = 0.52.29
+spyne = 2.14.0
 strict-rfc3339 = 0.7
 StructuredText = 2.11.1
 suds = 0.4
 toolz = 0.9.0
 typing = 3.10.0.0
 unidiff = 0.5.5
+uri-template = 1.2.0
 urlnorm = 1.1.4+SlapOSPatched001
 uuid = 1.30
 validictory = 1.1.0
-webcolors = 1.10
-webencodings = 0.5.1
+webcolors = 1.12
 WebOb = 1.8.5
 WebTest = 2.0.33
-WSGIProxy2 = 0.4.6
 WSGIUtils = 0.7
 xfw = 0.10
 xupdate-processor = 0.5
@@ -888,4 +889,14 @@ zope.session = 4.5
 zope.testbrowser = 5.5.1
 
 [versions:python2]
+ipython = 5.3.0
+jedi = 0.15.1
+numpy = 1.13.1
+ocropy = 1.0+SlapOSPatched001
+openpyxl = 2.4.8
+parso = 0.5.1
+prompt-toolkit = 1.0.13
 PyStemmer = 1.3.0
+SOAPpy = 0.12.0nxd001
+waitress = 1.4.4+SlapOSPatched006
+webcolors = 1.10

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 55463b0abdbe0118ef1c27e6b71c3324
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = ae9c380ae04dde4f20e139c66ef7c22a
+md5sum = 456b57eda17628170ceea6b56469daa6
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -200,7 +200,7 @@ config-zodb-dict = {{ dumps(zodb_dict) }}
 {% for server_type, server_dict in six.iteritems(storage_dict) -%}
 {%   if server_type == 'neo' -%}
 config-neo-cluster = ${publish-early:neo-cluster}
-config-neo-name = {{ server_dict.keys()[0] }}
+config-neo-name = {{ list(server_dict.keys())[0] }}
 config-neo-masters = ${publish-early:neo-masters}
 {%   else -%}
 config-zodb-zeo = ${request-zodb:connection-storage-dict}