Merge remote-tracking branch 'upstream/master' into zope4py2

component/ceph/buildout.cfg

@@ -35,6 +35,11 @@ recipe = slapos.recipe.build
 shared = true
 pyyaml = ${pyyaml-download:target}
 pyaml = ${pyaml-download:target}
+init =
+  # add the python executable in the options dict so that
+  # buildout signature changes if python executable changes
+  import sys
+  options['python-executable'] = sys.executable
 
 install =
   import os, sys

component/jupyter-py2/buildout.cfg

 [buildout]
-extends = 
+extends =
   buildout.hash.cfg
   ../../stack/slapos.cfg
   ../openssl/buildout.cfg
@@ -62,48 +62,26 @@ context =
 
 [versions]
 Pygments = 2.2.0
-astor = 0.5
-backports-abc = 0.5
-backports.shutil-get-terminal-size = 1.0.0
 ipykernel = 4.5.2
 ipython = 5.3.0
-ipython-genutils = 0.1.0
-ipywidgets = 6.0.0
 jupyter-client = 5.0.0
 jupyter-core = 4.3.0
-jupyterlab = 0.26.3
-jupyterlab-launcher = 0.3.1
-matplotlib = 2.1.2
 mistune = 0.7.3
 nbformat = 4.3.0
 notebook = 4.4.1
 prompt-toolkit = 1.0.13
-ptyprocess = 0.5.1
 pyzmq = 16.0.2
 scikit-learn = 0.18.1
-seaborn = 0.7.1
-simplegeneric = 0.8.1
 statsmodels = 0.8.0
 terminado = 0.6
 tornado = 4.4.2
-widgetsnbextension = 2.0.0
+traitlets = 4.3.3
 
 # nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
 nbconvert = 4.1.0
 pathlib2 = 2.2.1
 patsy = 0.4.1
 pexpect = 4.2.1
-pickleshare = 0.7.4
 scandir = 1.5
-singledispatch = 3.4.0.3
 wcwidth = 0.1.7
-jupyter = 1.0.0
 jupyter-console = 5.1.0
-qtconsole = 4.3.0
-et-xmlfile = 1.0.1
-h5py = 2.7.1
-mpmath = 1.0.0
-openpyxl = 2.5.2
-sympy = 1.1.1
-xlrd = 1.1.0
-jdcal = 1.4

component/jupyter/buildout.cfg

@@ -63,24 +63,24 @@ setup-eggs =
   ${numpy:egg}
   ${python-pyzmq:egg}
   ${ipython:egg}
-  
-scripts = 
+
+scripts =
   jupyter-kernelspec
   pythonjupyter
   jupyter
   jupyter-trust
-  jupyter-nbconvert
   jupyter-console
   jupyter-migrate
   jupyter-troubleshoot
   jupyter-run
-  
+
 [jupyter-notebook-initialized-scripts]
 recipe = zc.recipe.egg:scripts
 eggs = ${jupyter:eggs}
 environment = jupyter-env
 
-scripts = 
+scripts =
+  jupyter-nbconvert
   jupyter-nbextension
   jupyter-notebook
   jupyter-serverextension

component/python3/buildout.cfg

@@ -18,7 +18,7 @@ parts =
     python3
 
 [python3]
-<= python3.8
+<= python3.9
 
 [python3-common]
 recipe = slapos.recipe.cmmi

component/selenium/buildout.cfg

+[buildout]
+parts =
+  selenium
+
+[selenium]
+recipe = zc.recipe.egg
+egg = selenium
+# patch to support python3.9
+selenium-patches = https://github.com/SeleniumHQ/selenium/commit/ddd163b681776292a72c39352581cf5c9d4f88f4.patch#c801fade1cd2019c063a0f0ef7cca3fe
+selenium-patch-options = -p2
+
+[versions]
+selenium = 3.141.0+SlapOSPatched001

component/xorg/buildout.cfg

@@ -97,8 +97,8 @@ environment =
 [xcbproto]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://xcb.freedesktop.org/dist/xcb-proto-1.13.tar.bz2
-md5sum = abe9aa4886138150bbc04ae4f29b90e3
+url = https://xcb.freedesktop.org/dist/xcb-proto-1.15.1.tar.gz
+md5sum = 3ee98337cda244996fab03df47e09df8
 environment =
   PATH=${libxml2:location}/bin:%(PATH)s
   PYTHON=${buildout:executable}

software/beremiz-runtime/software.cfg

@@ -7,7 +7,7 @@ extends =
   ../../stack/monitor/buildout.cfg
   ../../stack/slapos.cfg
 
-parts = 
+parts =
   beremiz-source
   slapos-cookbook
   instance-profile
@@ -95,7 +95,6 @@ Automat = 0.3.0
 zope.interface = 4.4.2
 Nevow = 0.14.5
 PyHamcrest = 2.0.2
-Pygments = 2.9.0
 Pyro = 3.16
 bitarray = 2.1.3
 constantly = 15.1.0
@@ -103,7 +102,6 @@ future = 0.18.2
 hyperlink = 21.0.0
 incremental = 21.3.0
 pathlib = 1.0.1
-prompt-toolkit = 3.0.19
 zeroconf-py2compat = 0.19.10
 
 # Required by:

software/erp5/upgrade_test/test.py

@@ -46,7 +46,7 @@ from slapos.testing.testcase import (
   makeModuleSetUpAndTestCaseClass,
 )
 
-old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.7/software/erp5/software.cfg'
+old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.8/software/erp5/software.cfg'
 new_software_release_url = os.path.abspath(
   os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
 

software/jstestnode/software.cfg

@@ -13,6 +13,7 @@ extends =
   ../../component/nginx/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/curl/buildout.cfg
+  ../../component/selenium/buildout.cfg
   ./buildout.hash.cfg
 
 parts =
@@ -36,7 +37,7 @@ parts =
 recipe = zc.recipe.egg
 eggs =
   erp5.util
-  selenium
+  ${selenium:egg}
   certifi
   ${lxml-python:egg}
 interpreter = pythonwitheggs
@@ -126,6 +127,3 @@ output = ${buildout:directory}/template-nginx.cfg.in
 [template-runTestSuite]
 <= macro-template
 output = ${buildout:directory}/runTestSuite.in
-
-[versions]
-selenium = 3.141.0

software/jupyter/software.cfg

@@ -43,64 +43,3 @@ output = ${buildout:directory}/template.cfg
 
 [instance-jupyter]
 <= download-file-base
-
-[versions]
-Pygments = 2.7.2
-astor = 0.5
-async-generator = 1.10
-backports-abc = 0.5
-backports.shutil-get-terminal-size = 1.0.0
-bleach = 3.2.1
-defusedxml = 0.6.0
-entrypoints = 0.3
-ipykernel = 5.3.4:whl
-ipython = 5.3.0
-ipython-genutils = 0.1.0
-ipywidgets = 6.0.0
-jupyter-client = 6.1.7
-jupyter-core = 4.7.0
-jupyterlab = 0.26.3
-jupyterlab-launcher = 0.3.1
-jupyterlab-pygments = 0.1.2
-matplotlib = 2.1.2
-mistune = 0.8.4
-nest-asyncio = 1.4.3
-nbclient = 0.5.1
-nbformat = 5.0.8
-notebook = 6.1.5
-pandocfilters = 1.4.3
-prompt-toolkit = 1.0.13
-ptyprocess = 0.5.1
-pyzmq = 20.0.0
-scikit-learn = 0.20.4
-seaborn = 0.7.1
-simplegeneric = 0.8.1
-statsmodels = 0.11.1
-testpath = 0.4.4
-terminado = 0.9.1
-tornado = 6.1
-traitlets = 5.0.5
-webencodings = 0.5.1
-widgetsnbextension = 2.0.0
-Send2Trash = 1.5.0
-argon2-cffi = 20.1.0
-nbconvert = 6.0.7
-pathlib2 = 2.2.1
-patsy = 0.5.1
-pexpect = 4.8.0
-pickleshare = 0.7.4
-prometheus-client = 0.9.0
-scandir = 1.5
-pytz = 2020.4
-singledispatch = 3.4.0.3
-wcwidth = 0.1.7
-jupyter = 1.0.0
-jupyter-console = 5.1.0
-qtconsole = 4.3.0
-et-xmlfile = 1.0.1
-h5py = 2.7.1
-mpmath = 1.0.0
-openpyxl = 2.5.2
-sympy = 1.1.1
-xlrd = 1.1.0
-jdcal = 1.4

software/jupyter/test/test.py

@@ -31,6 +31,7 @@ import json
 import os
 import requests
 import sqlite3
+import subprocess
 
 from slapos.proxy.db_version import DB_VERSION
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -268,3 +269,67 @@ class TestJupyterCustomAdditional(SelectMixin, InstanceTestCase):
 
     # clean up the fake master
     r.destroyed()
+
+
+class TestIPython(InstanceTestCase):
+
+  converted_notebook = 'test.nbconvert.ipynb'
+  notebook_filename = 'test.ipynb'
+  test_sentence = 'test'
+
+  def setUp(self):
+    super().setUp()
+    notebook_source = {
+      "cells": [
+        {
+        "cell_type": "code",
+        "execution_count": None,
+        "metadata": {},
+        "outputs": [],
+        "source": [
+          "import sys\n",
+          "print('" + self.test_sentence + "')"
+        ]
+        }
+      ],
+      "metadata": {},
+      "nbformat": 4,
+      "nbformat_minor": 4
+    }
+    with open(self.notebook_filename, 'w') as notebook:
+      notebook.write(json.dumps(notebook_source))
+
+  def tearDown(self):
+    os.remove(self.notebook_filename)
+    if os.path.exists(self.converted_notebook):
+      os.remove(self.converted_notebook)
+    super().tearDown()
+
+  def test(self):
+    conversion_output = subprocess.check_output([
+      os.path.join(
+        self.computer_partition_root_path,
+        'software_release',
+        'bin',
+        'jupyter-nbconvert',
+      ),
+      '--execute',
+      '--to',
+      'notebook',
+      self.notebook_filename,
+    ], stderr=subprocess.STDOUT, text=True)
+    self.assertIn(
+      '[NbConvertApp] Converting notebook %s to notebook' % self.notebook_filename,
+      conversion_output,
+    )
+    self.assertRegex(
+      conversion_output,
+      r'\[NbConvertApp\] Writing \d+ bytes to %s' % self.converted_notebook
+    )
+
+    self.assertTrue(os.path.exists(self.converted_notebook))
+    with open(self.converted_notebook) as json_result:
+      self.assertEqual(
+        json.loads(json_result.read())['cells'][0]['outputs'][0]['text'][0],
+        self.test_sentence + '\n',
+      )

software/slapos-sr-testing/software.cfg

@@ -13,6 +13,7 @@ extends =
   ../../component/python-mysqlclient/buildout.cfg
   ../../component/python-pynacl/buildout.cfg
   ../../component/python-backports-lzma/buildout.cfg
+  ../../component/selenium/buildout.cfg
 
   ../../stack/slapos.cfg
   ../../stack/nxdtest.cfg
@@ -296,6 +297,7 @@ eggs +=
   ${backports.lzma:egg}
   ${bcrypt:egg}
   ${psycopg2:egg}
+  ${selenium:egg}
   slapos.libnetworkcache
   supervisor
   ${slapos.cookbook-setup:egg}
@@ -443,7 +445,6 @@ image = 1.5.25
 plantuml = 0.3.0:whl
 pysftp = 0.2.9
 requests-toolbelt = 0.8.0
-selenium = 3.141.0
 testfixtures = 6.11.0
 mysqlclient = 2.1.1
 pexpect = 4.8.0

stack/erp5/buildout.cfg

@@ -64,6 +64,7 @@ extends =
   ../../component/bcrypt/buildout.cfg
   ../../component/python-pynacl/buildout.cfg
   ../../component/python-xmlsec/buildout.cfg
+  ../../component/selenium/buildout.cfg
   ../../stack/caucase/buildout.cfg
   ../../software/neoppod/software-common.cfg
 # keep neoppod extends last
@@ -610,7 +611,7 @@ eggs = ${neoppod:eggs}
   pycountry
   xfw
   jsonschema
-  selenium
+  ${selenium:egg}
   pytesseract
   decorator
   networkx
@@ -853,7 +854,6 @@ uuid = 1.30
 validictory = 1.1.0
 xfw = 0.10
 xupdate-processor = 0.5
-selenium = 3.14.1
 scikit-image = 0.14.0
 PyWavelets = 0.5.2
 networkx = 2.1

stack/slapos.cfg

@@ -137,16 +137,23 @@ zc.buildout = 2.7.1+slapos019
 zc.recipe.egg = 2.0.3+slapos003
 
 apache-libcloud = 2.4.0
+argon2-cffi = 20.1.0
 asn1crypto = 1.3.0
+astor = 0.5
+async-generator = 1.10
 atomicwrites = 1.4.0
 atomize = 0.2.0
 attrs = 22.1.0
+backcall = 0.2.0
+backports-abc = 0.5
 backports.functools-lru-cache = 1.6.1:whl
 backports.lzma = 0.0.14
+backports.shutil-get-terminal-size = 1.0.0
 bcrypt = 3.1.4
+bleach = 5.0.1
 CacheControl = 0.12.6:whl
 certifi = 2022.6.15
-cffi = 1.14.0
+cffi = 1.15.0
 chardet = 3.0.4
 charset-normalizer = 2.1.1
 click = 8.1.3
@@ -158,13 +165,20 @@ configparser = 4.0.2:whl
 contextlib2 = 0.6.0.post1
 croniter = 0.3.25
 cryptography = 3.3.2
+dataclasses = 0.8
 dateparser = 0.7.6
 decorator = 4.3.0
+defusedxml = 0.6.0
 distro = 1.7.0
 dnspython = 1.16.0
+entrypoints = 0.3
 enum34 = 1.1.10
 erp5.util = 0.4.74
-feedparser = 5.2.1
+et-xmlfile = 1.0.1
+# need wheel because there is no setup.py
+# (see https://erp5js.nexedi.net/#/bug_module/20221102-1C1B293)
+exceptiongroup = 1.0.0:whl
+feedparser = 6.0.10
 Flask = 1.1.2
 funcsigs = 1.0.2
 functools32 = 3.2.3.post2
@@ -173,59 +187,102 @@ geventmp = 0.0.1
 gitdb2 = 2.0.5
 GitPython = 2.1.11
 greenlet = 0.4.17
+h5py = 2.7.1
+h11 = 0.14.0
 idna = 2.9
 igmp = 1.0.4
 Importing = 1.10
 importlib-metadata = 1.7.0:whl
 inotify-simple = 1.1.1
 ipaddress = 1.0.23
+ipykernel = 5.3.4:whl
+ipython = 7.16.3
+ipython-genutils = 0.1.0
+ipywidgets = 6.0.0
 itsdangerous = 0.24
+jdcal = 1.4
+jedi = 0.17.2
 Jinja2 = 2.11.3
 jsonschema = 3.0.2:whl
+jupyter = 1.0.0
+jupyter-client = 7.3.1
+jupyter-console = 6.4.4
+jupyter-core = 4.9.2
+jupyterlab = 0.26.3
+jupyterlab-launcher = 0.3.1
+jupyterlab-pygments = 0.1.2
 lock-file = 2.0
 lockfile = 0.12.2:whl
 lxml = 4.9.1
 MarkupSafe = 2.0.1
+matplotlib = 2.1.2
 meld3 = 1.0.2
+mistune = 0.8.4
 mock = 3.0.5
 more-itertools = 5.0.0
+mpmath = 1.0.0
 msgpack = 0.6.2
+nbclient = 0.5.1
+nbconvert = 6.0.7
+nbformat = 5.0.8
+nest-asyncio = 1.5.6
 netaddr = 0.7.19
 netifaces = 0.10.7
+notebook = 6.1.5
+openpyxl = 2.5.2
+outcome = 1.2.0
 packaging = 16.8
+pandocfilters = 1.4.3
 paramiko = 2.11.0
+parso = 0.7.1
 passlib = 1.7.1
 pathlib2 = 2.3.5
-pbr = 2.0.0
+patsy = 0.5.1
+pbr = 5.9.0
+pexpect = 4.8.0
+pickleshare = 0.7.4
 pim-dm = 1.4.0nxd001
 pkgconfig = 1.5.1
 plone.recipe.command = 1.1
 pluggy = 0.13.1:whl
 ply = 3.11
 prettytable = 0.7.2
+prometheus-client = 0.9.0
+prompt-toolkit = 3.0.19
 psutil = 5.8.0
+ptyprocess = 0.5.1
 py = 1.11.0:whl
 py-mld = 1.0.3
 pyasn1 = 0.4.5
 pycparser = 2.20
 pycurl = 7.43.0
+Pygments = 2.9.0
 PyNaCl = 1.3.0
 pyOpenSSL = 19.1.0
 pyparsing = 3.0.9:whl
 pyroute2 = 0.6.9
 pyrsistent = 0.18.1
 PyRSS2Gen = 1.1
+PySocks = 1.7.1
 pytest-runner = 5.2:whl
 python-dateutil = 2.8.2:whl
 pytz = 2022.2.1
 PyYAML = 5.4.1
+pyzmq = 22.3.0
+qtconsole = 4.3.0
 regex = 2020.9.27
 requests = 2.28.1
 rpdb = 0.1.5
 rubygemsrecipe  = 0.4.3
 scandir = 1.10.0
+scikit-learn = 0.20.4
+seaborn = 0.7.1
+Send2Trash = 1.5.0
 setproctitle = 1.1.10
 setuptools-dso = 1.7
+sgmllib3k = 1.0.0
+simplegeneric = 0.8.1
+singledispatch = 3.4.0.3
 six = 1.16.0
 slapos.cookbook = 1.0.297
 slapos.core = 1.8.5
@@ -236,19 +293,32 @@ slapos.recipe.build = 0.56
 slapos.recipe.cmmi = 0.19
 slapos.recipe.template = 5.0
 slapos.toolbox = 0.128
+statsmodels = 0.11.1
 smmap2 = 2.0.5
+sniffio = 1.3.0
+sortedcontainers = 2.4.0
 stevedore = 1.21.0:whl
 subprocess32 = 3.5.4
 supervisor = 4.1.0
-traitlets = 4.3.3
+sympy = 1.1.1
+terminado = 0.9.1
+testpath = 0.4.4
+tornado = 6.1
+traitlets = 5.0.5
+trio = 0.22.0
+trio-websocket = 0.9.2
 tzlocal = 1.5.1
 unicodecsv = 0.14.1
 uritemplate = 3.0.0
 urllib3 = 1.26.12
 wcwidth = 0.2.5
+webencodings = 0.5.1
 Werkzeug = 2.0.2
 wheel = 0.35.1:whl
+widgetsnbextension = 2.0.0
+wsproto = 1.2.0
 xml-marshaller = 1.0.2
+xlrd = 1.1.0
 zc.lockfile = 1.4
 ZConfig = 3.6.1
 zdaemon = 4.2.0
@@ -264,6 +334,7 @@ certifi = 2020.4.5.1
 charset-normalizer = 2.0.12
 click = 6.7
 distro = 1.6.0
+feedparser = 5.2.1
 pyparsing = 2.2.0
 pyrsistent = 0.16.1
 requests = 2.27.1