software/jupyter: move to python 3

See merge request !868

component/jupyter-py2/README.md

+This directory is only temporary (until ERP5 is Python3) and will not actively
+be maintained.
+
+It has to be removed once ERP5 uses `software/jupyter` or do not use jupyter at
+all.

component/jupyter-py2/buildout.cfg

+[buildout]
+extends = 
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../openssl/buildout.cfg
+  ../jupyter/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+parts +=
+  slapos-cookbook
+  jupyter
+  jupyter-notebook-initialized-scripts
+  instance-jupyter-notebook
+
+[gcc]
+# Always build GCC for Fortran (see openblas).
+max_version = 0
+
+[jupyter]
+python_executable = ${buildout:bin-directory}/${:interpreter}
+
+[download-file-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+mode = 0644
+
+[jupyter-notebook-config]
+<= download-file-base
+
+[jupyter-set-password]
+<= download-file-base
+
+[erp5-kernel]
+<= download-file-base
+
+[kernel-json]
+<= download-file-base
+
+[custom-js]
+<= download-file-base
+
+[instance-jupyter-notebook]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+rendered = ${buildout:directory}/template.cfg
+mode = 0644
+context =
+  key bin_directory buildout:bin-directory
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key openssl_output openssl-output:openssl
+  key python_executable jupyter:python_executable
+  key jupyter_config_location jupyter-notebook-config:location
+  key jupyter_config_filename jupyter-notebook-config:filename
+  key jupyter_set_password_location jupyter-set-password:location
+  key jupyter_set_password_filename jupyter-set-password:filename
+  key erp5_kernel_location erp5-kernel:location
+  key erp5_kernel_filename erp5-kernel:filename
+  key kernel_json_location kernel-json:location
+  key kernel_json_filename kernel-json:filename
+  key custom_js_location custom-js:location
+  key custom_js_filename custom-js:filename
+  key monitor_template_rendered buildout:directory
+
+[versions]
+Pygments = 2.2.0
+astor = 0.5
+backports-abc = 0.5
+backports.functools-lru-cache = 1.6.1
+backports.shutil-get-terminal-size = 1.0.0
+cycler = 0.10.0
+ipykernel = 4.5.2
+ipython = 5.3.0
+ipython-genutils = 0.1.0
+ipywidgets = 6.0.0
+jupyter-client = 5.0.0
+jupyter-core = 4.3.0
+jupyterlab = 0.26.3
+jupyterlab-launcher = 0.3.1
+matplotlib = 2.1.2
+mistune = 0.7.3
+nbformat = 4.3.0
+notebook = 4.4.1
+pandas = 0.19.2
+plone.recipe.command = 1.1
+prompt-toolkit = 1.0.13
+ptyprocess = 0.5.1
+pyzmq = 16.0.2
+scikit-learn = 0.18.1
+seaborn = 0.7.1
+simplegeneric = 0.8.1
+slapos.recipe.template = 4.4
+statsmodels = 0.8.0
+terminado = 0.6
+tornado = 4.4.2
+traitlets = 4.3.2
+widgetsnbextension = 2.0.0
+# numpy >= 1.13.1 is required for numpy.core.multiarray
+numpy = 1.13.1
+
+
+# Required by:
+# tornado==4.4.2
+certifi = 2020.6.20
+
+# Required by:
+# notebook==4.3.2
+# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
+nbconvert = 4.1.0
+
+# Required by:
+# ipython==5.3.0
+pathlib2 = 2.2.1
+
+# Required by:
+# statsmodels==0.8.0
+patsy = 0.4.1
+
+# Required by:
+# ipython==5.3.0
+pexpect = 4.2.1
+
+# Required by:
+# ipython==5.3.0
+pickleshare = 0.7.4
+
+# Required by:
+# matplotlib==2.1.2
+# pandas==0.19.2
+python-dateutil = 2.6.0
+
+# Required by:
+# pathlib2==2.2.1
+scandir = 1.5
+
+# Required by:
+# statsmodels==0.8.0
+scipy = 0.19.0
+
+# Required by:
+# tornado==4.4.2
+singledispatch = 3.4.0.3
+
+# Required by:
+# prompt-toolkit==1.0.13
+wcwidth = 0.1.7
+
+jupyter = 1.0.0
+jupyter-console = 5.1.0
+
+# Required by:
+# jupyter==1.0.0
+qtconsole = 4.3.0
+
+et-xmlfile = 1.0.1
+h5py = 2.7.1
+mpmath = 1.0.0
+openpyxl = 2.5.2
+sympy = 1.1.1
+xlrd = 1.1.0
+
+# Required by:
+# openpyxl==2.5.2
+jdcal = 1.4

component/jupyter-py2/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance-jupyter-notebook]
+filename = instance.cfg.in
+md5sum = 1d5fe6cc4e48672ae7be1c223794a932
+
+[jupyter-notebook-config]
+filename = jupyter_notebook_config.py.jinja
+md5sum = 720e90a829c63371696bc3009917a743
+
+[jupyter-set-password]
+filename = jupyter_set_password.cgi.jinja
+md5sum = ac10fbcf790bd8e58750cfdd069812d2
+
+[erp5-kernel]
+filename = ERP5kernel.py
+md5sum = 7d5309fe79afbcb455c0d8181b42e56c
+
+[kernel-json]
+filename = kernel.json.jinja
+md5sum = 33547be93a67530165e079dc3ecfdac3
+
+[custom-js]
+filename = custom.js
+md5sum = 147ccce38f7d7cf10664975d0dd80e33

component/jupyter-py2/custom.js

+// leave at least 2 line with only a star on it below, or doc generation fails
+/**
+ *
+ *
+ * Placeholder for custom user javascript
+ * mainly to be overridden in profile/static/custom/custom.js
+ * This will always be an empty file in IPython
+ *
+ * User could add any javascript in the `profile/static/custom/custom.js` file.
+ * It will be executed by the ipython notebook at load time.
+ *
+ * Same thing with `profile/static/custom/custom.css` to inject custom css into the notebook.
+ *
+ *
+ * The object available at load time depend on the version of IPython in use.
+ * there is no guaranties of API stability.
+ *
+ * The example below explain the principle, and might not be valid.
+ *
+ * Instances are created after the loading of this file and might need to be accessed using events:
+ *     define([
+ *        'base/js/namespace',
+ *        'base/js/events'
+ *     ], function(IPython, events) {
+ *         events.on("app_initialized.NotebookApp", function () {
+ *             IPython.keyboard_manager....
+ *         });
+ *     });
+ *
+ * __Example 1:__
+ *
+ * Create a custom button in toolbar that execute `%qtconsole` in kernel
+ * and hence open a qtconsole attached to the same kernel as the current notebook
+ *
+ *    define([
+ *        'base/js/namespace',
+ *        'base/js/events'
+ *    ], function(IPython, events) {
+ *        events.on('app_initialized.NotebookApp', function(){
+ *            IPython.toolbar.add_buttons_group([
+ *                {
+ *                    'label'   : 'run qtconsole',
+ *                    'icon'    : 'icon-terminal', // select your icon from http://fortawesome.github.io/Font-Awesome/icons
+ *                    'callback': function () {
+ *                        IPython.notebook.kernel.execute('%qtconsole')
+ *                    }
+ *                }
+ *                // add more button here if needed.
+ *                ]);
+ *        });
+ *    });
+ *
+ * __Example 2:__
+ *
+ * At the completion of the dashboard loading, load an unofficial javascript extension
+ * that is installed in profile/static/custom/
+ *
+ *    define([
+ *        'base/js/events'
+ *    ], function(events) {
+ *        events.on('app_initialized.DashboardApp', function(){
+ *            require(['custom/unofficial_extension.js'])
+ *        });
+ *    });
+ *
+ * __Example 3:__
+ *
+ *  Use `jQuery.getScript(url [, success(script, textStatus, jqXHR)] );`
+ *  to load custom script into the notebook.
+ *
+ *    // to load the metadata ui extension example.
+ *    $.getScript('/static/notebook/js/celltoolbarpresets/example.js');
+ *    // or
+ *    // to load the metadata ui extension to control slideshow mode / reveal js for nbconvert
+ *    $.getScript('/static/notebook/js/celltoolbarpresets/slideshow.js');
+ *
+ *
+ * @module IPython
+ * @namespace IPython
+ * @class customjs
+ * @static
+ */
+
+$([Jupyter.events]).on('notebook_loaded.Notebook', function(){
+  var kernelname = Jupyter.notebook.kernel_selector.current_selection;
+  var display_text="<div class='output_subarea output_text output_result'>\
+  <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
+  1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
+  2. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
+  3. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
+  4. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
+  It would be better to set the reference to match with erp5 reference pattern.</br>\
+  5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
+  <p><u>OTHER USEFUL MAGICS</u> -</br>\
+  <b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
+  <b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
+  NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
+  so changes to them would be disregarded and would throw an error.</br>\
+  <p><u>About classes, functions and global state on modules:</u></p>\
+  Your code is going to be executed by ERP5, which can have many nodes </br>\
+  and there is no guarantee that your code is always going to be executed by the same server.</br>\
+  This means that objects which cannot be stored in the ZODB, like functions, classes and modules </br>\
+  won't be available across nodes. To solve this issue, you need to use a special object </br>\
+  called 'environment' to store your global setup. This object was designed to hold global </br>\
+  state and restore it for each code cell. Example:</br></br>\
+  <b>def my_setup():</br>\
+     # import modules, define functions and classes</br>\
+     # and set global state on modules</br>\
+     # return dict of variables to be available in code cells</br>\
+     {'my_var': 1}</br>\
+  environment.define(my_setup, 'my custom setup')</b></br></br>\
+  After you execute this cell, the <b>my_setup</b> function will run before each of the</br>\
+  following cells and the <b>my_var</b> variable will be created and set to 1.</br></br>\
+  <b>WARNING:</b> it is not recommended to have too many setup functions in the environment, </br>\
+  because they will be executed in every code cell and can cause a substantial slow down.\
+  </pre></div>";
+  if (kernelname=="erp5"){
+  $('div#notebook-container').prepend(display_text);
+  }
+  });

component/jupyter-py2/instance.cfg.in

+[buildout]
+parts =
+  instance
+  jupyter_notebook
+  read-knowledge0
+  publish-connection-parameter
+  jupyter-notebook-config
+  erp5-kernel
+  kernel-json
+  custom-js
+  monitor-base
+extends =
+  {{ monitor_template_rendered }}/template-monitor.cfg
+
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slapconfiguration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+# ERP5 URL to use in Jupyter by default
+# default value is empty - which means no default ERP5 URL
+configuration.erp5-url =
+
+[instance-parameter]
+port = 8888
+host = ${slapconfiguration:ipv6-random}
+cert_file = ${generate-certificate:cert_file}
+key_file = ${generate-certificate:key_file}
+logfile = ${directory:log}/jupyter_notebook.log
+notebook_dir = ${directory:notebook_dir}
+
+[dynamic-jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+mode = 0644
+
+[generate-certificate]
+; TODO: there is a slapos recipe to generate certificates. Use it instead
+recipe = plone.recipe.command
+command =
+  if [ ! -e ${instance-parameter:key_file} ]
+  then
+    {{ openssl_output }} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
+      -newkey rsa:1024 -keyout ${instance-parameter:key_file} \
+      -out ${instance-parameter:cert_file}
+  fi
+update-command = ${:command}
+cert_file = ${directory:etc}/jupyter_cert.crt
+key_file = ${directory:etc}/jupyter_cert.key
+
+[instance]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ bin_directory }}/jupyter-lab
+  --no-browser
+  --ip=${instance-parameter:host}
+  --port=${instance-parameter:port}
+  --port-retries=0
+  --certfile=${instance-parameter:cert_file}
+  --keyfile=${instance-parameter:key_file}
+  --notebook-dir=${instance-parameter:notebook_dir}
+  --log-level="DEBUG"
+wrapper-path = ${directory:service}/jupyter-lab
+environment =
+  JUPYTER_PATH=${directory:jupyter_dir}
+  JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
+  JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
+  LANG=C.UTF-8
+
+[jupyter-notebook-config]
+<= dynamic-jinja2-template-base
+template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
+rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
+mode = 0744
+context =
+  raw config_cfg ${buildout:directory}/knowledge0.cfg
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+script = ${:etc}/run/
+service = ${:etc}/service
+log = ${:var}/log
+notebook_dir = ${:var}/notebooks
+# Add folders to explicitly define jupyter directory
+jupyter_dir = ${:home}/jupyter
+jupyter_config_dir = ${:jupyter_dir}/etc
+jupyter_kernel_dir = ${:jupyter_dir}/kernels
+jupyter_runtime_dir = ${:jupyter_dir}/runtime
+jupyter_custom_dir = ${:jupyter_config_dir}/custom
+jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
+erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
+
+[jupyter_notebook]
+# This part is called like this because knowledge0.write uses the part name for
+# the section name in the config file.
+recipe = slapos.cookbook:zero-knowledge.write
+password = 
+filename = knowledge0.cfg
+
+[read-knowledge0]
+recipe = slapos.cookbook:zero-knowledge.read
+filename = knowledge0.cfg
+password =
+
+[monitor-instance-parameter]
+monitor-base-url = ${monitor-frontend-promise:url}
+# In case you're using a developer instance you should edit these in:
+# monitor-base-url = ${monitor-httpd-conf-parameter:url}
+# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
+# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
+instance-configuration =
+  raw jupyter-password ${read-knowledge0:password}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish.serialised
+jupyter-classic-url = https://[${instance-parameter:host}]:${instance-parameter:port}/tree
+url = ${:jupyter-classic-url}
+jupyterlab-url = https://[${instance-parameter:host}]:${instance-parameter:port}/lab
+
+[erp5-kernel]
+recipe = slapos.cookbook:symbolic.link
+link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
+target-directory = ${directory:erp5_kernel_dir}
+
+[kernel-json]
+<= dynamic-jinja2-template-base
+template = {{ kernel_json_location }}/{{ kernel_json_filename }}
+rendered = ${directory:erp5_kernel_dir}/kernel.json
+# Use python2.7 executable bin file for kernel config
+context =
+  raw python_executable {{ python_executable }}
+  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
+  key erp5_url slapconfiguration:configuration.erp5-url 
+  raw display_name ERP5
+  raw language_name python
+
+[custom-js]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:jupyter_custom_dir}
+link-binary = {{ custom_js_location }}/custom.js

component/jupyter-py2/jupyter_notebook_config.py.jinja

+'''
+  This script initializes Jupyter's configuration such as passwords and other
+  things. It is run by IPython hence why it can use functions like get_config().
+'''
+import ConfigParser
+import random
+from notebook.auth import passwd
+import os
+
+def random_password(length = 10):
+  result = ""
+  for i in range(0, length):
+    result = result + chr(random.randint(0, 25) + ord('a'))
+  return result
+
+knowledge_0 = '{{ config_cfg }}'
+
+if not os.path.exists(knowledge_0):
+  print "Your software does <b>not</b> embed 0-knowledge. \
+  This interface is useless in this case</body></html>"
+  exit(0)
+
+c = get_config()
+
+parser = ConfigParser.ConfigParser()
+parser.read(knowledge_0)
+
+if not parser.has_section("jupyter_notebook"):
+  parser.add_section("jupyter_notebook")
+
+if not parser.has_option("jupyter_notebook", "password") or \
+  parser.get("jupyter_notebook", "password") == "":
+  parser.set("jupyter_notebook", "password", random_password())
+
+c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
+
+with open(knowledge_0, 'w') as file:
+  parser.write(file)

component/jupyter-py2/jupyter_set_password.cgi.jinja

+#!{{ python_executable }}
+
+import cgi
+import cgitb
+import ConfigParser
+import os
+import re
+import subprocess
+
+from IPython.lib import passwd
+
+#cgitb.enable(display=0, logdir="/tmp/cgi.log")
+cgitb.enable()
+form = cgi.FieldStorage()
+
+config_file = "{{ config_cfg }}"
+
+
+if not os.path.exists(config_file):
+  print "Your software does <b>not</b> embed 0-knowledge. \
+  This interface is useless in this case</body></html>"
+  exit(0)
+
+
+parser = ConfigParser.ConfigParser()
+parser.read(config_file)
+if not parser.has_section("jupyter_notebook"):
+  parser.add_section("jupyter_notebook")
+
+if not parser.has_option("jupyter_notebook", "password"):
+  parser.set("jupyter_notebook", "password", "")
+
+if "password" in form:
+  parser.set("jupyter_notebook", "password", passwd(form["password"].value))
+  # subprocess.call('{{ httpd_graceful }}')
+  # TODO: we should restart jupyter
+
+with open(config_file, 'w') as file:
+  parser.write(file)
+
+# TODO cleanup
+print "<html><head>"
+print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
+print "<link rel=\"stylesheet\" href=\"static/style.css\">"
+print "</head><body>"
+print "<h1>Jupyter Notebook Password :</h1>"
+print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
+print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
+
+print """<div class="pure-control-group">
+<label for="password">Password*:</label>
+<input placeholder="Set your password" type="password" name="password" id="password"></br>
+</div><div class="pure-control-group">
+<label for="password">Verify Password*:</label>
+<input placeholder="Verify password" type="password" name="password_2" id="password_2"></br>
+</div><p id="validate-status" style="color:red"></p>
+<div class="pure-controls">
+<button id="register-button" type="submit" class="pure-button pure-button-primary" disabled>Access</button></div>
+</form>
+<script type="text/javascript" src="static/jquery-1.10.2.min.js"></script>
+<script type="text/javascript" src="static/monitor-register.js"></script>
+</body></html>
+"""

component/jupyter-py2/kernel.json.jinja

+{
+  "argv": [
+    "{{python_executable}}",
+    "{{kernel_dir}}",
+    "{{erp5_url}}",
+    "-f",
+    "{connection_file}"
+  ],
+  "display_name": "{{display_name}}",
+  "language": "{{language_name}}",
+  "language_info": {"name": "python"}
+}

software/erp5/test/test/test_erp5.py

@@ -119,6 +119,29 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   def getInstanceParameterDict(cls):
     return {'_': json.dumps({'wsgi': False})}
 
+class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test ERP5 Jupyter notebook
+  """
+  __partition_reference__ = 'jupyter'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'jupyter': {'enable': True}})}
+
+  def test_jupyter_notebook_is_reachable(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+
+    self.assertEqual(
+      'https://[%s]:8888/tree' % self._ipv6_address,
+      param_dict['jupyter-url']
+    )
+
+    result = requests.get(
+      param_dict['jupyter-url'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [requests.codes.found, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
 
 class TestBalancerPorts(ERP5InstanceTestCase):
   """Instantiate with two zope families, this should create for each family:

software/jupyter/ERP5kernel.py

@@ -75,7 +75,7 @@ class ERP5Kernel(Kernel):
     self.title = None
     # Allowed HTTP request code list for making request to erp5 from Kernel
     # This list should be to used check status_code before making requests to erp5
-    self.allowed_HTTP_request_code_list = range(500, 511)
+    self.allowed_HTTP_request_code_list = list(range(500, 511))
     # Append request code 200 in the allowed HTTP status code list
     self.allowed_HTTP_request_code_list.append(200)
 

software/jupyter/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 1d5fe6cc4e48672ae7be1c223794a932
 
 [jupyter-notebook-config]
 filename = jupyter_notebook_config.py.jinja
-md5sum = 720e90a829c63371696bc3009917a743
+md5sum = 3da50c37760a42e42ae3f9d78d9ca449
 
 [jupyter-set-password]
 filename = jupyter_set_password.cgi.jinja
@@ -27,7 +27,7 @@ md5sum = b8d31441780b524a7e52d1710dd78385
 
 [erp5-kernel]
 filename = ERP5kernel.py
-md5sum = 7d5309fe79afbcb455c0d8181b42e56c
+md5sum = da04b99b70b2e327c9e9b4cdd056098e
 
 [kernel-json]
 filename = kernel.json.jinja
@@ -35,4 +35,4 @@ md5sum = 33547be93a67530165e079dc3ecfdac3
 
 [custom-js]
 filename = custom.js
-md5sum = 40d938bb09261c65421a7725b40f87dc
+md5sum = 295aeee765bc4d09bf0686021f32f72c

software/jupyter/custom.js

@@ -81,41 +81,51 @@
  * @static
  */
 
-$([Jupyter.events]).on('notebook_loaded.Notebook', function(){
+function prependERP5Help() {
   var kernelname = Jupyter.notebook.kernel_selector.current_selection;
-  var display_text="<div class='output_subarea output_text output_result'>\
-  <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
-  1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
-  2. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
-  3. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
-  4. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
-  It would be better to set the reference to match with erp5 reference pattern.</br>\
-  5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
-  <p><u>OTHER USEFUL MAGICS</u> -</br>\
-  <b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
-  <b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
-  NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
-  so changes to them would be disregarded and would throw an error.</br>\
-  <p><u>About classes, functions and global state on modules:</u></p>\
-  Your code is going to be executed by ERP5, which can have many nodes </br>\
-  and there is no guarantee that your code is always going to be executed by the same server.</br>\
-  This means that objects which cannot be stored in the ZODB, like functions, classes and modules </br>\
-  won't be available across nodes. To solve this issue, you need to use a special object </br>\
-  called 'environment' to store your global setup. This object was designed to hold global </br>\
-  state and restore it for each code cell. Example:</br></br>\
-  <b>def my_setup():</br>\
-     # import modules, define functions and classes</br>\
-     # and set global state on modules</br>\
-     # return dict of variables to be available in code cells</br>\
-     {'my_var': 1}</br>\
-  environment.define(my_setup, 'my custom setup')</b></br></br>\
-  After you execute this cell, the <b>my_setup</b> function will run before each of the</br>\
-  following cells and the <b>my_var</b> variable will be created and set to 1.</br></br>\
-  <b>WARNING:</b> it is not recommended to have too many setup functions in the environment, </br>\
-  because they will be executed in every code cell and can cause a substantial slow down.\
-  </pre></div>";
-  if (kernelname=="erp5"){
-  $('div#notebook-container').prepend(display_text);
-  }
+  var display_text = "<div class='output_subarea output_text output_result'>\
+    <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
+    1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
+    2. <b>%erp5_url &lt;your_erp5_url&gt;/Base_executeJupyter</b></br>\
+    3. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
+    4. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
+    5. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
+    It would be better to set the reference to match with erp5 reference pattern.</br>\
+    6. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
+    <p><u>OTHER USEFUL MAGICS</u> -</br>\
+    <b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
+    <b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
+    NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
+    so changes to them would be disregarded and would throw an error.</br>\
+    <p><u>About classes, functions and global state on modules:</u></p>\
+    Your code is going to be executed by ERP5, which can have many nodes </br>\
+    and there is no guarantee that your code is always going to be executed by the same server.</br>\
+    This means that objects which cannot be stored in the ZODB, like functions, classes and modules </br>\
+    won't be available across nodes. To solve this issue, you need to use a special object </br>\
+    called 'environment' to store your global setup. This object was designed to hold global </br>\
+    state and restore it for each code cell. Example:</br></br>\
+    <b>def my_setup():</br>\
+      # import modules, define functions and classes</br>\
+      # and set global state on modules</br>\
+      # return dict of variables to be available in code cells</br>\
+      {'my_var': 1}</br>\
+    environment.define(my_setup, 'my custom setup')</b></br></br>\
+    After you execute this cell, the <b>my_setup</b> function will run before each of the</br>\
+    following cells and the <b>my_var</b> variable will be created and set to 1.</br></br>\
+    <b>WARNING:</b> it is not recommended to have too many setup functions in the environment, </br>\
+    because they will be executed in every code cell and can cause a substantial slow down.\
+    </pre></div>";
+
+    if (kernelname=="erp5"){
+      $('div#notebook-container').prepend(display_text);
+    }
+}
+
+define([
+  'base/js/namespace',
+  'base/js/promises'
+], function(Jupyter, promises) {
+  promises.notebook_loaded.then(function() {
+    prependERP5Help();
   });
-  
+});
\ No newline at end of file

software/jupyter/jupyter_notebook_config.py.jinja

@@ -2,7 +2,7 @@
   This script initializes Jupyter's configuration such as passwords and other
   things. It is run by IPython hence why it can use functions like get_config().
 '''
-import ConfigParser
+import configparser
 import random
 from notebook.auth import passwd
 import os
@@ -16,13 +16,13 @@ def random_password(length = 10):
 knowledge_0 = '{{ config_cfg }}'
 
 if not os.path.exists(knowledge_0):
-  print "Your software does <b>not</b> embed 0-knowledge. \
-  This interface is useless in this case</body></html>"
+  print("Your software does <b>not</b> embed 0-knowledge. \
+  This interface is useless in this case</body></html>")
   exit(0)
 
 c = get_config()
 
-parser = ConfigParser.ConfigParser()
+parser = configparser.ConfigParser()
 parser.read(knowledge_0)
 
 if not parser.has_section("jupyter_notebook"):

software/jupyter/software.cfg

@@ -11,6 +11,9 @@ parts +=
   jupyter-notebook-initialized-scripts
   instance-jupyter-notebook
 
+[python]
+part = python3
+
 [gcc]
 # Always build GCC for Fortran (see openblas).
 max_version = 0
@@ -63,40 +66,49 @@ context =
   key monitor_template_rendered buildout:directory
 
 [versions]
-Pygments = 2.2.0
+Pygments = 2.7.2
 astor = 0.5
+async-generator = 1.10
 backports-abc = 0.5
 backports.functools-lru-cache = 1.6.1
 backports.shutil-get-terminal-size = 1.0.0
+bleach = 3.2.1
 cycler = 0.10.0
-ipykernel = 4.5.2
+defusedxml = 0.6.0
+entrypoints = 0.3
+ipykernel = 5.3.4
 ipython = 5.3.0
 ipython-genutils = 0.1.0
 ipywidgets = 6.0.0
-jupyter-client = 5.0.0
-jupyter-core = 4.3.0
+jupyter-client = 6.1.7
+jupyter-core = 4.7.0
 jupyterlab = 0.26.3
 jupyterlab-launcher = 0.3.1
+jupyterlab-pygments = 0.1.2
 matplotlib = 2.1.2
-mistune = 0.7.3
-nbformat = 4.3.0
-notebook = 4.4.1
-pandas = 0.19.2
+mistune = 0.8.4
+nest-asyncio = 1.4.3
+nbclient = 0.5.1
+nbformat = 5.0.8
+notebook = 6.1.5
+numpy = 1.14.6
+pandas = 0.25.3
+pandocfilters = 1.4.3
 plone.recipe.command = 1.1
 prompt-toolkit = 1.0.13
 ptyprocess = 0.5.1
-pyzmq = 16.0.2
-scikit-learn = 0.18.1
+pyzmq = 20.0.0
+scikit-learn = 0.20.4
 seaborn = 0.7.1
 simplegeneric = 0.8.1
 slapos.recipe.template = 4.4
-statsmodels = 0.8.0
-terminado = 0.6
-tornado = 4.4.2
-traitlets = 4.3.2
+statsmodels = 0.11.1
+testpath = 0.4.4
+terminado = 0.9.1
+tornado = 6.1
+traitlets = 5.0.5
+webencodings = 0.5.1
 widgetsnbextension = 2.0.0
-# numpy >= 1.13.1 is required for numpy.core.multiarray
-numpy = 1.13.1
 
 
 # Required by:
@@ -104,41 +116,55 @@ numpy = 1.13.1
 certifi = 2020.6.20
 
 # Required by:
-# notebook==4.3.2
-# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
-nbconvert = 4.1.0
+# notebook==6.1.5
+Send2Trash = 1.5.0
+
+# Required by:
+# notebook==6.1.5
+argon2-cffi = 20.1.0
+
+# Required by:
+# notebook==6.1.5
+nbconvert = 6.0.7
 
 # Required by:
 # ipython==5.3.0
 pathlib2 = 2.2.1
 
 # Required by:
-# statsmodels==0.8.0
-patsy = 0.4.1
+# statsmodels==0.11.1
+patsy = 0.5.1
 
 # Required by:
 # ipython==5.3.0
-pexpect = 4.2.1
+pexpect = 4.8.0
 
 # Required by:
 # ipython==5.3.0
 pickleshare = 0.7.4
 
 # Required by:
-# matplotlib==2.1.2
-# pandas==0.19.2
-python-dateutil = 2.6.0
+# notebook==6.1.5
+prometheus-client = 0.9.0
+
+# Required by:
+# statsmodels==0.11.1
+python-dateutil = 2.8.1
 
 # Required by:
 # pathlib2==2.2.1
 scandir = 1.5
 
 # Required by:
-# statsmodels==0.8.0
-scipy = 0.19.0
+# statsmodels==0.11.1
+pytz = 2020.4
 
 # Required by:
-# tornado==4.4.2
+# statsmodels==0.11.1
+scipy = 1.0.1
+
+# Required by:
+# tornado==6.1
 singledispatch = 3.4.0.3
 
 # Required by:

software/jupyter/test/test.py

@@ -26,7 +26,7 @@
 ##############################################################################
 
 
-import httplib
+import http.client
 import json
 import os
 import requests
@@ -60,7 +60,7 @@ class TestJupyter(InstanceTestCase):
     result = requests.get(
       connection_dict['url'], verify=False, allow_redirects=False)
     self.assertEqual(
-      [httplib.FOUND, True, '/login?next=%2Ftree'],
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
 
@@ -68,7 +68,7 @@ class TestJupyter(InstanceTestCase):
       connection_dict['jupyter-classic-url'],
       verify=False, allow_redirects=False)
     self.assertEqual(
-      [httplib.FOUND, True, '/login?next=%2Ftree'],
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
 
@@ -76,6 +76,6 @@ class TestJupyter(InstanceTestCase):
       connection_dict['jupyterlab-url'],
       verify=False, allow_redirects=False)
     self.assertEqual(
-      [httplib.FOUND, True, '/login?next=%2Flab'],
+      [http.client.FOUND, True, '/login?next=%2Flab'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )

software/slapos-sr-testing/software-py3.cfg

@@ -13,6 +13,7 @@ extra-eggs +=
 [template]
 extra =
   helloworld ${slapos.test.helloworld-setup:setup}
+  jupyter ${slapos.test.jupyter-setup:setup}
   monitor ${slapos.test.monitor-setup:setup}
   plantuml ${slapos.test.plantuml-setup:setup}
   powerdns ${slapos.test.powerdns-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -271,7 +271,6 @@ extra =
   re6stnet ${slapos.test.re6stnet-setup:setup}
   seleniumserver ${slapos.test.seleniumserver-setup:setup}
   jstestnode ${slapos.test.jstestnode-setup:setup}
-  jupyter ${slapos.test.jupyter-setup:setup}
   nextcloud ${slapos.test.nextcloud-setup:setup}
   turnserver ${slapos.test.turnserver-setup:setup}
   theia ${slapos.test.theia-setup:setup}

stack/erp5/buildout.cfg

@@ -56,8 +56,8 @@ extends =
   ../../component/pylint/buildout.cfg
   ../../component/perl-Image-ExifTool/buildout.cfg
   ../../component/wendelin.core/buildout.cfg
+  ../../component/jupyter-py2/buildout.cfg
   ../../stack/caucase/buildout.cfg
-  ../../software/jupyter/software.cfg
   ../../software/neoppod/software-common.cfg
 # keep neoppod extends last
 

stack/slapos.cfg

@@ -191,7 +191,7 @@ scandir = 1.10.0
 setuptools-dso = 1.7
 rubygemsrecipe  = 0.3.0
 six = 1.12.0
-slapos.cookbook = 1.0.167
+slapos.cookbook = 1.0.171
 slapos.core = 1.6.3
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0