erp5: ERP5 and Jupyter integrated together

This patch series teaches ERP5 software release to automatically instantiate
Jupyter notebook web UI and tune it to connect to ERP5 by default. When Jupyter
is enabled, it also installs on-server erp5_data_notebook bt5 (see
nexedi/erp5!29 and nexedi/erp5@f662b5a2) which handles code execution requested
for Jupyter.

For ERP5 - for security and backward compatibility reasons - Jupyter
instantiation and erp5_data_notebook bt5 install happen only if jupyter is
explicitly enabled in instance parameters. The default is not to have Jupyter
out of the box.

On the other hand for Wendelin SR, which inherits from ERP5 SR, the
default is to have Jupyter out of the box, because Wendelin SR is fresh
enough without lots of backward compatibility needs, and Jupyter is
usually very handy for people who use Wendelin.

--------

NOTE Currently erp5-data-notebook bt5 has the following limitations (see
details on nexedi/slapos!43 and nexedi/erp5!29):

- errors are not reported properly to users;
- state is not fully saved to ZODB.

the latter point means notebook works only if it is connected to Zope family
with only 1 zope process. Hopefully this will be resolved some day.

Technical overview about how the integration is done itself on slapos part and
other notes are in 0a446263.

/proposed-for-review-on nexedi/slapos!43

software/erp5/README.rst

@@ -39,6 +39,7 @@ This software release assigns the following port ranges by default:
   zeo                   2100-2149
   balancer              2150-2199
   zope                  2200-*
+  jupyter               8888
   ====================  ==========
 
 Non-zope partitions are unique in an ERP5 cluster, so you shouldn't have to

software/erp5/instance-erp5-input-schema.json

@@ -235,6 +235,22 @@
         "type": "object"
       },
       "type": "array"
+    },
+    "jupyter": {
+      "description": "Jupyter slave instance parameters",
+      "properties": {
+        "enable": {
+          "description": "Whether to enable creation of associated slave Jupyter instance",
+          "default": false,
+          "type": "boolean"
+        },
+        "zope-family": {
+          "description": "Zope family to connect Jupyter to by default",
+          "default": "<first instantiated Zope family>",
+          "type": "string"
+        }
+      },
+      "type": "object"
     }
   }
 }

software/erp5/instance-erp5-output-schema.json

@@ -40,6 +40,11 @@
       "description": "Relational database access information",
       "type": "string"
     }
+    "jupyter-url": {
+      "description": "Jupyter notebook web UI access information",
+      "type": "string",
+      "optional": true
+    }
   },
   "patternProperties": {
     "family-.*": {

software/ipython_notebook/instance.cfg.in

@@ -34,13 +34,17 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slapconfiguration]
-recipe = slapos.cookbook:slapconfiguration
+recipe = slapos.cookbook:slapconfiguration.serialised
 computer = ${slap-connection:computer-id}
 partition = ${slap-connection:partition-id}
 url = ${slap-connection:server-url}
 key = ${slap-connection:key-file}
 cert = ${slap-connection:cert-file}
 
+# ERP5 URL to use in Jupyter by default
+# default value is empty - which means no default ERP5 URL
+configuration.erp5-url =
+
 [instance-parameter]
 port = 8888
 host = ${slapconfiguration:ipv6-random}
@@ -141,6 +145,7 @@ rendered = ${directory:erp5_kernel_dir}/ERP5kernel.py
 # Use ipython as executable python as we'll be needing requests library in kernel
 context =
   raw python_executable {{ bin_directory }}/ipython
+  key erp5_url slapconfiguration:configuration.erp5-url
 
 [kernel-json]
 <= dynamic-jinja2-template-base

software/ipython_notebook/software.cfg

@@ -43,7 +43,7 @@ md5sum = d7d4a7e19d55bf14007819258bf42100
 [erp5-kernel]
 <= download-file-base
 filename = ERP5kernel.py.jinja
-md5sum = da2f592075c414d4bb26cf7a7dfd147b
+md5sum = 3dfc6a7c16828bff55dec4cf96b730d3
 
 [kernel-json]
 <= download-file-base
@@ -60,7 +60,7 @@ recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance.cfg.in
 rendered = ${buildout:directory}/template.cfg
 mode = 0644
-md5sum = 1a993b1f8fa3f001c45075fe95a48332
+md5sum = c6b82a386a72ed72301302c3132ffb71
 context =
   key bin_directory buildout:bin-directory
   key develop_eggs_directory buildout:develop-eggs-directory

software/ipython_notebook/template/ERP5kernel.py.jinja

@@ -9,9 +9,11 @@ import requests
 import json
 
 # erp5_url from buildout
-# TODO: Uncomment after adding automated installation of erp5-data-notebook bt5
-# url = ""
-# url  = "%s/erp5/Base_executeJupyter"%url
+erp5_url = "{{ erp5_url }}"
+if not erp5_url:
+    erp5_url = None
+else:
+    erp5_url = "%s/erp5/Base_executeJupyter" % erp5_url
 
 class MagicInfo:
   """
@@ -69,9 +71,12 @@ class ERP5Kernel(Kernel):
     super(ERP5Kernel, self).__init__(*args, **kwargs)
     self.user = user
     self.password = password
-    # Use URL provided by buildout during initiation
+    # By default use URL provided by buildout during initiation
     # It can later be overridden
-    self.url = url
+    if url is None:
+        self.url = erp5_url
+    else:
+        self.url = url
     self.status_code = status_code
     self.reference = None
     self.title = None
@@ -167,11 +172,11 @@ class ERP5Kernel(Kernel):
     """
 
     try:
-      erp5_request = requests.get(
+      erp5_request = requests.post(
         self.url,
         verify=False,
         auth=(self.user, self.password),
-        params={
+        data={
           'python_expression': code,
           'reference': self.reference,
           'title': self.title,

software/wendelin/software.cfg

 [buildout]
 versions = versions
 extends =
-  ../../software/ipython_notebook/software.cfg
   ../../component/fluentd/buildout.cfg
-  ../../component/matplotlib/buildout.cfg
-  ../../component/ipython/buildout.cfg
-  ../../component/pandas/buildout.cfg
   ../../component/wendelin.core/buildout.cfg
   ../../component/msgpack-python/buildout.cfg
   ../../component/scipy/buildout.cfg
-  ../../component/scikit-learn/buildout.cfg
   ../../software/erp5/software.cfg
 parts +=
   wendelin
   scipy
-  scikit-learn
-  pandas
   msgpack-python
   ipython
   wendelin.core
-  matplotlib
   fluentd
   ipython-notebook
 
@@ -29,13 +21,10 @@ initialization =
 extra-paths +=
   ${wendelin:location}
 eggs +=
-  ${scikit-learn:egg}
   ${scipy:egg}
-  ${pandas:egg}
   ${msgpack-python:egg}
   ${wendelin.core:egg}
   ${ipython:egg}
-  ${matplotlib:egg}
 
 [erp5_repository_list]
 repository_id_list += wendelin
@@ -44,15 +33,16 @@ repository_id_list += wendelin
 # we need to override it
 list = ${erp5:location}/bt5 ${erp5:location}/product/ERP5/bootstrap ${wendelin:location}/bt5/
 
+# Jupyter is by default enabled in Wendelin
+[erp5-defaults]
+jupyter-enable-default = true
+
 [wendelin]
 <= erp5
 repository = https://lab.nexedi.com/nexedi/wendelin.git
 branch = master
 
 [versions]
-scikit-learn = 0.16.1
 scipy = 0.15.1
-pandas = 0.16.1
 msgpack-python = 0.4.6
 wendelin.core = 0.5
-matplotlib = 1.4.3

stack/erp5/buildout.cfg

@@ -21,14 +21,17 @@ extends =
   ../../component/libffi/buildout.cfg
   ../../component/libpng/buildout.cfg
   ../../component/libreoffice-bin/buildout.cfg
+  ../../component/matplotlib/buildout.cfg
   ../../component/mesa/buildout.cfg
   ../../component/numpy/buildout.cfg
+  ../../component/pandas/buildout.cfg
   ../../component/percona-toolkit/buildout.cfg
   ../../component/patch/buildout.cfg
   ../../component/pillow/buildout.cfg
   ../../component/pysvn-python/buildout.cfg
   ../../component/python-ldap-python/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
+  ../../component/scikit-learn/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/subversion/buildout.cfg
   ../../component/tesseract/buildout.cfg
@@ -49,6 +52,7 @@ extends =
   ../../component/findutils/buildout.cfg
   ../../component/userhosts/buildout.cfg
   ../../component/postfix/buildout.cfg
+  ../../software/ipython_notebook/software.cfg
   ../../software/neoppod/software-common.cfg
 # keep neoppod extends last
 
@@ -123,6 +127,15 @@ parts +=
 # Create instance template
   template
 
+# jupyter
+  ipython-notebook
+  instance-jupyter
+  monitor-eggs
+
+# override instance-jupyter not to render into default template.cfg
+[instance-jupyter]
+rendered = ${buildout:directory}/template-jupyter.cfg
+
 [download-base]
 <= download-base-neo
 url = ${:_profile_base_location_}/${:filename}
@@ -220,7 +233,7 @@ recipe = slapos.recipe.template:jinja2
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 rendered = ${buildout:directory}/template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 540956c635acc9707045510c11f80016
+md5sum = 98a4edfb18cfd810ea570f56d502a2cc
 mode = 640
 context =
     key mariadb_link_binary template-mariadb:link-binary
@@ -250,6 +263,7 @@ context =
     key haproxy_location haproxy:location
     key instance_common_cfg instance-common:rendered
     key jsl_location jsl:location
+    key jupyter_enable_default erp5-defaults:jupyter-enable-default
     key kumo_location kumo:location
     key libICE_location libICE:location
     key libSM_location libSM:location
@@ -283,6 +297,7 @@ context =
     key template_create_erp5_site_real template-create-erp5-site-real:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
+    key template_jupyter_cfg instance-jupyter:rendered
     key template_kumofs template-kumofs:target
     key template_mariadb template-mariadb:target
     key template_mariadb_initial_setup template-mariadb-initial-setup:target
@@ -314,7 +329,7 @@ rendered = ${monitor-template-dummy:target}
 [template-erp5]
 <= download-base
 filename = instance-erp5.cfg.in
-md5sum = 977119d0b876df827c97bb64e6e98273
+md5sum = 66edf64eeaecded8977459acb26f4424
 
 [template-zeo]
 <= download-base
@@ -384,6 +399,11 @@ update-command = ${:command}
 [erp5_repository_list]
 repository_id_list = erp5
 
+# ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin)
+[erp5-defaults]
+# Jupyter is by default disabled in ERP5
+jupyter-enable-default = false
+
 [erp5]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/erp5.git
@@ -451,12 +471,15 @@ initialization =
 <= neoppod
 eggs =
   ${numpy:egg}
+  ${matplotlib:egg}
   ${mysql-python:egg}
   ${lxml-python:egg}
+  ${pandas:egg}
   ${pillow-python:egg}
   ${python-ldap-python:egg}
   ${pysvn-python:egg}
   ${pycrypto-python:egg}
+  ${scikit-learn:egg}
   lock_file
   PyStemmer
   PyXML
@@ -673,8 +696,10 @@ interval = 1.0.0
 ipdb = 0.8.1
 ipython = 4.0.0
 logilab-common = 1.1.0
+matplotlib = 1.4.3
 numpy = 1.10.4
 objgraph = 2.0.1
+pandas = 0.16.1
 ply = 3.8
 polib = 1.0.7
 pprofile = 1.7.3
@@ -688,6 +713,7 @@ pytracemalloc = 1.2
 qrcode = 5.2.2
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
+scikit-learn = 0.16.1
 simplegeneric = 0.8.1
 socketpool = 0.5.3
 spyne = 2.12.11

stack/erp5/instance-erp5.cfg.in

@@ -5,6 +5,9 @@
 {% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%}
 {% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%}
 {% set has_posftix = slapparameter_dict.get('smtp', {}).get('postmaster') -%}
+{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
+{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default).lower() in ('true', 'yes') -%}
+{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
 [request-common]
 <= request-common-base
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
@@ -119,7 +122,11 @@ name = neo-${gen-neo-cluster-base:passwd}
 return =
   zope-address-list
   hosts-dict
-config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc')) }}
+{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc' -%}
+{% if has_jupyter -%}
+{%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
+{% endif -%}
+config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-cloudooo-url = ${request-cloudooo:connection-url}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
@@ -150,10 +157,17 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
 software-type = zope
 
 {% set zope_family_dict = {} -%}
+{% set jupyter_zope_family_default = [] -%}
 {% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%}
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
-{%   do zope_family_dict.setdefault(zope_parameter_dict.get('family', 'default'), []).append(section_name) -%}
+{%   set zope_family = zope_parameter_dict.get('family', 'default') -%}
+{#   # default jupyter zope family is first zope family. -#}
+{#   # use list.append() to update it, because in jinja2 set changes only local scope. -#}
+{%   if not jupyter_zope_family_default -%}
+{%     do jupyter_zope_family_default.append(zope_family) -%}
+{%   endif -%}
+{%   do zope_family_dict.setdefault(zope_family, []).append(section_name) -%}
 [{{ section_name }}]
 <= request-zope-base
 name = {{ partition_name }}
@@ -168,6 +182,12 @@ config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
 {% endfor -%}
 
+{# if not explicitly configured, connect jupyter to first zope family, which  -#}
+{# will be 'default' if zope families are not configured also -#}
+{% if not jupyter_zope_family and jupyter_zope_family_default -%}
+{%   set jupyter_zope_family = jupyter_zope_family_default[0] -%}
+{% endif -%}
+
 {# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#}
 {% set zope_address_list_id_dict = {} -%}
 {% set zope_family_parameter_dict = {} -%}
@@ -190,6 +210,20 @@ config-url = ${request-balancer:connection-{{ family_name }}-v6}
 {%   endif -%}
 {% endfor -%}
 
+{% if has_jupyter -%}
+{# request jupyter connected to balancer of proper zope family -#}
+{{   request('jupyter', 'jupyter', 'jupyter', {}, key_config={'erp5-url': 'request-balancer:connection-' ~ jupyter_zope_family}) }}
+
+{%   if has_frontend -%}
+[frontend-jupyter]
+<= request-frontend-base
+name = frontend-jupyter
+config-url = ${request-jupyter:connection-url}
+{#     # override jupyter-url in publish_dict with frontend address -#}
+{%     do publish_dict.__setitem__('jupyter-url', '${frontend-jupyter:connection-site_url}') -%}
+{%   endif -%}
+{%- endif %}
+
 {% set balancer_dict = slapparameter_dict.get('balancer', {}) -%}
 [request-balancer]
 <= request-common

stack/erp5/instance.cfg.in

@@ -64,6 +64,7 @@ extra-context =
     import urllib urllib
 
 [dynamic-template-erp5-parameters]
+jupyter-enable-default = {{ jupyter_enable_default }}
 local-bt5-repository = {{ local_bt5_repository }}
 
 [dynamic-template-erp5]
@@ -71,6 +72,7 @@ local-bt5-repository = {{ local_bt5_repository }}
 template = {{ template_erp5 }}
 filename = instance-erp5.cfg
 extra-context =
+    key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     import urlparse urlparse
 import-list =
@@ -177,6 +179,11 @@ filename = instance-create-erp5-site.cfg
 extra-context =
     section parameter_dict dynamic-template-create-erp5-site-parameters
 
+# we need this value to be present in a section,
+# for slapos.cookbook:switch-softwaretype to work
+[dynamic-template-jupyter]
+rendered = {{ template_jupyter_cfg }}
+
 [switch-softwaretype]
 recipe = slapos.cookbook:switch-softwaretype
 override = {{ dumps(override_switch_softwaretype |default) }}
@@ -195,3 +202,4 @@ postfix = dynamic-template-postfix:rendered
 zodb-zeo = dynamic-template-zeo:rendered
 zodb-neo = neo-storage-mysql:rendered
 zope = dynamic-template-zope:rendered
+jupyter = dynamic-template-jupyter:rendered