Commit b7b73a33 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

software/jupyter: move to python 3

See merge request !868
parents 6acab77d b15cac5f
from ipykernel.kernelbase import Kernel
from ipykernel.kernelapp import IPKernelApp
from IPython.core.display import HTML
import requests
import json
import sys
erp5_url = None
if len(sys.argv) > 1:
erp5_url = "%s/erp5/Base_executeJupyter" % (sys.argv[1],)
class MagicInfo:
"""
Magics definition structure.
Initializes a new MagicInfo class with specific paramters to identify a magic.
"""
def __init__(self, magic_name, variable_name, send_request, request_reference, display_message):
self.magic_name = magic_name
self.variable_name = variable_name
self.send_request = send_request
self.request_reference = request_reference
self.display_message = display_message
# XXX: New magics to be added here in the dictionary.
# In this dictionary,
# key = magic_name,
# value = MagicInfo Structure corresponding to the magics
# Different parameters of the structures are :-
# magics_name(str) = Name which would be used on jupyter frontend
# variable_name(str) = Name of variable on which magic would be set in kernel
# send_request(boolean) = Magics for which requests to erp5 backend need to be made
# request_reference(boolean) = Request for notebook references(and titles) from erp5
# display_message(boolean) = If the magics need to display message after
# making request. Useful for magics which do get some
# useful content from erp5 backend and need to display
MAGICS = {
'erp5_user': MagicInfo('erp5_user', 'user', True, False, True),
'erp5_password': MagicInfo('erp5_password', 'password', True, False, True),
'erp5_url': MagicInfo('erp5_url', 'url', True, False, True),
'notebook_set_reference': MagicInfo('notebook_set_reference', 'reference', True, False, True),
'notebook_set_title': MagicInfo('notebook_set_title', 'title', False, False, True),
'my_notebooks': MagicInfo('my_notebooks', '', True, True, False)}
class ERP5Kernel(Kernel):
"""
Jupyter Kernel class to interact with erp5 backend for code from frontend.
To use this kernel with erp5, user need to install 'erp5_data_notebook' bt5
Also, handlers(aka magics) starting with '%' are predefined.
Each request to erp5 for code execution requires erp5_user, erp5_password
and reference of the notebook.
"""
implementation = 'ERP5'
implementation_version = '1.0'
language = 'ERP5'
language_version = '0.1'
language_info = {'mimetype': 'text/plain', 'name':'python'}
banner = "ERP5 integration with jupyter notebook"
def __init__(self, user=None, password=None, url=None, status_code=None,
*args, **kwargs):
super(ERP5Kernel, self).__init__(*args, **kwargs)
self.user = user
self.password = password
# By default use URL provided by buildout during initiation
# It can later be overridden
if url is None:
self.url = erp5_url
else:
self.url = url
self.status_code = status_code
self.reference = None
self.title = None
# Allowed HTTP request code list for making request to erp5 from Kernel
# This list should be to used check status_code before making requests to erp5
self.allowed_HTTP_request_code_list = range(500, 511)
# Append request code 200 in the allowed HTTP status code list
self.allowed_HTTP_request_code_list.append(200)
def display_response(self, response=None):
"""
Dispays the stream message response to jupyter frontend.
"""
if response:
stream_content = {'name': 'stdout', 'text': response}
self.send_response(self.iopub_socket, 'stream', stream_content)
def set_magic_attribute(self, magic_info=None, code=None):
"""
Set attribute for magic which are necessary for making requests to erp5.
Catch errors and display message. Since user is in contact with jupyter
frontend, so its better to catch exceptions and dispaly messages than to
let them fail in backend and stuck the kernel.
For a making a request to erp5, we need -
erp5_url, erp5_user, erp5_password, notebook_set_reference
"""
# Set attributes only for magic who do have any varible to set value to
if magic_info.variable_name:
try:
# Get the magic value recived via code from frontend
magic_value = code.split()[1]
# Set magic_value to the required attribute
if magic_info.magic_name == 'notebook_set_reference':
required_attributes = ['url', 'password', 'user']
missing_attributes = []
for attribute in required_attributes:
if not getattr(self, attribute):
missing_attributes.append(attribute)
if missing_attributes != []:
self.response = "You still haven't entered all required magics. \
Please do so before inputting your reference."
else:
if self.check_existing_reference(reference=magic_value):
self.response = 'WARNING: You already have a notebook with \
reference %s. It might be a good idea to use different references for new \
notebooks. \n' % magic_value
else:
self.response = ''
setattr(self, magic_info.variable_name , magic_value)
self.response = self.response + 'Your %s is %s. '%(magic_info.magic_name, magic_value)
elif magic_info.magic_name != 'erp5_password':
setattr(self, magic_info.variable_name , magic_value)
self.response = 'Your %s is %s. '%(magic_info.magic_name, magic_value)
else:
setattr(self, magic_info.variable_name , magic_value)
self.response = ""
# Catch exception while setting attribute and set message in response
except AttributeError:
self.response = 'Please enter %s magic value'%magic_info.variable_name
# Catch IndexError while getting magic_value and set message in response object
except IndexError:
self.response = 'Empty value for %s magic'%magic_info.variable_name
# Catch all other exceptions and set error_message in response object
# XXX: Might not be best way, but its better to display error to the user
# via notebook frontend than to fail in backend and stuck the Kernel without
# any failure message to user.
except Exception as e:
self.response = str(e)
# Display the message/response from this fucntion before moving forward so
# as to keep track of the status
if self.response != "":
self.display_response(response=(self.response + '\n'))
def check_required_attributes(self):
"""
Check if the required attributes for making a request are already set or not.
Display message to frontend to provide with the values in case they aren't.
This function can be called anytime to check if the attributes are set. The
output result will be in Boolean form.
Also, in case any of attribute is not set, call to display_response would be
made to ask user to enter value.
"""
result_list = []
required_attributes = ['url', 'user', 'password', 'reference']
missing_attributes = []
# Set response to empty so as to flush the response set by some earlier fucntion call
self.response = ''
# Loop to check if the required attributes are set
for attribute in required_attributes:
if getattr(self, attribute):
result_list.append(True)
else:
# Set response/message for attributes which aren't set
missing_attributes.append(attribute)
result_list.append(False)
# Compare result_list to get True for all True results and False for any False result
check_attributes = all(result_list)
if check_attributes:
self.response = 'You have entered all required magics. You may now use your notebook.'
else:
self.response = '''You have these required magics remaining: %s. \n''' % (
', '.join(map(str, missing_attributes)))
# Display response to frontend before moving forward
self.display_response(response=(self.response + '\n'))
return check_attributes
def make_erp5_request(self, request_reference=False, display_message=True,
code=None, message=None, title=None, *args, **kwargs):
"""
Function to make request to erp5 as per the magics.
Should return the response json object.
"""
try:
erp5_request = requests.post(
self.url,
verify=False,
auth=(self.user, self.password),
data={
'python_expression': code,
'reference': self.reference,
'title': self.title,
'request_reference': request_reference,
'store_history': kwargs.get('store_history')
})
# Set value for status_code for self object which would later be used to
# dispaly response after statement check
self.status_code = erp5_request.status_code
# Dispaly error response in case the request give any other status
# except 200 and 5xx(which is for errors on server side)
if self.status_code not in self.allowed_HTTP_request_code_list:
self.response = '''Error code %s on request to ERP5,\n
check credentials or ERP5 family URL'''%self.status_code
else:
# Set value of self.response to the given value in case response from function
# call. In all other case, response should be the content from request
if display_message and message:
self.response = message
else:
self.response = erp5_request.content
except requests.exceptions.RequestException as e:
self.response = str(e)
def do_execute(self, code, silent, store_history=True, user_expressions=None,
allow_stdin=False):
"""
Validate magic and call functions to make request to erp5 backend where
the code is being executed and response is sent back which is then sent
to jupyter frontend.
"""
# By default, take the status of response as 'ok' so as show the responses
# for erp5_url and erp5_user on notebook frontend as successful response.
status = 'ok'
if not silent:
# Remove spaces and newlines from both ends of code
code = code.strip()
extra_data_list = []
print_result = {}
displayhook_result = {}
if code.startswith('%'):
# No need to try-catch here as its already been taken that the code
# starts-with '%', so we'll get magic_name, no matter what be after '%'
magic_name = code.split()[0][1:]
magics_name_list = [magic.magic_name for magic in MAGICS.values()]
# Check validation of magic
if magic_name and magic_name in magics_name_list:
# Get MagicInfo object related to the magic
magic_info = MAGICS.get(magic_name)
# Function call to set the required magics
self.set_magic_attribute(magic_info=magic_info, code=code)
# Call to check if the required_attributes are set
checked_attribute = self.check_required_attributes()
if checked_attribute and magic_info.send_request:
# Call the function to send request to erp5 with the arguments given
self.make_erp5_request(message='Please proceed\n',
request_reference=magic_info.request_reference,
display_message=magic_info.display_message)
# Display response from erp5 request for magic
# Since this response would be either success message or failure
# error message, both of which are string type, so, we can simply
# display the stream response.
if self.response != 'Please proceed\n':
self.display_response(response=self.response)
else:
# Set response if there is no magic or the magic name is not in MAGICS
self.response = 'Invalid Magics'
self.display_response(response=self.response)
else:
# Check for status_code before making request to erp5 and make request in
# only if the status_code is in the allowed_HTTP_request_code_list
if self.status_code in self.allowed_HTTP_request_code_list:
self.make_erp5_request(code=code, store_history=store_history)
# For 200 status_code, Kernel will receive predefined format for data
# from erp5 which is either json of result or simple result string
if self.status_code == 200:
mime_type = 'text/plain'
try:
content = json.loads(self.response)
# Example format for the json result we are expecting is :
# content = {
# "status": "ok",
# "ename": null,
# "evalue": null,
# "traceback": null,
# "code_result": "",
# "print_result": {},
# "displayhook_result": {},
# "mime_type": "text/plain",
# "extra_data_list": []
# }
# So, we can easily use any of the key to update values as such.
# Getting code_result for succesfull execution of code
code_result = content['code_result']
print_result = content['print_result']
displayhook_result = content['displayhook_result']
# Update mime_type with the mime_type from the http response result
# Required in case the mime_type is anything other than 'text/plain'
mime_type = content['mime_type']
extra_data_list = content.get('extra_data_list', [])
# Display to frontend the error message for content status as 'error'
if content['status']=='error':
reply_content = {
'status': 'error',
'execution_count': self.execution_count,
'ename': content['ename'],
'evalue': content['evalue'],
'traceback': content['traceback']}
self.send_response(self.iopub_socket, u'error', reply_content)
return reply_content
# Catch exception for content which isn't json
except ValueError:
content = self.response
code_result = content
print_result = {'data':{'text/plain':content}, 'metadata':{}}
# Display basic error message to frontend in case of error on server side
else:
self.make_erp5_request(code=code)
code_result = "Error at Server Side"
print_result = {'data':{'text/plain':'Error at Server Side'}, 'metadata':{}}
mime_type = 'text/plain'
# For all status_code except allowed_HTTP_response_code_list show unauthorized message
else:
code_result = 'Unauthorized access'
print_result = {'data':{'text/plain':'Unauthorized access'}, 'metadata':{}}
mime_type = 'text/plain'
if print_result.get('data'):
self.send_response(self.iopub_socket, 'display_data', print_result)
if displayhook_result.get('data'):
displayhook_result['execution_count'] = self.execution_count
self.send_response(self.iopub_socket, 'execute_result', displayhook_result)
for extra_data in extra_data_list:
self.send_response(self.iopub_socket, 'display_data', extra_data)
reply_content = {
'status': status,
# The base class increments the execution count
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}}
return reply_content
# Checks the ERP5 site if there are existing notebooks with the same reference.
# Returns True if there are.
def check_existing_reference(self, reference):
if reference == None:
return False
modified_url = self.url[:self.url.rfind('/')] + '/Base_checkExistingReference'
result = True
try:
erp5_request = requests.post(
modified_url,
verify=False,
auth=(self.user, self.password),
data={
'reference': reference,
})
result = erp5_request.content
except requests.exceptions.RequestException as e:
self.response = str(e)
return result
if __name__ == '__main__':
IPKernelApp.launch_instance(kernel_class=ERP5Kernel)
This directory is only temporary (until ERP5 is Python3) and will not actively
be maintained.
It has to be removed once ERP5 uses `software/jupyter` or do not use jupyter at
all.
[buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../openssl/buildout.cfg
../jupyter/buildout.cfg
../../stack/monitor/buildout.cfg
parts +=
slapos-cookbook
jupyter
jupyter-notebook-initialized-scripts
instance-jupyter-notebook
[gcc]
# Always build GCC for Fortran (see openblas).
max_version = 0
[jupyter]
python_executable = ${buildout:bin-directory}/${:interpreter}
[download-file-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
download-only = true
mode = 0644
[jupyter-notebook-config]
<= download-file-base
[jupyter-set-password]
<= download-file-base
[erp5-kernel]
<= download-file-base
[kernel-json]
<= download-file-base
[custom-js]
<= download-file-base
[instance-jupyter-notebook]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/template.cfg
mode = 0644
context =
key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key openssl_output openssl-output:openssl
key python_executable jupyter:python_executable
key jupyter_config_location jupyter-notebook-config:location
key jupyter_config_filename jupyter-notebook-config:filename
key jupyter_set_password_location jupyter-set-password:location
key jupyter_set_password_filename jupyter-set-password:filename
key erp5_kernel_location erp5-kernel:location
key erp5_kernel_filename erp5-kernel:filename
key kernel_json_location kernel-json:location
key kernel_json_filename kernel-json:filename
key custom_js_location custom-js:location
key custom_js_filename custom-js:filename
key monitor_template_rendered buildout:directory
[versions]
Pygments = 2.2.0
astor = 0.5
backports-abc = 0.5
backports.functools-lru-cache = 1.6.1
backports.shutil-get-terminal-size = 1.0.0
cycler = 0.10.0
ipykernel = 4.5.2
ipython = 5.3.0
ipython-genutils = 0.1.0
ipywidgets = 6.0.0
jupyter-client = 5.0.0
jupyter-core = 4.3.0
jupyterlab = 0.26.3
jupyterlab-launcher = 0.3.1
matplotlib = 2.1.2
mistune = 0.7.3
nbformat = 4.3.0
notebook = 4.4.1
pandas = 0.19.2
plone.recipe.command = 1.1
prompt-toolkit = 1.0.13
ptyprocess = 0.5.1
pyzmq = 16.0.2
scikit-learn = 0.18.1
seaborn = 0.7.1
simplegeneric = 0.8.1
slapos.recipe.template = 4.4
statsmodels = 0.8.0
terminado = 0.6
tornado = 4.4.2
traitlets = 4.3.2
widgetsnbextension = 2.0.0
# numpy >= 1.13.1 is required for numpy.core.multiarray
numpy = 1.13.1
# Required by:
# tornado==4.4.2
certifi = 2020.6.20
# Required by:
# notebook==4.3.2
# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
nbconvert = 4.1.0
# Required by:
# ipython==5.3.0
pathlib2 = 2.2.1
# Required by:
# statsmodels==0.8.0
patsy = 0.4.1
# Required by:
# ipython==5.3.0
pexpect = 4.2.1
# Required by:
# ipython==5.3.0
pickleshare = 0.7.4
# Required by:
# matplotlib==2.1.2
# pandas==0.19.2
python-dateutil = 2.6.0
# Required by:
# pathlib2==2.2.1
scandir = 1.5
# Required by:
# statsmodels==0.8.0
scipy = 0.19.0
# Required by:
# tornado==4.4.2
singledispatch = 3.4.0.3
# Required by:
# prompt-toolkit==1.0.13
wcwidth = 0.1.7
jupyter = 1.0.0
jupyter-console = 5.1.0
# Required by:
# jupyter==1.0.0
qtconsole = 4.3.0
et-xmlfile = 1.0.1
h5py = 2.7.1
mpmath = 1.0.0
openpyxl = 2.5.2
sympy = 1.1.1
xlrd = 1.1.0
# Required by:
# openpyxl==2.5.2
jdcal = 1.4
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance-jupyter-notebook]
filename = instance.cfg.in
md5sum = 1d5fe6cc4e48672ae7be1c223794a932
[jupyter-notebook-config]
filename = jupyter_notebook_config.py.jinja
md5sum = 720e90a829c63371696bc3009917a743
[jupyter-set-password]
filename = jupyter_set_password.cgi.jinja
md5sum = ac10fbcf790bd8e58750cfdd069812d2
[erp5-kernel]
filename = ERP5kernel.py
md5sum = 7d5309fe79afbcb455c0d8181b42e56c
[kernel-json]
filename = kernel.json.jinja
md5sum = 33547be93a67530165e079dc3ecfdac3
[custom-js]
filename = custom.js
md5sum = 147ccce38f7d7cf10664975d0dd80e33
// leave at least 2 line with only a star on it below, or doc generation fails
/**
*
*
* Placeholder for custom user javascript
* mainly to be overridden in profile/static/custom/custom.js
* This will always be an empty file in IPython
*
* User could add any javascript in the `profile/static/custom/custom.js` file.
* It will be executed by the ipython notebook at load time.
*
* Same thing with `profile/static/custom/custom.css` to inject custom css into the notebook.
*
*
* The object available at load time depend on the version of IPython in use.
* there is no guaranties of API stability.
*
* The example below explain the principle, and might not be valid.
*
* Instances are created after the loading of this file and might need to be accessed using events:
* define([
* 'base/js/namespace',
* 'base/js/events'
* ], function(IPython, events) {
* events.on("app_initialized.NotebookApp", function () {
* IPython.keyboard_manager....
* });
* });
*
* __Example 1:__
*
* Create a custom button in toolbar that execute `%qtconsole` in kernel
* and hence open a qtconsole attached to the same kernel as the current notebook
*
* define([
* 'base/js/namespace',
* 'base/js/events'
* ], function(IPython, events) {
* events.on('app_initialized.NotebookApp', function(){
* IPython.toolbar.add_buttons_group([
* {
* 'label' : 'run qtconsole',
* 'icon' : 'icon-terminal', // select your icon from http://fortawesome.github.io/Font-Awesome/icons
* 'callback': function () {
* IPython.notebook.kernel.execute('%qtconsole')
* }
* }
* // add more button here if needed.
* ]);
* });
* });
*
* __Example 2:__
*
* At the completion of the dashboard loading, load an unofficial javascript extension
* that is installed in profile/static/custom/
*
* define([
* 'base/js/events'
* ], function(events) {
* events.on('app_initialized.DashboardApp', function(){
* require(['custom/unofficial_extension.js'])
* });
* });
*
* __Example 3:__
*
* Use `jQuery.getScript(url [, success(script, textStatus, jqXHR)] );`
* to load custom script into the notebook.
*
* // to load the metadata ui extension example.
* $.getScript('/static/notebook/js/celltoolbarpresets/example.js');
* // or
* // to load the metadata ui extension to control slideshow mode / reveal js for nbconvert
* $.getScript('/static/notebook/js/celltoolbarpresets/slideshow.js');
*
*
* @module IPython
* @namespace IPython
* @class customjs
* @static
*/
$([Jupyter.events]).on('notebook_loaded.Notebook', function(){
var kernelname = Jupyter.notebook.kernel_selector.current_selection;
var display_text="<div class='output_subarea output_text output_result'>\
<pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
2. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
3. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
4. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
It would be better to set the reference to match with erp5 reference pattern.</br>\
5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
<p><u>OTHER USEFUL MAGICS</u> -</br>\
<b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
<b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
so changes to them would be disregarded and would throw an error.</br>\
<p><u>About classes, functions and global state on modules:</u></p>\
Your code is going to be executed by ERP5, which can have many nodes </br>\
and there is no guarantee that your code is always going to be executed by the same server.</br>\
This means that objects which cannot be stored in the ZODB, like functions, classes and modules </br>\
won't be available across nodes. To solve this issue, you need to use a special object </br>\
called 'environment' to store your global setup. This object was designed to hold global </br>\
state and restore it for each code cell. Example:</br></br>\
<b>def my_setup():</br>\
# import modules, define functions and classes</br>\
# and set global state on modules</br>\
# return dict of variables to be available in code cells</br>\
{'my_var': 1}</br>\
environment.define(my_setup, 'my custom setup')</b></br></br>\
After you execute this cell, the <b>my_setup</b> function will run before each of the</br>\
following cells and the <b>my_var</b> variable will be created and set to 1.</br></br>\
<b>WARNING:</b> it is not recommended to have too many setup functions in the environment, </br>\
because they will be executed in every code cell and can cause a substantial slow down.\
</pre></div>";
if (kernelname=="erp5"){
$('div#notebook-container').prepend(display_text);
}
});
[buildout]
parts =
instance
jupyter_notebook
read-knowledge0
publish-connection-parameter
jupyter-notebook-config
erp5-kernel
kernel-json
custom-js
monitor-base
extends =
{{ monitor_template_rendered }}/template-monitor.cfg
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slapconfiguration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
# ERP5 URL to use in Jupyter by default
# default value is empty - which means no default ERP5 URL
configuration.erp5-url =
[instance-parameter]
port = 8888
host = ${slapconfiguration:ipv6-random}
cert_file = ${generate-certificate:cert_file}
key_file = ${generate-certificate:key_file}
logfile = ${directory:log}/jupyter_notebook.log
notebook_dir = ${directory:notebook_dir}
[dynamic-jinja2-template-base]
recipe = slapos.recipe.template:jinja2
mode = 0644
[generate-certificate]
; TODO: there is a slapos recipe to generate certificates. Use it instead
recipe = plone.recipe.command
command =
if [ ! -e ${instance-parameter:key_file} ]
then
{{ openssl_output }} req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
-newkey rsa:1024 -keyout ${instance-parameter:key_file} \
-out ${instance-parameter:cert_file}
fi
update-command = ${:command}
cert_file = ${directory:etc}/jupyter_cert.crt
key_file = ${directory:etc}/jupyter_cert.key
[instance]
recipe = slapos.cookbook:wrapper
command-line =
{{ bin_directory }}/jupyter-lab
--no-browser
--ip=${instance-parameter:host}
--port=${instance-parameter:port}
--port-retries=0
--certfile=${instance-parameter:cert_file}
--keyfile=${instance-parameter:key_file}
--notebook-dir=${instance-parameter:notebook_dir}
--log-level="DEBUG"
wrapper-path = ${directory:service}/jupyter-lab
environment =
JUPYTER_PATH=${directory:jupyter_dir}
JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
LANG=C.UTF-8
[jupyter-notebook-config]
<= dynamic-jinja2-template-base
template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
mode = 0744
context =
raw config_cfg ${buildout:directory}/knowledge0.cfg
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
script = ${:etc}/run/
service = ${:etc}/service
log = ${:var}/log
notebook_dir = ${:var}/notebooks
# Add folders to explicitly define jupyter directory
jupyter_dir = ${:home}/jupyter
jupyter_config_dir = ${:jupyter_dir}/etc
jupyter_kernel_dir = ${:jupyter_dir}/kernels
jupyter_runtime_dir = ${:jupyter_dir}/runtime
jupyter_custom_dir = ${:jupyter_config_dir}/custom
jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
[jupyter_notebook]
# This part is called like this because knowledge0.write uses the part name for
# the section name in the config file.
recipe = slapos.cookbook:zero-knowledge.write
password =
filename = knowledge0.cfg
[read-knowledge0]
recipe = slapos.cookbook:zero-knowledge.read
filename = knowledge0.cfg
password =
[monitor-instance-parameter]
monitor-base-url = ${monitor-frontend-promise:url}
# In case you're using a developer instance you should edit these in:
# monitor-base-url = ${monitor-httpd-conf-parameter:url}
# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
instance-configuration =
raw jupyter-password ${read-knowledge0:password}
[publish-connection-parameter]
recipe = slapos.cookbook:publish.serialised
jupyter-classic-url = https://[${instance-parameter:host}]:${instance-parameter:port}/tree
url = ${:jupyter-classic-url}
jupyterlab-url = https://[${instance-parameter:host}]:${instance-parameter:port}/lab
[erp5-kernel]
recipe = slapos.cookbook:symbolic.link
link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
target-directory = ${directory:erp5_kernel_dir}
[kernel-json]
<= dynamic-jinja2-template-base
template = {{ kernel_json_location }}/{{ kernel_json_filename }}
rendered = ${directory:erp5_kernel_dir}/kernel.json
# Use python2.7 executable bin file for kernel config
context =
raw python_executable {{ python_executable }}
raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
key erp5_url slapconfiguration:configuration.erp5-url
raw display_name ERP5
raw language_name python
[custom-js]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:jupyter_custom_dir}
link-binary = {{ custom_js_location }}/custom.js
'''
This script initializes Jupyter's configuration such as passwords and other
things. It is run by IPython hence why it can use functions like get_config().
'''
import ConfigParser
import random
from notebook.auth import passwd
import os
def random_password(length = 10):
result = ""
for i in range(0, length):
result = result + chr(random.randint(0, 25) + ord('a'))
return result
knowledge_0 = '{{ config_cfg }}'
if not os.path.exists(knowledge_0):
print "Your software does <b>not</b> embed 0-knowledge. \
This interface is useless in this case</body></html>"
exit(0)
c = get_config()
parser = ConfigParser.ConfigParser()
parser.read(knowledge_0)
if not parser.has_section("jupyter_notebook"):
parser.add_section("jupyter_notebook")
if not parser.has_option("jupyter_notebook", "password") or \
parser.get("jupyter_notebook", "password") == "":
parser.set("jupyter_notebook", "password", random_password())
c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
with open(knowledge_0, 'w') as file:
parser.write(file)
#!{{ python_executable }}
import cgi
import cgitb
import ConfigParser
import os
import re
import subprocess
from IPython.lib import passwd
#cgitb.enable(display=0, logdir="/tmp/cgi.log")
cgitb.enable()
form = cgi.FieldStorage()
config_file = "{{ config_cfg }}"
if not os.path.exists(config_file):
print "Your software does <b>not</b> embed 0-knowledge. \
This interface is useless in this case</body></html>"
exit(0)
parser = ConfigParser.ConfigParser()
parser.read(config_file)
if not parser.has_section("jupyter_notebook"):
parser.add_section("jupyter_notebook")
if not parser.has_option("jupyter_notebook", "password"):
parser.set("jupyter_notebook", "password", "")
if "password" in form:
parser.set("jupyter_notebook", "password", passwd(form["password"].value))
# subprocess.call('{{ httpd_graceful }}')
# TODO: we should restart jupyter
with open(config_file, 'w') as file:
parser.write(file)
# TODO cleanup
print "<html><head>"
print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
print "<link rel=\"stylesheet\" href=\"static/style.css\">"
print "</head><body>"
print "<h1>Jupyter Notebook Password :</h1>"
print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
print """<div class="pure-control-group">
<label for="password">Password*:</label>
<input placeholder="Set your password" type="password" name="password" id="password"></br>
</div><div class="pure-control-group">
<label for="password">Verify Password*:</label>
<input placeholder="Verify password" type="password" name="password_2" id="password_2"></br>
</div><p id="validate-status" style="color:red"></p>
<div class="pure-controls">
<button id="register-button" type="submit" class="pure-button pure-button-primary" disabled>Access</button></div>
</form>
<script type="text/javascript" src="static/jquery-1.10.2.min.js"></script>
<script type="text/javascript" src="static/monitor-register.js"></script>
</body></html>
"""
{
"argv": [
"{{python_executable}}",
"{{kernel_dir}}",
"{{erp5_url}}",
"-f",
"{connection_file}"
],
"display_name": "{{display_name}}",
"language": "{{language_name}}",
"language_info": {"name": "python"}
}
......@@ -119,6 +119,29 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wsgi': False})}
class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 Jupyter notebook
"""
__partition_reference__ = 'jupyter'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'jupyter': {'enable': True}})}
def test_jupyter_notebook_is_reachable(self):
param_dict = self.getRootPartitionConnectionParameterDict()
self.assertEqual(
'https://[%s]:8888/tree' % self._ipv6_address,
param_dict['jupyter-url']
)
result = requests.get(
param_dict['jupyter-url'], verify=False, allow_redirects=False)
self.assertEqual(
[requests.codes.found, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
class TestBalancerPorts(ERP5InstanceTestCase):
"""Instantiate with two zope families, this should create for each family:
......
......@@ -75,7 +75,7 @@ class ERP5Kernel(Kernel):
self.title = None
# Allowed HTTP request code list for making request to erp5 from Kernel
# This list should be to used check status_code before making requests to erp5
self.allowed_HTTP_request_code_list = range(500, 511)
self.allowed_HTTP_request_code_list = list(range(500, 511))
# Append request code 200 in the allowed HTTP status code list
self.allowed_HTTP_request_code_list.append(200)
......
......@@ -19,7 +19,7 @@ md5sum = 1d5fe6cc4e48672ae7be1c223794a932
[jupyter-notebook-config]
filename = jupyter_notebook_config.py.jinja
md5sum = 720e90a829c63371696bc3009917a743
md5sum = 3da50c37760a42e42ae3f9d78d9ca449
[jupyter-set-password]
filename = jupyter_set_password.cgi.jinja
......@@ -27,7 +27,7 @@ md5sum = b8d31441780b524a7e52d1710dd78385
[erp5-kernel]
filename = ERP5kernel.py
md5sum = 7d5309fe79afbcb455c0d8181b42e56c
md5sum = da04b99b70b2e327c9e9b4cdd056098e
[kernel-json]
filename = kernel.json.jinja
......@@ -35,4 +35,4 @@ md5sum = 33547be93a67530165e079dc3ecfdac3
[custom-js]
filename = custom.js
md5sum = 40d938bb09261c65421a7725b40f87dc
md5sum = 295aeee765bc4d09bf0686021f32f72c
......@@ -81,16 +81,17 @@
* @static
*/
$([Jupyter.events]).on('notebook_loaded.Notebook', function(){
function prependERP5Help() {
var kernelname = Jupyter.notebook.kernel_selector.current_selection;
var display_text="<div class='output_subarea output_text output_result'>\
var display_text = "<div class='output_subarea output_text output_result'>\
<pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
2. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
3. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
4. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
2. <b>%erp5_url &lt;your_erp5_url&gt;/Base_executeJupyter</b></br>\
3. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
4. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
5. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
It would be better to set the reference to match with erp5 reference pattern.</br>\
5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
6. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
<p><u>OTHER USEFUL MAGICS</u> -</br>\
<b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
<b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
......@@ -114,8 +115,17 @@ $([Jupyter.events]).on('notebook_loaded.Notebook', function(){
<b>WARNING:</b> it is not recommended to have too many setup functions in the environment, </br>\
because they will be executed in every code cell and can cause a substantial slow down.\
</pre></div>";
if (kernelname=="erp5"){
$('div#notebook-container').prepend(display_text);
}
});
}
define([
'base/js/namespace',
'base/js/promises'
], function(Jupyter, promises) {
promises.notebook_loaded.then(function() {
prependERP5Help();
});
});
......@@ -2,7 +2,7 @@
This script initializes Jupyter's configuration such as passwords and other
things. It is run by IPython hence why it can use functions like get_config().
'''
import ConfigParser
import configparser
import random
from notebook.auth import passwd
import os
......@@ -16,13 +16,13 @@ def random_password(length = 10):
knowledge_0 = '{{ config_cfg }}'
if not os.path.exists(knowledge_0):
print "Your software does <b>not</b> embed 0-knowledge. \
This interface is useless in this case</body></html>"
print("Your software does <b>not</b> embed 0-knowledge. \
This interface is useless in this case</body></html>")
exit(0)
c = get_config()
parser = ConfigParser.ConfigParser()
parser = configparser.ConfigParser()
parser.read(knowledge_0)
if not parser.has_section("jupyter_notebook"):
......
......@@ -11,6 +11,9 @@ parts +=
jupyter-notebook-initialized-scripts
instance-jupyter-notebook
[python]
part = python3
[gcc]
# Always build GCC for Fortran (see openblas).
max_version = 0
......@@ -63,40 +66,49 @@ context =
key monitor_template_rendered buildout:directory
[versions]
Pygments = 2.2.0
Pygments = 2.7.2
astor = 0.5
async-generator = 1.10
backports-abc = 0.5
backports.functools-lru-cache = 1.6.1
backports.shutil-get-terminal-size = 1.0.0
bleach = 3.2.1
cycler = 0.10.0
ipykernel = 4.5.2
defusedxml = 0.6.0
entrypoints = 0.3
ipykernel = 5.3.4
ipython = 5.3.0
ipython-genutils = 0.1.0
ipywidgets = 6.0.0
jupyter-client = 5.0.0
jupyter-core = 4.3.0
jupyter-client = 6.1.7
jupyter-core = 4.7.0
jupyterlab = 0.26.3
jupyterlab-launcher = 0.3.1
jupyterlab-pygments = 0.1.2
matplotlib = 2.1.2
mistune = 0.7.3
nbformat = 4.3.0
notebook = 4.4.1
pandas = 0.19.2
mistune = 0.8.4
nest-asyncio = 1.4.3
nbclient = 0.5.1
nbformat = 5.0.8
notebook = 6.1.5
numpy = 1.14.6
pandas = 0.25.3
pandocfilters = 1.4.3
plone.recipe.command = 1.1
prompt-toolkit = 1.0.13
ptyprocess = 0.5.1
pyzmq = 16.0.2
scikit-learn = 0.18.1
pyzmq = 20.0.0
scikit-learn = 0.20.4
seaborn = 0.7.1
simplegeneric = 0.8.1
slapos.recipe.template = 4.4
statsmodels = 0.8.0
terminado = 0.6
tornado = 4.4.2
traitlets = 4.3.2
statsmodels = 0.11.1
testpath = 0.4.4
terminado = 0.9.1
tornado = 6.1
traitlets = 5.0.5
webencodings = 0.5.1
widgetsnbextension = 2.0.0
# numpy >= 1.13.1 is required for numpy.core.multiarray
numpy = 1.13.1
# Required by:
......@@ -104,41 +116,55 @@ numpy = 1.13.1
certifi = 2020.6.20
# Required by:
# notebook==4.3.2
# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
nbconvert = 4.1.0
# notebook==6.1.5
Send2Trash = 1.5.0
# Required by:
# notebook==6.1.5
argon2-cffi = 20.1.0
# Required by:
# notebook==6.1.5
nbconvert = 6.0.7
# Required by:
# ipython==5.3.0
pathlib2 = 2.2.1
# Required by:
# statsmodels==0.8.0
patsy = 0.4.1
# statsmodels==0.11.1
patsy = 0.5.1
# Required by:
# ipython==5.3.0
pexpect = 4.2.1
pexpect = 4.8.0
# Required by:
# ipython==5.3.0
pickleshare = 0.7.4
# Required by:
# matplotlib==2.1.2
# pandas==0.19.2
python-dateutil = 2.6.0
# notebook==6.1.5
prometheus-client = 0.9.0
# Required by:
# statsmodels==0.11.1
python-dateutil = 2.8.1
# Required by:
# pathlib2==2.2.1
scandir = 1.5
# Required by:
# statsmodels==0.8.0
scipy = 0.19.0
# statsmodels==0.11.1
pytz = 2020.4
# Required by:
# tornado==4.4.2
# statsmodels==0.11.1
scipy = 1.0.1
# Required by:
# tornado==6.1
singledispatch = 3.4.0.3
# Required by:
......
......@@ -26,7 +26,7 @@
##############################################################################
import httplib
import http.client
import json
import os
import requests
......@@ -60,7 +60,7 @@ class TestJupyter(InstanceTestCase):
result = requests.get(
connection_dict['url'], verify=False, allow_redirects=False)
self.assertEqual(
[httplib.FOUND, True, '/login?next=%2Ftree'],
[http.client.FOUND, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
......@@ -68,7 +68,7 @@ class TestJupyter(InstanceTestCase):
connection_dict['jupyter-classic-url'],
verify=False, allow_redirects=False)
self.assertEqual(
[httplib.FOUND, True, '/login?next=%2Ftree'],
[http.client.FOUND, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
......@@ -76,6 +76,6 @@ class TestJupyter(InstanceTestCase):
connection_dict['jupyterlab-url'],
verify=False, allow_redirects=False)
self.assertEqual(
[httplib.FOUND, True, '/login?next=%2Flab'],
[http.client.FOUND, True, '/login?next=%2Flab'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
......@@ -13,6 +13,7 @@ extra-eggs +=
[template]
extra =
helloworld ${slapos.test.helloworld-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
plantuml ${slapos.test.plantuml-setup:setup}
powerdns ${slapos.test.powerdns-setup:setup}
......
......@@ -271,7 +271,6 @@ extra =
re6stnet ${slapos.test.re6stnet-setup:setup}
seleniumserver ${slapos.test.seleniumserver-setup:setup}
jstestnode ${slapos.test.jstestnode-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
nextcloud ${slapos.test.nextcloud-setup:setup}
turnserver ${slapos.test.turnserver-setup:setup}
theia ${slapos.test.theia-setup:setup}
......
......@@ -56,8 +56,8 @@ extends =
../../component/pylint/buildout.cfg
../../component/perl-Image-ExifTool/buildout.cfg
../../component/wendelin.core/buildout.cfg
../../component/jupyter-py2/buildout.cfg
../../stack/caucase/buildout.cfg
../../software/jupyter/software.cfg
../../software/neoppod/software-common.cfg
# keep neoppod extends last
......
......@@ -191,7 +191,7 @@ scandir = 1.10.0
setuptools-dso = 1.7
rubygemsrecipe = 0.3.0
six = 1.12.0
slapos.cookbook = 1.0.167
slapos.cookbook = 1.0.171
slapos.core = 1.6.3
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment