Jupyter: Hook ERP5 Kernel

ERP5 kernel basic info/workflow:

1. User enters code on notebook cell and executes
2. Code is sent to kernel via websockets
3. Kernel sends request to ERP5
4. Code is executed by ERP5 and the result is returned back via request.
5. Result is received and rendered on the notebook frontend.
6. Other message formats such as error and status are also conveyed by the Kernel.

[ kirr: in IPython notebook speak kernel is something that allows IPython
  notebook server side to talk to execution backend. ERP5 kernel is a thing that
  allows ipython notbook to talk to ERP5 (with help on-ERP5-server special bt5
  installed which accepts and executes commands).

  The bt5 to handle notebook calls on ERP5 side - erp5-data-notebook - is
  proposed to be merged into erp5.git on nexedi/erp5!29 ]

/initially-reviewed-by @kirr, @Tyagov  (in a lot of places, last time on nexedi/slapos!33)

software/ipython_notebook/instance.cfg.in

@@ -23,6 +23,9 @@ parts =
 ## Monitor for ipython
   monitor-current-log-access
   monitor-deploy-set-password-cgi
+  erp5-kernel
+  kernel-json
+  custom-js
 
 extends = {{ monitor_template }}
 
@@ -122,8 +125,34 @@ notebook_dir = ${:var}/notebooks
 # Add folders to explicitly define ipython directory
 ipython_dir = ${:home}/ipython
 ipython_kernel_dir = ${:ipython_dir}/kernels
+erp5_kernel_dir = ${:ipython_kernel_dir}/ERP5
 
 [publish-connection-parameter]
 recipe = slapos.cookbook:publish
 url = https://[${instance-parameter:host}]:${instance-parameter:port}
 monitor_url = ${monitor-parameters:url}
+
+[erp5-kernel]
+<= dynamic-jinja2-template-base
+template = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
+rendered = ${directory:erp5_kernel_dir}/ERP5kernel.py
+# Use ipython as executable python as we'll be needing requests library in kernel
+context =
+  raw python_executable {{ bin_directory }}/ipython
+
+[kernel-json]
+<= dynamic-jinja2-template-base
+template = {{ kernel_json_location }}/{{ kernel_json_filename }}
+rendered = ${directory:erp5_kernel_dir}/kernel.json
+# Use python2.7 executable bin file for kernel config
+context =
+  raw python_executable {{ python_executable }}
+  key kernel_dir erp5-kernel:rendered
+  raw display_name ERP5
+  raw language_name python
+
+[custom-js]
+<= dynamic-jinja2-template-base
+template = {{ custom_js_location }}/{{ custom_js_filename }}
+rendered = ${directory:ipython_dir}/profile_default/static/custom/custom.js
+mode = 0744

software/ipython_notebook/software.cfg

@@ -18,6 +18,7 @@ parts =
 ; In the ipython notebook software, we use more eggs than in the minimal
 ; ipython notebook component
 eggs +=
+  requests
   ${scipy:egg}
   ${pandas:egg}
   ${scikit-learn:egg}
@@ -39,22 +40,44 @@ md5sum = a5bc4ee8539109d1de7ab33b4c2c97ea
 filename = ipython_set_password.cgi.jinja
 md5sum = d7d4a7e19d55bf14007819258bf42100
 
+[erp5-kernel]
+<= download-file-base
+filename = ERP5kernel.py.jinja
+md5sum = a6a0ad790d100167c42c354fe5744e11
+
+[kernel-json]
+<= download-file-base
+filename = kernel.json.jinja
+md5sum = ab6e78ea20855e07d388b5b86d1770fe
+
+[custom-js]
+<= download-file-base
+filename = custom.js.jinja
+md5sum = 0e8262d04a6dafbc1b77d95aea2192bc
+
 [instance]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance.cfg.in
 rendered = ${buildout:directory}/template.cfg
 mode = 0644
-md5sum = e67b3cc33db32993a70895173e7b0150
+md5sum = 6bc65c138215dc995e4721cc74bdf7ef
 context =
   key bin_directory buildout:bin-directory
   key develop_eggs_directory buildout:develop-eggs-directory
   key eggs_directory buildout:eggs-directory
   key monitor_template monitor-template:output
   key openssl_output openssl-output:openssl
+  key python_executable python2.7:executable
   key ipython_notebook_config_location ipython-notebook-config:location
   key ipython_notebook_config_filename ipython-notebook-config:filename
   key ipython_notebook_set_password_location ipython-notebook-set-password:location
   key ipython_notebook_set_password_filename ipython-notebook-set-password:filename
+  key erp5_kernel_location erp5-kernel:location
+  key erp5_kernel_filename erp5-kernel:filename
+  key kernel_json_location kernel-json:location
+  key kernel_json_filename kernel-json:filename
+  key custom_js_location custom-js:location
+  key custom_js_filename custom-js:filename
 
 [versions]
 PyRSS2Gen = 1.1
@@ -73,6 +96,7 @@ scipy = 0.15.1
 slapos.recipe.template = 2.8
 terminado = 0.5
 tornado = 4.2
+requests = 2.7.0
 
 # Required by:
 # tornado==4.2

software/ipython_notebook/template/ERP5kernel.py.jinja

+#!{{ python_executable }}
+
+from IPython.kernel.zmq.kernelbase import Kernel
+from IPython.kernel.zmq.kernelapp import IPKernelApp
+
+from IPython.core.display import HTML
+
+import requests
+import json
+
+# erp5_url from buildout
+# TODO: Uncomment after adding automated installation of erp5-data-notebook bt5
+# url = ""
+# url  = "%s/erp5/Base_executeJupyter"%url
+
+class MagicInfo:
+  """
+  Magics definition structure.
+  Initializes a new MagicInfo class with specific paramters to identify a magic.
+  """
+  def __init__(self, magic_name, variable_name, send_request, request_reference, display_message):
+    self.magic_name = magic_name
+    self.variable_name = variable_name
+    self.send_request = send_request
+    self.request_reference = request_reference
+    self.display_message = display_message
+
+# XXX: New magics to be added here in the dictionary.
+# In this dictionary,
+# key = magic_name,
+# value = MagicInfo Structure corresponding to the magics
+# Different parameters of the structures are :-
+# magics_name(str) = Name which would be used on jupyter frontend
+# variable_name(str) = Name of variable on which magic would be set in kernel
+# send_request(boolean) = Magics for which requests to erp5 backend need to be made
+# request_reference(boolean) = Request for notebook references(and titles) from erp5
+# display_message(boolean) = If the magics need to display message after
+#                             making request. Useful for magics which do get some
+#                             useful content from erp5 backend and need to display
+
+MAGICS = {
+  'erp5_user': MagicInfo('erp5_user', 'user', True, False, True),
+  'erp5_password': MagicInfo('erp5_password', 'password', True, False, True),
+  'erp5_url': MagicInfo('erp5_url', 'url', True, False, True),
+  'notebook_set_reference': MagicInfo('notebook_set_reference', 'reference', True, False, True),
+  'notebook_set_title': MagicInfo('notebook_set_title', 'title', False, False, True),
+  'my_notebooks': MagicInfo('my_notebooks', '', True, True, False)
+}
+
+class ERP5Kernel(Kernel):
+  """
+  Jupyter Kernel class to interact with erp5 backend for code from frontend.
+  To use this kernel with erp5, user need to install 'erp5_data_notebook' bt5 
+  Also, handlers(aka magics) starting with '%' are predefined.
+
+  Each request to erp5 for code execution requires erp5_user, erp5_password
+  and reference of the notebook.
+  """
+
+  implementation = 'ERP5'
+  implementation_version = '1.0'
+  language = 'ERP5'
+  language_version = '0.1'
+  language_info = {'mimetype': 'text/plain', 'name':'python'}
+  banner = "ERP5 integration with ipython notebook"
+
+  def __init__(self, user=None, password=None, url=None, status_code=None,
+              *args, **kwargs):
+    super(ERP5Kernel, self).__init__(*args, **kwargs)
+    self.user = user
+    self.password = password
+    # Use URL provided by buildout during initiation
+    # It can later be overridden
+    self.url = url
+    self.status_code = status_code
+    self.reference = None
+    self.title = None
+    # Allowed HTTP request code list for making request to erp5 from Kernel
+    # This list should be to used check status_code before making requests to erp5
+    self.allowed_HTTP_request_code_list = range(500, 511)
+    # Append request code 200 in the allowed HTTP status code list
+    self.allowed_HTTP_request_code_list.append(200)
+
+  def display_response(self, response=None):
+    """
+      Dispays the stream message response to jupyter frontend.
+    """
+    if response:
+      stream_content = {'name': 'stdout', 'text': response}
+      self.send_response(self.iopub_socket, 'stream', stream_content)
+
+  def set_magic_attribute(self, magic_info=None, code=None):
+    """
+      Set attribute for magic which are necessary for making requests to erp5.
+      Catch errors and display message. Since user is in contact with jupyter
+      frontend, so its better to catch exceptions and dispaly messages than to
+      let them fail in backend and stuck the kernel.
+      For a making a request to erp5, we need -
+      erp5_url, erp5_user, erp5_password, notebook_set_reference
+    """
+    # Set attributes only for magic who do have any varible to set value to
+    if magic_info.variable_name:
+
+      try:
+        # Get the magic value recived via code from frontend
+        magic_value = code.split()[1]
+        # Set magic_value to the required attribute
+        setattr(self, magic_info.variable_name , magic_value)
+        self.response = 'Your %s is %s. '%(magic_info.magic_name, magic_value)
+
+      # Catch exception while setting attribute and set message in response
+      except AttributeError:
+        self.response = 'Please enter %s magic value'%magic_info.variable_name
+
+      # Catch IndexError while getting magic_value and set message in response object
+      except IndexError:
+        self.response = 'Empty value for %s magic'%magic_info.variable_name
+
+      # Catch all other exceptions and set error_message in response object
+      # XXX: Might not be best way, but its better to display error to the user
+      # via notebook frontend than to fail in backend and stuck the Kernel without
+      # any failure message to user.
+      except Exception as e:
+        self.response = str(e)
+
+      # Display the message/response from this fucntion before moving forward so
+      # as to keep track of the status
+      self.display_response(response=self.response)
+
+  def check_required_attributes(self):
+    """
+      Check if the required attributes for making a request are already set or not.
+      Display message to frontend to provide with the values in case they aren't.
+      This function can be called anytime to check if the attributes are set. The
+      output result will be in Boolean form.
+      Also, in case any of attribute is not set, call to display_response would be
+      made to ask user to enter value.
+    """
+    result_list = []
+    required_attributes  = ['url', 'password', 'user', 'reference']
+
+    # Set response to empty so as to flush the response set by some earlier fucntion call
+    self.response = ''
+
+    # Loop to check if the attributes are set
+    for attribute in required_attributes:
+      if getattr(self, attribute):
+        result_list.append(True)
+      else:
+        # Set response/message for attributes which aren't set
+        self.response = '\nPlease enter %s in next cell. '%attribute
+        result_list.append(False)
+
+    # Compare result_list to get True for all True results and False for any False result 
+    check_attributes = all(result_list)
+
+    # Display response to frontend before moving forward
+    self.display_response(response=self.response)
+
+    return check_attributes
+
+  def make_erp5_request(self, request_reference=False, display_message=True,
+                        code=None, message=None, title=None, *args, **kwargs):
+    """
+      Function to make request to erp5 as per the magics.
+      Should return the response json object.
+    """
+
+    try:
+      erp5_request = requests.get(
+        self.url,
+        verify=False,
+        auth=(self.user, self.password),
+        params={
+          'python_expression': code,
+          'reference': self.reference,
+          'title': self.title,
+          'request_reference': request_reference,
+          },
+      )
+
+      # Set value for status_code for self object which would later be used to
+      # dispaly response after statement check
+      self.status_code = erp5_request.status_code
+
+      # Dispaly error response in case the request give any other status
+      # except 200 and 5xx(which is for errors on server side)
+      if self.status_code not in self.allowed_HTTP_request_code_list:
+        self.response = '''Error code %s on request to ERP5,\n
+        check credentials or ERP5 family URL'''%self.status_code
+      else:
+        # Set value of self.response to the given value in case response from function
+        # call. In all other case, response should be the content from request
+        if display_message and message:
+          self.response = message
+        else:
+          self.response = erp5_request.content
+
+    except requests.exceptions.RequestException as e:
+      self.response = str(e)
+
+  def do_execute(self, code, silent, store_history=True, user_expressions=None,
+                  allow_stdin=False):
+    """
+      Validate magic and call functions to make request to erp5 backend where
+      the code is being executed and response is sent back which is then sent
+      to jupyter frontend.
+    """
+    # By default, take the status of response as 'ok' so as show the responses
+    # for erp5_url and erp5_user on notebook frontend as successful response.
+    status = 'ok'
+
+    if not silent:
+      # Remove spaces and newlines from both ends of code
+      code = code.strip()
+
+      if code.startswith('%'):
+          # No need to try-catch here as its already been taken that the code
+          # starts-with '%', so we'll get magic_name, no matter what be after '%'
+          magic_name = code.split()[0][1:]
+          magics_name_list = [magic.magic_name for magic in MAGICS.values()]
+
+          # Check validation of magic
+          if magic_name and magic_name in magics_name_list:
+
+            # Get MagicInfo object related to the magic
+            magic_info = MAGICS.get(magic_name)
+
+            # Function call to set the required magics
+            self.set_magic_attribute(magic_info=magic_info, code=code)
+
+            # Call to check if the required_attributes are set
+            checked_attribute = self.check_required_attributes()
+            if checked_attribute and magic_info.send_request:
+              # Call the function to send request to erp5 with the arguments given
+              self.make_erp5_request(message='\nPlease proceed',
+              request_reference=magic_info.request_reference,
+              display_message=magic_info.display_message)
+
+              # Display response from erp5 request for magic
+              # Since this response would be either success message or failure
+              # error message, both of which are string type, so, we can simply
+              # display the stream response.
+              self.display_response(response=self.response)
+
+          else:
+            # Set response if there is no magic or the magic name is not in MAGICS
+            self.response = 'Invalid Magics'
+            self.display_response(response=self.response)
+
+      else:
+        # Check for status_code before making request to erp5 and make request in
+        # only if the status_code is in the allowed_HTTP_request_code_list
+        if self.status_code in self.allowed_HTTP_request_code_list:
+          self.make_erp5_request(code=code)
+
+          # For 200 status_code, Kernel will receive predefined format for data
+          # from erp5 which is either json of result or simple result string
+          if self.status_code == 200:
+            mime_type = 'text/plain'
+            try:
+              content = json.loads(self.response)
+              code_result = content['code_result']
+            # Display to frontend the error message for content status as 'error'
+              if content['status']=='error':
+                reply_content = {
+                  'status': 'error',
+                  'execution_count': self.execution_count,
+                  'ename': content['ename'],
+                  'evalue': content['evalue'],
+                  'traceback': content['traceback']}
+                self.send_response(self.iopub_socket, u'error', reply_content)
+                return reply_content
+            # Catch exception for content which isn't json
+            except ValueError:
+              content = self.response
+              code_result = content
+          # Display basic error message to frontend in case of error on server side
+          else:
+            self.make_erp5_request(code=code)
+            code_result = "Error at Server Side"
+            mime_type = 'text/plain'
+
+        # For all status_code except allowed_HTTP_response_code_list show unauthorized message
+        else:
+          code_result = 'Unauthorized access'
+          mime_type = 'text/plain'
+
+        data = {
+          'data': {mime_type: code_result},
+          'metadata': {}
+        }
+        self.send_response(self.iopub_socket, 'display_data', data)
+
+    reply_content = {
+      'status': status,
+      # The base class increments the execution count
+      'execution_count': self.execution_count,
+      'payload': [],
+      'user_expressions': {},
+    }
+    return reply_content
+
+if __name__ == '__main__':
+  IPKernelApp.launch_instance(kernel_class=ERP5Kernel)

software/ipython_notebook/template/custom.js.jinja

+// leave at least 2 line with only a star on it below, or doc generation fails
+/**
+ *
+ *
+ * Placeholder for custom user javascript
+ * mainly to be overridden in profile/static/custom/custom.js
+ * This will always be an empty file in IPython
+ *
+ * User could add any javascript in the `profile/static/custom/custom.js` file.
+ * It will be executed by the ipython notebook at load time.
+ *
+ * Same thing with `profile/static/custom/custom.css` to inject custom css into the notebook.
+ *
+ *
+ * The object available at load time depend on the version of IPython in use.
+ * there is no guaranties of API stability.
+ *
+ * The example below explain the principle, and might not be valid.
+ *
+ * Instances are created after the loading of this file and might need to be accessed using events:
+ *     define([
+ *        'base/js/namespace',
+ *        'base/js/events'
+ *     ], function(IPython, events) {
+ *         events.on("app_initialized.NotebookApp", function () {
+ *             IPython.keyboard_manager....
+ *         });
+ *     });
+ *
+ * __Example 1:__
+ *
+ * Create a custom button in toolbar that execute `%qtconsole` in kernel
+ * and hence open a qtconsole attached to the same kernel as the current notebook
+ *
+ *    define([
+ *        'base/js/namespace',
+ *        'base/js/events'
+ *    ], function(IPython, events) {
+ *        events.on('app_initialized.NotebookApp', function(){
+ *            IPython.toolbar.add_buttons_group([
+ *                {
+ *                    'label'   : 'run qtconsole',
+ *                    'icon'    : 'icon-terminal', // select your icon from http://fortawesome.github.io/Font-Awesome/icons
+ *                    'callback': function () {
+ *                        IPython.notebook.kernel.execute('%qtconsole')
+ *                    }
+ *                }
+ *                // add more button here if needed.
+ *                ]);
+ *        });
+ *    });
+ *
+ * __Example 2:__
+ *
+ * At the completion of the dashboard loading, load an unofficial javascript extension
+ * that is installed in profile/static/custom/
+ *
+ *    define([
+ *        'base/js/events'
+ *    ], function(events) {
+ *        events.on('app_initialized.DashboardApp', function(){
+ *            require(['custom/unofficial_extension.js'])
+ *        });
+ *    });
+ *
+ * __Example 3:__
+ *
+ *  Use `jQuery.getScript(url [, success(script, textStatus, jqXHR)] );`
+ *  to load custom script into the notebook.
+ *
+ *    // to load the metadata ui extension example.
+ *    $.getScript('/static/notebook/js/celltoolbarpresets/example.js');
+ *    // or
+ *    // to load the metadata ui extension to control slideshow mode / reveal js for nbconvert
+ *    $.getScript('/static/notebook/js/celltoolbarpresets/slideshow.js');
+ *
+ *
+ * @module IPython
+ * @namespace IPython
+ * @class customjs
+ * @static
+ */
+
+$([IPython.events]).on('notebook_loaded.Notebook', function(){
+  var kernelname = IPython.notebook.kernel_selector.current_selection;
+  var display_text="<div class='output_subarea output_text output_result'>\
+  <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
+  1. Add <b>%erp5_url &lt;your_erp5_url&gt;/erp5/Base_executeJupyter</b>.</br>\
+  Make sure you have 'erp5_data_notebook' business template installed in your erp5</br>\
+  2. <b>%notebook_set_reference &lt;your_notebook_refernce&gt;</b></br>\
+  It would be better to set the reference to match with erp5 reference pattern.</br>\
+  3. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
+  4. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
+  5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
+  <p><u>OTHER USEFUL MAGICS</u> -</br>\
+  <b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
+  NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
+  so changes to them would be disregarded and would throw an error.</br>\
+  </pre></div>";
+  if (kernelname=="erp5"){
+  $('div#notebook-container').prepend(display_text);
+  }
+  });
\ No newline at end of file

software/ipython_notebook/template/kernel.json.jinja

+{
+  "argv": [
+    "{{python_executable}}",
+    "{{kernel_dir}}",
+    "-f",
+    "{connection_file}"
+  ],
+  "display_name": "{{display_name}}",
+  "language": "{{language_name}}",
+  "language_info": {"name": "python"}
+}