Jupyter Kernel: added automatic error rendering

@kirr, @Tyagov and @tatuya, please review.

Now the ERP5 Jupyter kernel automatically renders errors that happens in the
user-side code. Errors are captured during the AST tree creation (to be able to
detect syntax errors) and at execution time. The current transaction is
automatically aborted on error detection.

/reviewed-on nexedi/erp5!85

bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterCompile.py

@@ -8,6 +8,9 @@ import sys
 import ast
 import types
 import inspect
+import traceback
+
+import transaction
 
 mime_type = 'text/plain'
 # IPython expects 2 status message - 'ok', 'error'
@@ -79,7 +82,13 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
   if jupyter_code:
   
     # Create ast parse tree
-    ast_node = ast.parse(jupyter_code)
+    try:
+      ast_node = ast.parse(jupyter_code)
+    except Exception as e:
+      # It's not necessary to abort the current transaction here 'cause the 
+      # user's code wasn't executed at all yet.
+      return getErrorMessageForException(self, e, local_variable_dict)
+      
     # Get the node list from the parsed tree
     nodelist = ast_node.body
 
@@ -116,13 +125,35 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
       for node in to_run_exec:
         mod = ast.Module([node])
         code = compile(mod, '<string>', "exec")
-        exec(code, g, g)
+        try:
+          exec(code, g, g)
+        except Exception as e:
+          # Abort the current transaction. As a consequence, the notebook lines
+          # are not added if an exception occurs.
+          #
+          # TODO: store which notebook line generated which exception.
+          #
+          transaction.abort()
+          # Clear the portal cache from previous transaction
+          self.getPortalObject().portal_caches.clearAllCache()
+          return getErrorMessageForException(self, e, local_variable_dict)
 
       # Execute the interactive nodes with 'single' mode
       for node in to_run_interactive:
         mod = ast.Interactive([node])
         code = compile(mod, '<string>', "single")
-        exec(code, g, g)
+        try:
+          exec(code, g, g)
+        except Exception as e:
+          # Abort the current transaction. As a consequence, the notebook lines
+          # are not added if an exception occurs.
+          #
+          # TODO: store which notebook line generated which exception.
+          #
+          transaction.abort()
+          # Clear the portal cache from previous transaction
+          self.getPortalObject().portal_caches.clearAllCache()
+          return getErrorMessageForException(self, e, local_variable_dict)
 
       # Letting the code fail in case of error while executing the python script/code
       # XXX: Need to be refactored so to acclimitize transactions failure as well as
@@ -168,6 +199,24 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
 
   return result
 
+def getErrorMessageForException(self, exception, local_variable_dict):
+  '''
+    getErrorMessageForException receives an Expcetion object and a context for
+    code execution (local_variable_dict) and will return a dict as Jupyter
+    requires for error rendering.
+  '''
+  etype, value, tb = sys.exc_info()
+  traceback_text = traceback.format_exc().split('\n')[:-1]
+  return {
+    'status': 'error',
+    'result_string': None,
+    'local_variable_dict': local_variable_dict,
+    'mime_type': 'text/plain',
+    'evalue': str(value),
+    'ename': exception.__class__.__name__,
+    'traceback': traceback_text
+  }
+
 def AddNewLocalVariableDict(self):
   """
   Function to add a new Local Variable for a Data Notebook
@@ -323,4 +372,4 @@ def getError(self, previous=1):
   evalue = unicode(error['value'])
   tb_list = [l+'\n' for l in error['tb_text'].split('\n')]
 
-  return None
+  return None
\ No newline at end of file

bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterCompile.xml

@@ -46,12 +46,14 @@
             <key> <string>text_content_warning_message</string> </key>
             <value>
               <tuple>
-                <string>W: 55,  2: Using the global statement (global-statement)</string>
-                <string>W: 95,  8: Use of exec (exec-used)</string>
-                <string>W:119,  8: Use of exec (exec-used)</string>
-                <string>W:125,  8: Use of exec (exec-used)</string>
-                <string>W:220,  4: Using the global statement (global-statement)</string>
-                <string>W:320,  2: Using the global statement (global-statement)</string>
+                <string>W: 58,  2: Using the global statement (global-statement)</string>
+                <string>W:104,  8: Use of exec (exec-used)</string>
+                <string>W:129, 10: Use of exec (exec-used)</string>
+                <string>W:146, 10: Use of exec (exec-used)</string>
+                <string>W:208,  2: Unused variable \'etype\' (unused-variable)</string>
+                <string>W:208, 16: Unused variable \'tb\' (unused-variable)</string>
+                <string>W:269,  4: Using the global statement (global-statement)</string>
+                <string>W:369,  2: Using the global statement (global-statement)</string>
               </tuple>
             </value>
         </item>

bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_renderAsHtml.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_function</string> </key>
+            <value> <string>renderAsHtml</string> </value>
+        </item>
+        <item>
+            <key> <string>_module</string> </key>
+            <value> <string>JupyterCompile</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>Base_renderAsHtml</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_data_notebook/TestTemplateItem/portal_components/test.erp5.testExecuteJupyter.py

@@ -32,7 +32,7 @@ from Products.ERP5Type.tests.utils import createZODBPythonScript, removeZODBPyth
 import time
 import json
 import base64
-import transaction
+
 
 class TestExecuteJupyter(ERP5TypeTestCase):
   
@@ -100,21 +100,25 @@ print an_undefined_variable
 portal = context.getPortalObject()
 portal.%s()
 """%script_id
-
+    
     # Make call to Base_runJupyter to run the jupyter code which is making
-    # a call to the newly created ZODB python_script and assert if the call raises
-    # NameError as we are sending an invalid python_code to it
-    self.assertRaises(
-                      NameError,
-                      portal.Base_runJupyter,
-                      jupyter_code=jupyter_code,
-                      old_local_variable_dict=portal.Base_addLocalVariableDict()
-                      )
-    # Abort the current transaction of test so that we can proceed to new one
-    transaction.abort()
-    # Clear the portal cache from previous transaction
-    self.portal.portal_caches.clearAllCache()
-    # Remove the ZODB python script created above
+    # a call to the newly created ZODB python_script and assert if the call 
+    # processes correctly the NameError as we are sending an invalid 
+    # python_code to it.
+    # 
+    result = portal.Base_runJupyter(
+      jupyter_code=jupyter_code, 
+      old_local_variable_dict=portal.Base_addLocalVariableDict()
+    )
+    
+    self.assertEquals(result['ename'], 'NameError')
+    self.assertEquals(result['result_string'], None)
+    
+    # There's no need to abort the current transaction. The error handling code
+    # should be responsible for this, so we check the script's title
+    script_title = script_container.JupyterCompile_errorResult.getTitle()
+    self.assertNotEqual(script_title, new_test_title)
+    
     removeZODBPythonScript(script_container, script_id)
 
     # Test that calling Base_runJupyter shouldn't change the context Title
@@ -248,13 +252,14 @@ portal.%s()
     reference = 'Test.Notebook.ExecuteJupyterErrorHandling %s' % time.time()
     title = 'Test NB Title %s' % time.time()
 
-    self.assertRaises(
-                      NameError, 
-                      portal.Base_executeJupyter,
-                      title=title,
-                      reference=reference,
-                      python_expression=python_expression
-                      )
+    result = json.loads(portal.Base_executeJupyter(
+      title=title, 
+      reference=reference, 
+      python_expression=python_expression
+    ))
+    
+    self.assertEquals(result['ename'], 'NameError')
+    self.assertEquals(result['code_result'], None)
 
   def testBaseExecuteJupyterSaveActiveResult(self):
     """
@@ -429,4 +434,4 @@ context.Base_displayImage(image_object=image)
       reference=reference,
       python_expression=jupyter_code2
       )
-    self.assertEquals(json.loads(result)['code_result'].rstrip(), 'sys')
+    self.assertEquals(json.loads(result)['code_result'].rstrip(), 'sys')
\ No newline at end of file