Skip to content

erp5
erp5
Commit d1085bd6 authored by Ivan Tyagov's avatar Ivan Tyagov
Browse files
Options

Jupyter improvements 

This fixes number of Jupyter bugs.

Tests passing here: 

https://nexedi.erp5.net/test_result_module/20161117-147BEEA9

/reviewed-on !198
parents d4afa48c 3fa7dd84
Hide whitespace changes
Inline Side-by-side
Showing with 77 additions and 73 deletions
bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterCompile.py
View file @ d1085bd6
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from matplotlib.figure import Figure
from IPython.core.display import DisplayObject
from IPython.lib.display import IFrame
from cStringIO import StringIO from cStringIO import StringIO
import cPickle
from erp5.portal_type import Image from erp5.portal_type import Image
from types import ModuleType from types import ModuleType
from ZODB.serialize import ObjectWriter from ZODB.serialize import ObjectWriter
import cPickle
import sys import sys
import traceback import traceback
import ast import ast
import base64 import base64
import json import json
import transaction import transaction
import Acquisition import Acquisition
import astor import astor
from Products.ERP5Type.Log import log
from matplotlib.figure import Figure def Base_executeJupyter(self, python_expression=None, reference=None, \
from IPython.core.display import DisplayObject title=None, request_reference=False, **kw):
from IPython.lib.display import IFrame
def Base_executeJupyter(self, python_expression=None, reference=None, title=None, request_reference=False, **kw):
# Check permissions for current user and display message to non-authorized user # Check permissions for current user and display message to non-authorized user
if not self.Base_checkPermission('portal_components', 'Manage Portal'): if not self.Base_checkPermission('portal_components', 'Manage Portal'):
return "You are not authorized to access the script" return "You are not authorized to access the script"
# Convert the request_reference argument string to their respeced boolean values # Convert the request_reference argument string to their respeced boolean values
request_reference = {'True': True, 'False': False}.get(request_reference, False) request_reference = {'True': True, \
'False': False}.get(request_reference, False)
# Return python dictionary with title and reference of all notebooks # Return python dictionary with title and reference of all notebooks
# for request_reference=True # for request_reference=True
if request_reference: if request_reference:
data_notebook_list = self.portal_catalog(portal_type='Data Notebook') data_notebook_list = self.portal_catalog(portal_type='Data Notebook')
notebook_detail_list = [{'reference': obj.getReference(), 'title': obj.getTitle()} for obj in data_notebook_list] notebook_detail_list = [{'reference': obj.getReference(), \
'title': obj.getTitle()} for obj in data_notebook_list]
return notebook_detail_list return notebook_detail_list
if not reference: if not reference:
...@@ -46,23 +44,22 @@ def Base_executeJupyter(self, python_expression=None, reference=None, title=None ...@@ -46,23 +44,22 @@ def Base_executeJupyter(self, python_expression=None, reference=None, title=None
python_expression = '' python_expression = ''
# Get Data Notebook with the specific reference # Get Data Notebook with the specific reference
data_notebook = self.portal_catalog.getResultValue(portal_type='Data Notebook', data_notebook = self.portal_catalog.getResultValue(
reference=reference) portal_type='Data Notebook',
reference=reference)
# Create new Data Notebook if reference doesn't match with any from existing ones # Create new Data Notebook if reference doesn't match with any from existing ones
if not data_notebook: if not data_notebook:
notebook_module = self.getDefaultModule(portal_type='Data Notebook') notebook_module = self.getDefaultModule(portal_type='Data Notebook')
data_notebook = notebook_module.DataNotebookModule_addDataNotebook( data_notebook = notebook_module.DataNotebookModule_addDataNotebook(
title=title, title=title,
reference=reference, reference=reference,
batch_mode=True batch_mode=True)
)
# Add new Data Notebook Line to the Data Notebook # Add new Data Notebook Line to the Data Notebook
data_notebook_line = data_notebook.DataNotebook_addDataNotebookLine( data_notebook_line = data_notebook.DataNotebook_addDataNotebookLine(
notebook_code=python_expression, notebook_code=python_expression,
batch_mode=True batch_mode=True)
)
# Gets the context associated to the data notebook being used # Gets the context associated to the data notebook being used
old_notebook_context = data_notebook.getNotebookContext() old_notebook_context = data_notebook.getNotebookContext()
...@@ -81,8 +78,7 @@ def Base_executeJupyter(self, python_expression=None, reference=None, title=None ...@@ -81,8 +78,7 @@ def Base_executeJupyter(self, python_expression=None, reference=None, title=None
u'evalue': final_result['evalue'], u'evalue': final_result['evalue'],
u'traceback': final_result['traceback'], u'traceback': final_result['traceback'],
u'status': final_result['status'], u'status': final_result['status'],
u'mime_type': final_result['mime_type'], u'mime_type': final_result['mime_type']}
}
# Updates the context in the notebook with the resulting context of code # Updates the context in the notebook with the resulting context of code
# execution. # execution.
...@@ -113,14 +109,12 @@ def Base_executeJupyter(self, python_expression=None, reference=None, title=None ...@@ -113,14 +109,12 @@ def Base_executeJupyter(self, python_expression=None, reference=None, title=None
u'evalue': None, u'evalue': None,
u'traceback': None, u'traceback': None,
u'status': u'error', u'status': u'error',
u'mime_type': result['mime_type'] u'mime_type': result['mime_type']}
}
serialized_result = json.dumps(result) serialized_result = json.dumps(result)
data_notebook_line.edit( data_notebook_line.edit(
notebook_code_result=result['code_result'], notebook_code_result = result['code_result'],
mime_type=result['mime_type'] mime_type = result['mime_type'])
)
return serialized_result return serialized_result
...@@ -175,14 +169,16 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -175,14 +169,16 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
# dictionary, but that might hamper the speed of exec or eval. # dictionary, but that might hamper the speed of exec or eval.
# Something like -- user_context = globals(); user_context['context'] = self; # Something like -- user_context = globals(); user_context['context'] = self;
user_context = {} user_context = {}
output = '' output = ''
# Saving the initial globals dict so as to compare it after code execution # Saving the initial globals dict so as to compare it after code execution
globals_dict = globals() globals_dict = globals()
result_string = ''
notebook_context = old_notebook_context notebook_context = old_notebook_context
inject_variable_dict = {}
current_var_dict = {}
current_setup_dict = {}
# Execute only if jupyter_code is not empty # Execute only if jupyter_code is not empty
if jupyter_code: if jupyter_code:
# Create ast parse tree # Create ast parse tree
...@@ -231,13 +227,13 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -231,13 +227,13 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
# Variables used at the display hook to get the proper form to display # Variables used at the display hook to get the proper form to display
# the last returning variable of any code cell. # the last returning variable of any code cell.
display_data = {'result': '', 'mime_type': None} display_data = {'result': '',
'mime_type': None}
# This is where one part of the display magic happens. We create an # This is where one part of the display magic happens. We create an
# instance of ProcessorList and add each of the built-in processors. # instance of ProcessorList and add each of the built-in processors.
# The classes which each of them are responsiblefor rendering are defined # The classes which each of them are responsiblefor rendering are defined
# in the classes themselves. # in the classes themselves.
#
# The customized display hook will automatically use the processor # The customized display hook will automatically use the processor
# of the matching class to decide how the object should be displayed. # of the matching class to decide how the object should be displayed.
processor_list = ProcessorList() processor_list = ProcessorList()
...@@ -253,8 +249,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -253,8 +249,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
'_display_data': display_data, '_display_data': display_data,
'_processor_list': processor_list, '_processor_list': processor_list,
'_volatile_variable_list': [], '_volatile_variable_list': [],
'_print': CustomPrint() '_print': CustomPrint()}
}
user_context.update(inject_variable_dict) user_context.update(inject_variable_dict)
user_context.update(notebook_context['variables']) user_context.update(notebook_context['variables'])
...@@ -291,11 +286,9 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -291,11 +286,9 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
'mime_type': 'text/plain', 'mime_type': 'text/plain',
'evalue': None, 'evalue': None,
'ename': None, 'ename': None,
'traceback': None, 'traceback': None}
}
return result return result
# Removing all the setup functions if user call environment.clearAll() # Removing all the setup functions if user call environment.clearAll()
if environment_collector.clearAll(): if environment_collector.clearAll():
keys = notebook_context ['setup'].keys() keys = notebook_context ['setup'].keys()
...@@ -333,9 +326,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -333,9 +326,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
) % (data['code'], func_name, func_name) ) % (data['code'], func_name, func_name)
notebook_context['setup'][data['alias']] = { notebook_context['setup'][data['alias']] = {
"func_name": func_name, "func_name": func_name,
"code": setup_string "code": setup_string}
}
inject_variable_dict['_print'].write(setup_string)
# Iterating over envinronment.define calls captured by the environment collector # Iterating over envinronment.define calls captured by the environment collector
# that are simple variables and saving them in the setup. # that are simple variables and saving them in the setup.
...@@ -343,8 +334,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -343,8 +334,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
setup_string = "%s = %s\n" % (variable, repr(value)) setup_string = "%s = %s\n" % (variable, repr(value))
notebook_context['setup'][variable] = { notebook_context['setup'][variable] = {
'func_name': variable, 'func_name': variable,
'code': setup_string 'code': setup_string}
}
user_context['_volatile_variable_list'] += variable user_context['_volatile_variable_list'] += variable
if environment_collector.showEnvironmentSetup(): if environment_collector.showEnvironmentSetup():
...@@ -359,10 +349,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -359,10 +349,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
except Exception as e: except Exception as e:
# Abort the current transaction. As a consequence, the notebook lines # Abort the current transaction. As a consequence, the notebook lines
# are not added if an exception occurs. # are not added if an exception occurs.
#
# TODO: store which notebook line generated which exception.
transaction.abort() transaction.abort()
# Clear the portal cache from previous transaction
return getErrorMessageForException(self, e, notebook_context) return getErrorMessageForException(self, e, notebook_context)
# Execute the interactive nodes with 'single' mode # Execute the interactive nodes with 'single' mode
...@@ -374,19 +361,15 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -374,19 +361,15 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
except Exception as e: except Exception as e:
# Abort the current transaction. As a consequence, the notebook lines # Abort the current transaction. As a consequence, the notebook lines
# are not added if an exception occurs. # are not added if an exception occurs.
#
# TODO: store which notebook line generated which exception.
transaction.abort() transaction.abort()
# Clear the portal cache from previous transaction
return getErrorMessageForException(self, e, notebook_context) return getErrorMessageForException(self, e, notebook_context)
mime_type = display_data['mime_type'] or mime_type mime_type = display_data['mime_type'] or mime_type
inject_variable_dict['_print'].write("\n".join(removed_setup_message_list) + display_data['result']) inject_variable_dict['_print'].write("\n".join(removed_setup_message_list) + display_data['result'])
# Saves a list of all the variables we injected into the user context and # Saves a list of all the variables we injected into the user context and
# shall be deleted before saving the context. # shall be deleted before saving the context.
volatile_variable_list = current_setup_dict.keys() + inject_variable_dict.keys() + user_context['_volatile_variable_list'] volatile_variable_list = current_setup_dict.keys() + inject_variable_dict.keys() + user_context.get('_volatile_variable_list', [])
volatile_variable_list.append('__builtins__') volatile_variable_list.append('__builtins__')
for key, val in user_context.items(): for key, val in user_context.items():
...@@ -409,7 +392,8 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -409,7 +392,8 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
if not key in user_context: if not key in user_context:
del notebook_context['variables'][key] del notebook_context['variables'][key]
output = inject_variable_dict['_print'].getCapturedOutputString() if inject_variable_dict.get('_print') is not None:
output = inject_variable_dict['_print'].getCapturedOutputString()
result = { result = {
'result_string': output, 'result_string': output,
...@@ -418,8 +402,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context): ...@@ -418,8 +402,7 @@ def Base_runJupyterCode(self, jupyter_code, old_notebook_context):
'mime_type': mime_type, 'mime_type': mime_type,
'evalue': evalue, 'evalue': evalue,
'ename': ename, 'ename': ename,
'traceback': tb_list, 'traceback': tb_list}
}
return result return result
...@@ -432,8 +415,7 @@ class EnvironmentDefinitionError(TypeError): ...@@ -432,8 +415,7 @@ class EnvironmentDefinitionError(TypeError):
def canSerialize(obj): def canSerialize(obj):
result = False
container_type_tuple = (list, tuple, dict, set, frozenset) container_type_tuple = (list, tuple, dict, set, frozenset)
# if object is a container, we need to check its elements for presence of # if object is a container, we need to check its elements for presence of
...@@ -479,7 +461,7 @@ def canSerialize(obj): ...@@ -479,7 +461,7 @@ def canSerialize(obj):
# #
# Even though the issue seems complicated, this quickfix should be # Even though the issue seems complicated, this quickfix should be
# properly rewritten in a better way as soon as possible. # properly rewritten in a better way as soon as possible.
except (cPickle.PicklingError, TypeError, NameError, AttributeError) as e: except (cPickle.PicklingError, TypeError, NameError, AttributeError):
return False return False
else: else:
return True return True
...@@ -687,9 +669,32 @@ class ImportFixer(ast.NodeTransformer): ...@@ -687,9 +669,32 @@ class ImportFixer(ast.NodeTransformer):
with the user context. with the user context.
""" """
module_name = node.names[0].name module_name = node.names[0].name
test_import_string = None
if getattr(node, "module", None) is not None:
# case when 'from <module_name> import <something>'
test_import_string = "from %s import %s" %(node.module, module_name)
# XXX: handle sub case when "from <module_name> import *"
#if module_name == '*':
# module_name = '%s_ALL' %(node.module)
#else:
# module_name = '%s_%s' %(node.module, module_name)
if getattr(node.names[0], 'asname'): if getattr(node.names[0], 'asname'):
# case when 'import <module_name> as <name>'
module_name = node.names[0].asname module_name = node.names[0].asname
test_import_string = "import %s as %s" %(node.names[0].name, module_name)
if test_import_string is None:
# case 'import <module_name>
test_import_string = "import %s" %node.names[0].name
#log('%s : %s' %(module_name, test_import_string))
if not self.import_func_dict.get(module_name): if not self.import_func_dict.get(module_name):
# try to import module before it is added to environment
# this way if user tries to import non existent module Exception
# is immediately raised and doesn't block next Jupyter cell execution
exec(test_import_string)
empty_function = self.newEmptyFunction("%s_setup" % module_name) empty_function = self.newEmptyFunction("%s_setup" % module_name)
return_dict = self.newReturnDict(module_name) return_dict = self.newReturnDict(module_name)
empty_function.body = [node, return_dict] empty_function.body = [node, return_dict]
...@@ -711,7 +716,7 @@ class ImportFixer(ast.NodeTransformer): ...@@ -711,7 +716,7 @@ class ImportFixer(ast.NodeTransformer):
""" """
Return an AST.Expr representing a returned dict with one single key named Return an AST.Expr representing a returned dict with one single key named
`'module_name'` (as string) which returns the variable `module_name` (as `'module_name'` (as string) which returns the variable `module_name` (as
exoression). expression).
""" """
return_dict = "return {'%s': %s}" % (module_name, module_name) return_dict = "return {'%s': %s}" % (module_name, module_name)
return ast.parse(return_dict).body[0] return ast.parse(return_dict).body[0]
...@@ -735,7 +740,7 @@ class ImportFixer(ast.NodeTransformer): ...@@ -735,7 +740,7 @@ class ImportFixer(ast.NodeTransformer):
"WARNING: Your imported the module %s without using " "WARNING: Your imported the module %s without using "
"the environment object, which is not recomended. " "the environment object, which is not recomended. "
"Your import was automatically converted to use such method." "Your import was automatically converted to use such method."
"The setup function registered was named %s_setup.\\n" "The setup function was named as: %s_setup.\\n"
"'") % (module_name, module_name) "'") % (module_name, module_name)
tree = ast.parse(warning) tree = ast.parse(warning)
return tree.body[0] return tree.body[0]
...@@ -763,7 +768,7 @@ def renderAsHtml(self, renderable_object): ...@@ -763,7 +768,7 @@ def renderAsHtml(self, renderable_object):
compile_jupyter_frame = sys._getframe(3) compile_jupyter_frame = sys._getframe(3)
compile_jupyter_locals = compile_jupyter_frame.f_locals compile_jupyter_locals = compile_jupyter_frame.f_locals
processor = compile_jupyter_locals['processor_list'].getProcessorFor(renderable_object) processor = compile_jupyter_locals['processor_list'].getProcessorFor(renderable_object)
result, mime_type = processor(renderable_object).process() result, _ = processor(renderable_object).process()
compile_jupyter_locals['inject_variable_dict']['_print'].write(result) compile_jupyter_locals['inject_variable_dict']['_print'].write(result)
compile_jupyter_locals['display_data']['mime_type'] = 'text/html' compile_jupyter_locals['display_data']['mime_type'] = 'text/html'
...@@ -773,7 +778,7 @@ def getErrorMessageForException(self, exception, notebook_context): ...@@ -773,7 +778,7 @@ def getErrorMessageForException(self, exception, notebook_context):
code execution (notebook_context) and will return a dict as Jupyter code execution (notebook_context) and will return a dict as Jupyter
requires for error rendering. requires for error rendering.
''' '''
etype, value, tb = sys.exc_info() _, value, _ = sys.exc_info()
traceback_text = traceback.format_exc().split('\n')[:-1] traceback_text = traceback.format_exc().split('\n')[:-1]
return { return {
'status': 'error', 'status': 'error',
......
bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterCompile.xml
View file @ d1085bd6
...@@ -46,14 +46,13 @@ ...@@ -46,14 +46,13 @@
<key> <string>text_content_warning_message</string> </key> <key> <string>text_content_warning_message</string> </key>
<value> <value>
<tuple> <tuple>
<string>W: 85, 2: Redefining name \'traceback\' from outer scope (line 9) (redefined-outer-name)</string> <string>W:302, 10: Use of exec (exec-used)</string>
<string>W:197, 8: Use of exec (exec-used)</string> <string>W:348, 10: Use of exec (exec-used)</string>
<string>W:252, 8: Use of exec (exec-used)</string> <string>W:360, 10: Use of exec (exec-used)</string>
<string>W:264, 8: Use of exec (exec-used)</string> <string>W:444, 6: No exception type(s) specified (bare-except)</string>
<string>W:327, 10: Unused variable \'mime_type\' (unused-variable)</string> <string>W:697, 6: Use of exec (exec-used)</string>
<string>W:462, 2: Redefining name \'IFrame\' from outer scope (line 17) (redefined-outer-name)</string> <string>W:923, 2: Redefining name \'IFrame\' from outer scope (line 4) (redefined-outer-name)</string>
<string>W: 9, 0: Unused import traceback (unused-import)</string> <string>W: 18, 0: Unused log imported from Products.ERP5Type.Log (unused-import)</string>
<string>W: 13, 0: Unused import transaction (unused-import)</string>
</tuple> </tuple>
</value> </value>
</item> </item>
......
bt5/erp5_data_notebook/bt/description
View file @ d1085bd6
Interaction between Jupyter(IPython Notebook) and ERP5. Interaction between Jupyter (former IPython Notebook) and ERP5.
!WARNING! !WARNING!
This business template is unsafe to install on a public server as one of the extensions uses eval and allows remote code execution. Proper security should be taken into account. This business template is unsafe to install on a public server as one of the extensions uses eval and allows remote code execution. Proper security should be taken into account.
This template includes a highly exprimental integration with PivotTableJs which doesn't follow ERP5 Javascript standards and will be refactored to use JIO and RenderJS. This template includes a highly experimental integration with PivotTableJs which doesn't follow ERP5 JavaScript standards and will be re factored to use jIO and RenderJS.
!WARNING! !WARNING!
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7