Commit 1ff72bd5 authored by Ayush Tiwari's avatar Ayush Tiwari

erp5_dat_notebook bt5: Do not save modules object in local_variable_dict

Problem: Module objects are not picklable, thus, trying them to save in
local_variable_dict which is further saved in ZODB via ActiveResult object
of CMFActivity was giving error. So, its better to save the module objects
as code_strings in local_variable_dict.

As for now, for example:
local_variable_dict = {'imports': ['import numpy as np', 'import matplotlib as mpl'],
			 'variables': {'a':1, 'b:2'}}
for jupyter_code = """
import numpy as np
import matplotlib as mlp
a= 1; b=2; print a+b
"""
parent 91f11dff
# -*- coding: utf-8 -*-
from StringIO import StringIO
from persistent.list import PersistentList
from Products.ERP5Type.Globals import PersistentMapping
import sys
import ast
import types
def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
"""
......@@ -12,6 +14,17 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
Code execution depends on 'interactivity', a.k.a , if the ast.node object has
ast.Expr instance(valid for expressions) or not.
old_local_variable_dict should contain both variables dict and imports list.
Here, imports list is basically a list of code lines which would be run
executed separately everytime before execution of jupyter_code to populate
sys modules beforehand.
For example :
old_local_variable_dict = {
'imports': ['import numpy as np', 'import sys as sys'],
'variables': {'np.split': <function split at 0x7f4e6eb48b90>}
}
The behaviour would be similar to that of jupyter notebook:-
( https://github.com/ipython/ipython/blob/master/IPython/core/interactiveshell.py#L2954 )
Example:
......@@ -40,7 +53,7 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
result_string = None
ename, evalue, tb_list = None, None, None
# Update globals dict and use it while running exec command
g.update(old_local_variable_dict)
g.update(old_local_variable_dict['variables'])
# IPython expects 2 status message - 'ok', 'error'
# XXX: The focus is on 'ok' status only, we're letting errors to be raised on
......@@ -51,6 +64,11 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
# Execute only if jupyter_code is not empty
if jupyter_code:
# Import all the modules from local_variable_dict['imports']
import_statement_code = '\n'.join(old_local_variable_dict['imports'])
exec(import_statement_code, g, g)
# Create ast parse tree
ast_node = ast.parse(jupyter_code)
# Get the node list from the parsed tree
......@@ -103,7 +121,24 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
# not be picklabale
local_variable_dict = old_local_variable_dict
local_variable_dict_new = {key: val for key, val in g.items() if key not in globals_dict.keys()}
local_variable_dict.update(local_variable_dict_new)
local_variable_dict['variables'].update(local_variable_dict_new)
# Differentiate 'module' objects from local_variable_dict and save them as
# string in the dict as {'imports': ['import numpy as np', 'import matplotlib as mp']}
if 'variables' in local_variable_dict:
for key, val in local_variable_dict['variables'].items():
# Check if the val in the dict is ModuleType and remove it in case it is
if isinstance(val, types.ModuleType):
# XXX: The next line is mutating the dict, beware in case any reference
# is made later on to local_variable_dict['variables'] dictionary
local_variable_dict['variables'].pop(key)
# While any execution, in locals() dict, a module is saved as:
# code : 'from os import path'
# {'path': <module 'posixpath'>}
# So, here we would try to get the name 'posixpath' and import it as 'path'
module_name = val.__name__
import_statement = 'import %s as %s'%(module_name, key)
local_variable_dict['imports'].append(import_statement)
result = {
'result_string': result_string,
......@@ -113,20 +148,31 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
'ename': ename,
'traceback': tb_list,
}
return result
def AddPersistentMapping(self):
def AddNewLocalVariableDict(self):
"""
Function to add PersistentMapping object which can be used as a dictionary
Function to add a new Local Variable for a Data Notebook
"""
new_dict = PersistentMapping()
variable_dict = PersistentMapping()
import_list = PersistentList()
new_dict['variables'] = variable_dict
new_dict['imports'] = import_list
return new_dict
def UpdatePersistentMapping(self, existing_dict):
def UpdateLocalVariableDict(self, existing_dict):
"""
Function to update PersistentMapping object
Function to update local_varibale_dict for a Data Notebook
"""
new_dict = PersistentMapping()
for key, value in existing_dict.iteritems():
new_dict[key]=value
variable_dict = PersistentMapping()
import_list = PersistentList()
new_dict['variables'] = variable_dict
new_dict['imports'] = import_list
for key, val in existing_dict['variables'].iteritems():
new_dict['variables'][key] = val
new_dict['imports'] = PersistentList(existing_dict['imports'])
return new_dict
\ No newline at end of file
......@@ -46,8 +46,9 @@
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple>
<string>W: 81, 6: Use of exec (exec-used)</string>
<string>W: 87, 6: Use of exec (exec-used)</string>
<string>W: 70, 4: Use of exec (exec-used)</string>
<string>W: 99, 6: Use of exec (exec-used)</string>
<string>W:105, 6: Use of exec (exec-used)</string>
</tuple>
</value>
</item>
......
......@@ -8,7 +8,7 @@
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>AddPersistentMapping</string> </value>
<value> <string>AddNewLocalVariableDict</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
......@@ -16,7 +16,7 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_addPersistentMapping</string> </value>
<value> <string>Base_addLocalVariableDict</string> </value>
</item>
<item>
<key> <string>title</string> </key>
......
......@@ -126,7 +126,7 @@ result_list = active_process.getResultList()\n
# persistent mapping object\n
old_local_variable_dict = result_list[0].summary\n
if not old_local_variable_dict:\n
old_local_variable_dict = context.Base_addPersistentMapping()\n
old_local_variable_dict = context.Base_addLocalVariableDict()\n
\n
# Pass all to code Base_runJupyter external function which would execute the code\n
# and returns a dict of result\n
......@@ -140,7 +140,7 @@ status = final_result[\'status\']\n
\n
# Call to function to update persistent mapping object with new local variables\n
# and save the variables in the Active Result pertaining to the current Data Notebook\n
new_dict = context.Base_updatePersistentMapping(new_local_variable_dict)\n
new_dict = context.Base_updateLocalVariableDict(new_local_variable_dict)\n
result_list[0].edit(summary=new_dict)\n
\n
result = {\n
......
......@@ -8,7 +8,7 @@
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>UpdatePersistentMapping</string> </value>
<value> <string>UpdateLocalVariableDict</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
......@@ -16,7 +16,7 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_updatePersistentMapping</string> </value>
<value> <string>Base_updateLocalVariableDict</string> </value>
</item>
<item>
<key> <string>title</string> </key>
......
......@@ -65,7 +65,7 @@ active_process = context.portal_activities.newActiveProcess()\n
active_process_id = active_process.getId()\n
\n
# Creating new dictionary via external method to save results in ZODB\n
new_dict = context.Base_addPersistentMapping()\n
new_dict = context.Base_addLocalVariableDict()\n
# Add new ActiveResult object and add it to the activeprocess concerned with ...\n
# Data Notebook in concern\n
result = ActiveResult(summary=new_dict)\n
......
......@@ -107,7 +107,7 @@ portal.%s()
NameError,
portal.Base_runJupyter,
jupyter_code=jupyter_code,
old_local_variable_dict={}
old_local_variable_dict=portal.Base_addLocalVariableDict()
)
# Abort the current transaction of test so that we can proceed to new one
transaction.abort()
......@@ -281,8 +281,9 @@ portal.%s()
process_id = notebook.getProcess()
active_process = portal.portal_activities[process_id]
result_list = active_process.getResultList()
local_variable_dict = result_list[0].summary['variables']
result = {'a':2, 'b':3}
self.assertDictContainsSubset(result, result_list[0].summary)
self.assertDictContainsSubset(result, local_variable_dict)
def testBaseExecuteJupyterRerunWithPreviousLocalVariables(self):
"""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment