erp5_dat_notebook bt5: Do not save modules object in local_variable_dict

Problem: Module objects are not picklable, thus, trying them to save in
local_variable_dict which is further saved in ZODB via ActiveResult object
of CMFActivity was giving error. So, its better to save the module objects
as code_strings in local_variable_dict.

As for now, for example:
local_variable_dict = {'imports': ['import numpy as np', 'import matplotlib as mpl'],
			 'variables': {'a':1, 'b:2'}}
for jupyter_code = """
import numpy as np
import matplotlib as mlp
a= 1; b=2; print a+b
"""

bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterCompile.py

 # -*- coding: utf-8 -*-
 
 from StringIO import StringIO
+from persistent.list import PersistentList
 from Products.ERP5Type.Globals import  PersistentMapping
 
 import sys
 import ast
+import types
 
 def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
   """
@@ -12,6 +14,17 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
     Code execution depends on 'interactivity', a.k.a , if the ast.node object has
     ast.Expr instance(valid for expressions) or not.
     
+    old_local_variable_dict should contain both variables dict and imports list.
+    Here, imports list is basically a list of code lines which would be run
+    executed separately everytime before execution of jupyter_code to populate
+    sys modules beforehand.
+
+    For example :
+    old_local_variable_dict = {
+                                'imports': ['import numpy as np', 'import sys as sys'],
+                                'variables': {'np.split': <function split at 0x7f4e6eb48b90>}
+                                }
+
     The behaviour would be similar to that of jupyter notebook:-
     ( https://github.com/ipython/ipython/blob/master/IPython/core/interactiveshell.py#L2954 )
     Example:
@@ -40,7 +53,7 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
   result_string = None
   ename, evalue, tb_list = None, None, None
   # Update globals dict and use it while running exec command
-  g.update(old_local_variable_dict)
+  g.update(old_local_variable_dict['variables'])
 
   # IPython expects 2 status message - 'ok', 'error'
   # XXX: The focus is on 'ok' status only, we're letting errors to be raised on
@@ -51,6 +64,11 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
 
   # Execute only if jupyter_code is not empty
   if jupyter_code:
+    # Import all the modules from local_variable_dict['imports']
+    import_statement_code = '\n'.join(old_local_variable_dict['imports'])
+
+    exec(import_statement_code, g, g)
+  
     # Create ast parse tree
     ast_node = ast.parse(jupyter_code)
     # Get the node list from the parsed tree
@@ -103,7 +121,24 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
   # not be picklabale
   local_variable_dict = old_local_variable_dict
   local_variable_dict_new = {key: val for key, val in g.items() if key not in globals_dict.keys()}
-  local_variable_dict.update(local_variable_dict_new)
+  local_variable_dict['variables'].update(local_variable_dict_new)
+
+  # Differentiate 'module' objects from local_variable_dict and save them as
+  # string in the dict as {'imports': ['import numpy as np', 'import matplotlib as mp']}
+  if 'variables' in local_variable_dict:
+    for key, val in local_variable_dict['variables'].items():
+      # Check if the val in the dict is ModuleType and remove it in case it is
+      if isinstance(val, types.ModuleType):
+        # XXX: The next line is mutating the dict, beware in case any reference
+        # is made later on to local_variable_dict['variables'] dictionary
+        local_variable_dict['variables'].pop(key)
+        # While any execution, in locals() dict, a module is saved as:
+        # code : 'from os import path'
+        # {'path': <module 'posixpath'>}
+        # So, here we would try to get the name 'posixpath' and import it as 'path'
+        module_name = val.__name__
+        import_statement = 'import %s as %s'%(module_name, key)
+        local_variable_dict['imports'].append(import_statement)
 
   result = {
     'result_string': result_string,
@@ -113,20 +148,31 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
     'ename': ename,
     'traceback': tb_list,
   }
+
   return result
 
-def AddPersistentMapping(self):
+def AddNewLocalVariableDict(self):
   """
-  Function to add PersistentMapping object which can be used as a dictionary
+  Function to add a new Local Variable for a Data Notebook
   """
   new_dict = PersistentMapping()
+  variable_dict = PersistentMapping()
+  import_list = PersistentList()
+  new_dict['variables'] = variable_dict
+  new_dict['imports'] = import_list
   return new_dict
 
-def UpdatePersistentMapping(self, existing_dict):
+def UpdateLocalVariableDict(self, existing_dict):
   """
-  Function to update PersistentMapping object
+  Function to update local_varibale_dict for a Data Notebook
   """
   new_dict = PersistentMapping()
-  for key, value in existing_dict.iteritems():
-    new_dict[key]=value
+  variable_dict = PersistentMapping()
+  import_list = PersistentList()
+  new_dict['variables'] = variable_dict
+  new_dict['imports'] = import_list
+  for key, val in existing_dict['variables'].iteritems():
+    new_dict['variables'][key] = val
+  new_dict['imports'] = PersistentList(existing_dict['imports'])
   return new_dict
+  
\ No newline at end of file

bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterCompile.xml

@@ -46,8 +46,9 @@
             <key> <string>text_content_warning_message</string> </key>
             <value>
               <tuple>
-                <string>W: 81,  6: Use of exec (exec-used)</string>
-                <string>W: 87,  6: Use of exec (exec-used)</string>
+                <string>W: 70,  4: Use of exec (exec-used)</string>
+                <string>W: 99,  6: Use of exec (exec-used)</string>
+                <string>W:105,  6: Use of exec (exec-used)</string>
               </tuple>
             </value>
         </item>

bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_addPersistentMapping.xml → bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_addLocalVariableDict.xml

@@ -8,7 +8,7 @@
       <dictionary>
         <item>
             <key> <string>_function</string> </key>
-            <value> <string>AddPersistentMapping</string> </value>
+            <value> <string>AddNewLocalVariableDict</string> </value>
         </item>
         <item>
             <key> <string>_module</string> </key>
@@ -16,7 +16,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>Base_addPersistentMapping</string> </value>
+            <value> <string>Base_addLocalVariableDict</string> </value>
         </item>
         <item>
             <key> <string>title</string> </key>

bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_executeJupyter.xml

@@ -126,7 +126,7 @@ result_list = active_process.getResultList()\n
 # persistent mapping object\n
 old_local_variable_dict = result_list[0].summary\n
 if not old_local_variable_dict:\n
-  old_local_variable_dict = context.Base_addPersistentMapping()\n
+  old_local_variable_dict = context.Base_addLocalVariableDict()\n
 \n
 # Pass all to code Base_runJupyter external function which would execute the code\n
 # and returns a dict of result\n
@@ -140,7 +140,7 @@ status = final_result[\'status\']\n
 \n
 # Call to function to update persistent mapping object with new local variables\n
 # and save the variables in the Active Result pertaining to the current Data Notebook\n
-new_dict = context.Base_updatePersistentMapping(new_local_variable_dict)\n
+new_dict = context.Base_updateLocalVariableDict(new_local_variable_dict)\n
 result_list[0].edit(summary=new_dict)\n
 \n
 result = {\n

bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_updatePersistentMapping.xml → bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/Base_updateLocalVariableDict.xml

@@ -8,7 +8,7 @@
       <dictionary>
         <item>
             <key> <string>_function</string> </key>
-            <value> <string>UpdatePersistentMapping</string> </value>
+            <value> <string>UpdateLocalVariableDict</string> </value>
         </item>
         <item>
             <key> <string>_module</string> </key>
@@ -16,7 +16,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>Base_updatePersistentMapping</string> </value>
+            <value> <string>Base_updateLocalVariableDict</string> </value>
         </item>
         <item>
             <key> <string>title</string> </key>

bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/DataNotebookModule_addDataNotebook.xml

@@ -65,7 +65,7 @@ active_process = context.portal_activities.newActiveProcess()\n
 active_process_id = active_process.getId()\n
 \n
 # Creating new dictionary via external method to save results in ZODB\n
-new_dict = context.Base_addPersistentMapping()\n
+new_dict = context.Base_addLocalVariableDict()\n
 # Add new ActiveResult object and add it to the activeprocess concerned with ...\n
 # Data Notebook in concern\n
 result = ActiveResult(summary=new_dict)\n

bt5/erp5_data_notebook/TestTemplateItem/portal_components/test.erp5.testExecuteJupyter.py

@@ -107,7 +107,7 @@ portal.%s()
                       NameError,
                       portal.Base_runJupyter,
                       jupyter_code=jupyter_code,
-                      old_local_variable_dict={}
+                      old_local_variable_dict=portal.Base_addLocalVariableDict()
                       )
     # Abort the current transaction of test so that we can proceed to new one
     transaction.abort()
@@ -281,8 +281,9 @@ portal.%s()
     process_id = notebook.getProcess()
     active_process = portal.portal_activities[process_id]
     result_list = active_process.getResultList()
+    local_variable_dict = result_list[0].summary['variables']
     result = {'a':2, 'b':3}
-    self.assertDictContainsSubset(result, result_list[0].summary)
+    self.assertDictContainsSubset(result, local_variable_dict)
 
   def testBaseExecuteJupyterRerunWithPreviousLocalVariables(self):
     """