Commit 5df56607 authored by Douglas's avatar Douglas

Jupyter: Added experimental integration between pivottablejs and Pandas.DataFrame

pivottablejs is a very useful pivot table implementation in Javascript that
alllows the user to create his own tables and charts. And also they had examples
of integration with Pandas.DataFrame objects and Jupyter. So this is highly
based on that.

**ATTENTION**: this is an experimental integration and does not follow the ERP5
Javascript standards. It will be refactored in the future to use RenderJS and
JIO.

The integration generates an HTML page template which starts the pivot table and
have a placeholder for the data, that will be later replaced with a Data Frame
data as CSV. After this replacement the page is stored in the memcached server
and then served from there, through a Script (Python) object, inside an HTML
iframe. The iframe is necessary because a lot of Javascript libraries that are
not included in the Jupyter web page are loaded.

A web page with id "PivotTableJs_getMovementHistoryList" was created to demo
how pivottablejs can be integrated within ERP5, either using AJAX or not.

In the process of this integration a simple external method to render
iPython's display classes (Images, Video, Youtube, IFrame, etc) was created. It
will be refactored and polished along with the kernel itself in the future.
parent c4f98af2
...@@ -2,20 +2,21 @@ ...@@ -2,20 +2,21 @@
from cStringIO import StringIO from cStringIO import StringIO
from Products.ERP5Type.Globals import PersistentMapping from Products.ERP5Type.Globals import PersistentMapping
from OFS.Image import Image as OFSImage from erp5.portal_type import Image
from types import ModuleType
import sys import sys
import traceback
import ast import ast
import types import types
import inspect
import traceback
import base64
import transaction import transaction
mime_type = 'text/plain' from matplotlib.figure import Figure
# IPython expects 2 status message - 'ok', 'error' from IPython.core.display import DisplayObject
status = u'ok' from IPython.lib.display import IFrame
ename, evalue, tb_list = None, None, None
def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict): def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
""" """
...@@ -51,26 +52,21 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict): ...@@ -51,26 +52,21 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
out2 = '12' out2 = '12'
""" """
# Updating global variable mime_type to its original value
# Required when call to Base_displayImage is made which is changing
# the value of gloabl mime_type
# Same for status, ename, evalue, tb_list
global mime_type, status, ename, evalue, tb_list
mime_type = 'text/plain' mime_type = 'text/plain'
status = u'ok' status = u'ok'
ename, evalue, tb_list = None, None, None ename, evalue, tb_list = None, None, None
# Other way would be to use all the globals variables instead of just an empty # Other way would be to use all the globals variables instead of just an empty
# dictionary, but that might hamper the speed of exec or eval. # dictionary, but that might hamper the speed of exec or eval.
# Something like -- g = globals(); g['context'] = self; # Something like -- user_context = globals(); user_context['context'] = self;
g = {} user_context = {}
# Saving the initial globals dict so as to compare it after code execution # Saving the initial globals dict so as to compare it after code execution
globals_dict = globals() globals_dict = globals()
g['context'] = self user_context['context'] = self
result_string = '' result_string = ''
# Update globals dict and use it while running exec command # Update globals dict and use it while running exec command
g.update(old_local_variable_dict['variables']) user_context.update(old_local_variable_dict['variables'])
# XXX: The focus is on 'ok' status only, we're letting errors to be raised on # XXX: The focus is on 'ok' status only, we're letting errors to be raised on
# erp5 for now, so as not to hinder the transactions while catching them. # erp5 for now, so as not to hinder the transactions while catching them.
...@@ -80,7 +76,6 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict): ...@@ -80,7 +76,6 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
# Execute only if jupyter_code is not empty # Execute only if jupyter_code is not empty
if jupyter_code: if jupyter_code:
# Create ast parse tree # Create ast parse tree
try: try:
ast_node = ast.parse(jupyter_code) ast_node = ast.parse(jupyter_code)
...@@ -101,7 +96,7 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict): ...@@ -101,7 +96,7 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
# So, here we would try to get the name 'posixpath' and import it as 'path' # So, here we would try to get the name 'posixpath' and import it as 'path'
for k, v in old_local_variable_dict['imports'].iteritems(): for k, v in old_local_variable_dict['imports'].iteritems():
import_statement_code = 'import %s as %s'%(v, k) import_statement_code = 'import %s as %s'%(v, k)
exec(import_statement_code, g, g) exec(import_statement_code, user_context, user_context)
# If the last node is instance of ast.Expr, set its interactivity as 'last' # If the last node is instance of ast.Expr, set its interactivity as 'last'
# This would be the case if the last node is expression # This would be the case if the last node is expression
...@@ -121,12 +116,43 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict): ...@@ -121,12 +116,43 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
result = StringIO() result = StringIO()
sys.stdout = result sys.stdout = result
# Variables used at the display hook to get the proper form to display
# the last returning variable of any code cell.
#
display_data = {'result': '', 'mime_type': None}
# This is where one part of the display magic happens. We create an
# instance of ProcessorList and add each of the built-in processors.
# The classes which each of them are responsiblefor rendering are defined
# in the classes themselves.
#
# The customized display hook will automatically use the processor
# of the matching class to decide how the object should be displayed.
#
processor_list = ProcessorList()
processor_list.addProcessor(IPythonDisplayObjectProcessor)
processor_list.addProcessor(MatplotlibFigureProcessor)
processor_list.addProcessor(ERP5ImageProcessor)
processor_list.addProcessor(IPythonDisplayObjectProcessor)
# Putting necessary variables in the `exec` calls context.
#
# - result: is required to store the order of manual calls to the rendering
# function;
#
# - display_data: is required to support mime type changes;
#
# - processor_list: is required for the proper rendering of the objects
#
user_context['_display_data'] = display_data
user_context['_processor_list'] = processor_list
# Execute the nodes with 'exec' mode # Execute the nodes with 'exec' mode
for node in to_run_exec: for node in to_run_exec:
mod = ast.Module([node]) mod = ast.Module([node])
code = compile(mod, '<string>', "exec") code = compile(mod, '<string>', "exec")
try: try:
exec(code, g, g) exec(code, user_context, user_context)
except Exception as e: except Exception as e:
# Abort the current transaction. As a consequence, the notebook lines # Abort the current transaction. As a consequence, the notebook lines
# are not added if an exception occurs. # are not added if an exception occurs.
...@@ -141,9 +167,9 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict): ...@@ -141,9 +167,9 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
# Execute the interactive nodes with 'single' mode # Execute the interactive nodes with 'single' mode
for node in to_run_interactive: for node in to_run_interactive:
mod = ast.Interactive([node]) mod = ast.Interactive([node])
code = compile(mod, '<string>', "single")
try: try:
exec(code, g, g) code = compile(mod, '<string>', 'single')
exec(code, user_context, user_context)
except Exception as e: except Exception as e:
# Abort the current transaction. As a consequence, the notebook lines # Abort the current transaction. As a consequence, the notebook lines
# are not added if an exception occurs. # are not added if an exception occurs.
...@@ -155,21 +181,16 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict): ...@@ -155,21 +181,16 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
self.getPortalObject().portal_caches.clearAllCache() self.getPortalObject().portal_caches.clearAllCache()
return getErrorMessageForException(self, e, local_variable_dict) return getErrorMessageForException(self, e, local_variable_dict)
# Letting the code fail in case of error while executing the python script/code
# XXX: Need to be refactored so to acclimitize transactions failure as well as
# normal python code failure and show it to user on jupyter frontend.
# Decided to let this fail silently in backend without letting the frontend
# user know the error so as to let tranasction or its error be handled by ZODB
# in uniform way instead of just using half transactions.
sys.stdout = old_stdout sys.stdout = old_stdout
result_string = result.getvalue() mime_type = display_data['mime_type'] or mime_type
result_string = result.getvalue() + display_data['result']
# Difference between the globals variable before and after exec/eval so that # Difference between the globals variable before and after exec/eval so that
# we don't have to save unnecessary variables in database which might or might # we don't have to save unnecessary variables in database which might or might
# not be picklabale # not be picklabale
local_variable_dict_new = {key: val for key, val in g.items() if key not in globals_dict.keys()} for key, val in user_context.items():
local_variable_dict['variables'].update(local_variable_dict_new) if key not in globals_dict.keys():
local_variable_dict['variables'][key] = val
# Differentiate 'module' objects from local_variable_dict and save them as # Differentiate 'module' objects from local_variable_dict and save them as
# string in the dict as {'imports': {'numpy': 'np', 'matplotlib': 'mp']} # string in the dict as {'imports': {'numpy': 'np', 'matplotlib': 'mp']}
...@@ -199,6 +220,32 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict): ...@@ -199,6 +220,32 @@ def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
return result return result
def renderAsHtml(self, renderable_object):
'''
renderAsHtml will render its parameter as HTML by using the matching
display processor for that class. Some processors can be found in this
file.
'''
# Ugly frame hack to access the processor list defined in the body of the
# kernel's code, where `exec` is called.
#
# At this point the stack should be, from top to the bottom:
#
# 5. ExternalMethod Patch call
# 4. Base_compileJupyterCode frame (where we want to change variable)
# 3. exec call to run the user's code
# 2. ExternalMethod Patch call through `context.Base_renderAsHtml` in the notebook
# 1. renderAsHtml frame (where the function is)
#
# So sys._getframe(3) is enough to get us up into the frame we want.
#
compile_jupyter_frame = sys._getframe(3)
compile_jupyter_locals = compile_jupyter_frame.f_locals
processor = compile_jupyter_locals['processor_list'].getProcessorFor(renderable_object)
result, mime_type = processor(renderable_object).process()
compile_jupyter_locals['result'].write(result)
compile_jupyter_locals['display_data']['mime_type'] = 'text/html'
def getErrorMessageForException(self, exception, local_variable_dict): def getErrorMessageForException(self, exception, local_variable_dict):
''' '''
getErrorMessageForException receives an Expcetion object and a context for getErrorMessageForException receives an Expcetion object and a context for
...@@ -239,137 +286,198 @@ def UpdateLocalVariableDict(self, existing_dict): ...@@ -239,137 +286,198 @@ def UpdateLocalVariableDict(self, existing_dict):
new_dict['imports'][key] = val new_dict['imports'][key] = val
return new_dict return new_dict
def Base_displayImage(self, image_object=None): class ObjectProcessor(object):
""" '''
External function to display Image objects to jupyter frontend. Basic object processor that stores the first parameters of the constructor
in the `subject` attribute and store the target classes for that processor.
'''
TARGET_CLASSES=None
TARGET_MODULES=None
XXX: This function is intented to be called from Base_executeJupyter @classmethod
or Jupyter frontend.That's why printing string and returning None. def getTargetClasses(cls):
Also, it clears the plot for Matplotlib object after every call, so return cls.TARGET_CLASSES
in case of saving the plot, its essential to call Base_saveImage before
calling Base_displayImage.
Parameters @classmethod
---------- def getTargetModules(cls):
return cls.TARGET_MODULES
image_object :Any image object from ERP5 def __init__(self, something):
Any matplotlib object from which we can create a plot. self.subject = something
Can be <matplotlib.lines.Line2D>, <matplotlib.text.Text>, etc.
Output class MatplotlibFigureProcessor(ObjectProcessor):
----- '''
MatplotlibFigureProcessor handles the rich display of
matplotlib.figure.Figure objects. It displays them using an img tag with
the inline png image encoded as base64.
'''
TARGET_CLASSES=[Figure,]
TARGET_MODULES=['matplotlib.pyplot',]
Prints base64 encoded string of the plot on which it has been called. def process(self):
image_io = StringIO()
self.subject.savefig(image_io, format='png')
image_io.seek(0)
return self._getImageHtml(image_io), 'text/html'
""" def _getImageHtml(self, image_io):
if image_object: return '<img src="data:image/png;base64,%s" /><br />' % base64.b64encode(image_io.getvalue())
import base64
# Chanage global variable 'mime_type' to 'image/png'
global mime_type
# Image object in ERP5 is instance of OFS.Image object
if isinstance(image_object, OFSImage):
figdata = base64.b64encode(image_object.getData())
mime_type = image_object.getContentType()
# Ensure that the object we are taking as `image_object` is basically a
# Matplotlib.pyplot module object from which we are seekign the data of the
# plot .
elif inspect.ismodule(image_object) and image_object.__name__=="matplotlib.pyplot":
# Create a ByteFile on the server which would be used to save the plot
figfile = StringIO()
# Save plot as 'png' format in the ByteFile
image_object.savefig(figfile, format='png')
figfile.seek(0)
# Encode the value in figfile to base64 string so as to serve it jupyter frontend
figdata = base64.b64encode(figfile.getvalue())
mime_type = 'image/png'
# Clear the plot figures after every execution
image_object.close()
# XXX: We are not returning anything because we want this function to be called
# by Base_executeJupyter , inside exec(), and its better to get the printed string
# instead of returned string from this function as after exec, we are getting
# value from stdout and using return we would get that value as string inside
# an string which is unfavourable.
print figdata
return None
def Base_saveImage(self, plot=None, reference=None, **kw):
"""
Saves generated plots from matplotlib in ERP5 Image module
XXX: Use only if bt5 'erp5_wendelin' installed class ERP5ImageProcessor(ObjectProcessor):
This function is intented to be called from Base_executeJupyter '''
or Jupyter frontend. ERP5ImageProcessor handles the rich display of ERP5's image_module object.
It gets the image data and content type and use them to create a proper img
tag.
'''
TARGET_CLASSES=[Image,]
Parameters def process(self):
---------- from base64 import b64encode
plot : Matplotlib plot object figure_data = b64encode(self.subject.getData())
mime_type = self.subject.getContentType()
return '<img src="data:%s;base64,%s" /><br />' % (mime_type, figure_data), 'text/html'
reference: Reference of Image object which would be generated class IPythonDisplayObjectProcessor(ObjectProcessor):
Id and reference should be always unique '''
IPythonDisplayObjectProcessor handles the display of all objects from the
IPython.display module, including: Audio, IFrame, YouTubeVideo, VimeoVideo,
ScribdDocument, FileLink, and FileLinks.
Output All these objects have the `_repr_html_` method, which is used by the class
------ to render them.
Returns None, but saves the plot object as ERP5 image in Image Module with '''
reference same as that of data_array_object. TARGET_CLASSES=[DisplayObject, IFrame]
""" def process(self):
html_repr = self.subject._repr_html_()
return html_repr + '<br />', 'text/html'
# As already specified in docstring, this function should be called from class GenericProcessor(ObjectProcessor):
# Base_executeJupyter or Jupyter Frontend which means that it would pass '''
# through exec and hence the printed result would be caught in a string and Generic processor to render objects as string.
# that's why we are using print and returning None. '''
if not reference:
print 'No reference specified for Image object' def process(self):
return None return str(self.subject), 'text/plain'
if not plot:
print 'No matplotlib plot object specified' class ProcessorList(object):
return None '''
ProcessorList is responsible to store all the processors in a dict using
filename = '%s.png'%reference the classes they handle as the key. Subclasses of these classes will have
# Save plot data in buffer the same processor of the eigen class. This means that the order of adding
buff = StringIO() processors is important, as some classes' processors may be overwritten in
plot.savefig(buff, format='png') some situations.
buff.seek(0)
data = buff.getvalue() The `getProcessorFor` method uses `something.__class__' and not
`type(something)` because using the later onobjects returned by portal
import time catalog queries will return an AcquisitionWrapper type instead of the
image_id = reference+str(time.time()) object's real class.
# Add new Image object in erp5 with id and reference '''
image_module = self.getDefaultModule(portal_type='Image')
image_module.newContent( def __init__(self, default=GenericProcessor):
portal_type='Image', self.processors = {}
id=image_id, self.default_processor = GenericProcessor
reference=reference,
data=data,
filename=filename)
return None
def getError(self, previous=1):
"""
Show error to the frontend and change status of code as 'error' from 'ok'
Parameters def addProcessor(self, processor):
---------- classes = processor.getTargetClasses()
previous: Type - int. The number of the error you want to see. modules = processor.getTargetModules()
Ex: 1 for last error
2 for 2nd last error and so on..
if classes and not len(classes) == 0:
for klass in classes:
self.processors[klass] = processor
for subclass in klass.__subclasses__():
self.processors[subclass] = processor
if modules and not len(modules) == 0:
for module in modules:
self.processors[module] = processor
def getProcessorFor(self, something):
if not isinstance(something, ModuleType):
return self.processors.get(something.__class__, self.default_processor)
else:
return self.processors.get(something.__name__, self.default_processor)
def storeIFrame(self, html, key):
memcached_tool = self.getPortalObject().portal_memcached
memcached_dict = memcached_tool.getMemcachedDict(key_prefix='pivottablejs', plugin_path='portal_memcached/default_memcached_plugin')
memcached_dict[key] = html
return True
# WARNING!
#
# This is a highly experimental PivotTableJs integration which does not follow
# ERP5 Javascrpt standards and it will be refactored to use JIO and RenderJS.
#
def erp5PivotTableUI(self, df, erp5_url):
from IPython.display import IFrame
template = """
<!DOCTYPE html>
<html>
<head>
<title>PivotTable.js</title>
<!-- external libs from cdnjs -->
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/c3/0.4.10/c3.min.css">
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jqueryui/1.11.4/jquery-ui.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jquery-csv/0.71/jquery.csv-0.71.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/c3/0.4.10/c3.min.js"></script>
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/pivot.min.css">
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/pivot.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/d3_renderers.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/c3_renderers.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/export_renderers.min.js"></script>
<style>
body {font-family: Verdana;}
.node {
border: solid 1px white;
font: 10px sans-serif;
line-height: 12px;
overflow: hidden;
position: absolute;
text-indent: 2px;
}
.c3-line, .c3-focused {stroke-width: 3px !important;}
.c3-bar {stroke: white !important; stroke-width: 1;}
.c3 text { font-size: 12px; color: grey;}
.tick line {stroke: white;}
.c3-axis path {stroke: grey;}
.c3-circle { opacity: 1 !important; }
</style>
</head>
<body>
<script type="text/javascript">
$(function(){
if(window.location != window.parent.location)
$("<a>", {target:"_blank", href:""})
.text("[pop out]").prependTo($("body"));
$("#output").pivotUI(
$.csv.toArrays($("#output").text()),
{
renderers: $.extend(
$.pivotUtilities.renderers,
$.pivotUtilities.c3_renderers,
$.pivotUtilities.d3_renderers,
$.pivotUtilities.export_renderers
),
hiddenAttributes: [""]
}
).show();
});
</script>
<div id="output" style="display: none;">%s</div>
</body>
</html>
""" """
error_log_list = self.error_log._getLog() html_string = template % df.to_csv()
if error_log_list: from hashlib import sha512
if isinstance(previous, int): key = sha512(html_string).hexdigest()
# We need to get the object for last index of list storeIFrame(self, html_string, key)
error = error_log_list[-previous] url = "%s/Base_displayPivotTableFrame?key=%s" % (erp5_url, key)
global status, ename, evalue, tb_list return IFrame(src=url, width='100%', height='500')
status = u'error'
ename = unicode(error['type'])
evalue = unicode(error['value'])
tb_list = [l+'\n' for l in error['tb_text'].split('\n')]
return None
\ No newline at end of file
...@@ -46,14 +46,14 @@ ...@@ -46,14 +46,14 @@
<key> <string>text_content_warning_message</string> </key> <key> <string>text_content_warning_message</string> </key>
<value> <value>
<tuple> <tuple>
<string>W: 58, 2: Using the global statement (global-statement)</string> <string>W: 85, 2: Redefining name \'traceback\' from outer scope (line 9) (redefined-outer-name)</string>
<string>W:104, 8: Use of exec (exec-used)</string> <string>W:197, 8: Use of exec (exec-used)</string>
<string>W:129, 10: Use of exec (exec-used)</string> <string>W:252, 8: Use of exec (exec-used)</string>
<string>W:146, 10: Use of exec (exec-used)</string> <string>W:264, 8: Use of exec (exec-used)</string>
<string>W:208, 2: Unused variable \'etype\' (unused-variable)</string> <string>W:327, 10: Unused variable \'mime_type\' (unused-variable)</string>
<string>W:208, 16: Unused variable \'tb\' (unused-variable)</string> <string>W:462, 2: Redefining name \'IFrame\' from outer scope (line 17) (redefined-outer-name)</string>
<string>W:269, 4: Using the global statement (global-statement)</string> <string>W: 9, 0: Unused import traceback (unused-import)</string>
<string>W:369, 2: Using the global statement (global-statement)</string> <string>W: 13, 0: Unused import transaction (unused-import)</string>
</tuple> </tuple>
</value> </value>
</item> </item>
......
<html>
<head>
<script type="text/javascript" src="http://localhost:2200/erp5/jquery/core/jquery.min.js"></script>
<script type="text/javascript" src="http://localhost:2200/erp5/jquery/ui/js/jquery-ui.min.js"></script>
<script type="text/javascript" src="http://evanplaice.github.io/jquery-csv/src/jquery.csv.js"></script>
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/c3/0.4.10/c3.min.css">
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/c3/0.4.10/c3.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/pivot.min.js"></script>
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/pivot.min.css">
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/d3_renderers.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/c3_renderers.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pivottable/2.0.2/export_renderers.min.js"></script>
<script type="text/javascript">
$(document).ready(function () {
$('#filter_button').on('click', function (){
var data = $('form').serializeArray();
$('#pivottablejs').html('Loading...');
$.ajax({
type: "POST",
url: "http://localhost:2200/erp5/portal_skins/erp5_inventory_pandas/filterDataFrame?as_csv=True",
data: data,
success: function (response) {
var input = $.csv.toArrays(response);
$('#pivottablejs').pivotUI(input, {
renderers: $.extend(
$.pivotUtilities.renderers,
$.pivotUtilities.c3_renderers,
$.pivotUtilities.d3_renderers,
$.pivotUtilities.export_renderers
),
hiddenAttributes: [""],
rows: 'Sequence',
cols: 'Data'
});
},
error: function (response) {
$('#pivottablejs').html('Error while requesting data from server.');
}
})
});
});
</script>
</head>
<body>
<h1>Integration between Pandas-based Inventory API and PivotTableJS</h1>
<p><b>NOTE: for this protoype the code will use the Big Array object with title "Wendelin + Jupyter"</b></p>
<form>
<fieldset>
<legend>Is accountable?</legend>
<input type="radio" name="is_accountable" value="1" checked> Yes
<input type="radio" name="is_accountable" value="0"> No
</fieldset>
<fieldset>
<legend>Omit</legend>
<input type="checkbox" name="omit_input"> Omit Input
<input type="checkbox" name="omit_output"> Omit Output
<input type="checkbox" name="omit_asset_increase"> Omit Asset Increase
<input type="checkbox" name="omit_asset_decrease"> Omit Asset Decrease
</fieldset>
<fieldset>
<legend>Simulation State</legend>
<p>Simulation State: <input name="simulation_state"></p>
<p>Input Simulation State: <input name="input_simulation_state"></p>
<p>Output Simulation State: <input name="output_simulation_state"></p>
</fieldset>
<fieldset>
<legend>Dates</legend>
<p>From date (yyyy-mm-dd): <input name="from_date"></p>
<p>To date (yyyy-mm-dd): <input name="to_date"></p>
</fieldset>
<button type="button" id="filter_button">Filter!</button>
</form>
<div id="pivottablejs">
</div>
</body>
</html>
\ No newline at end of file
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Web Page" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Anonymous</string>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Change_local_roles_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Anonymous</string>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>classification/collaborative/team</string>
</tuple>
</value>
</item>
<item>
<key> <string>content_md5</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>content_type</string> </key>
<value> <string>text/html</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>PivotTableJs_getMovementHistoryList</string> </value>
</item>
<item>
<key> <string>language</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Web Page</string> </value>
</item>
<item>
<key> <string>short_title</string> </key>
<value> <string>getMovementHistoryList</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Web Interface for getMovementHistoryList</string> </value>
</item>
<item>
<key> <string>version</string> </key>
<value>
<none/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>Base_displayHTML</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>JupyterCompile</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_displayHTML</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
memcached_tool = context.getPortalObject().portal_memcached
memcached_dict = memcached_tool.getMemcachedDict(key_prefix='pivottablejs', plugin_path='portal_memcached/default_memcached_plugin')
return memcached_dict[key]
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>key</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_displayPivotTableFrame</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>displayPivotTableFrame</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>erp5PivotTableUI</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>JupyterCompile</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_erp5PivotTableUI</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -73,7 +73,9 @@ class TestExecuteJupyter(ERP5TypeTestCase): ...@@ -73,7 +73,9 @@ class TestExecuteJupyter(ERP5TypeTestCase):
def testJupyterCompileErrorRaise(self): def testJupyterCompileErrorRaise(self):
""" """
Test if JupyterCompile portal_component raises error on the server side. Test if JupyterCompile portal_component correctly catches exceptions as
expected by the Jupyter frontend as also automatically abort the current
transaction.
Take the case in which one line in a statement is valid and another is not. Take the case in which one line in a statement is valid and another is not.
""" """
portal = self.getPortalObject() portal = self.getPortalObject()
...@@ -365,19 +367,20 @@ import sys ...@@ -365,19 +367,20 @@ import sys
self.assertEquals(json.loads(result)['code_result'].rstrip(), 'imghdr') self.assertEquals(json.loads(result)['code_result'].rstrip(), 'imghdr')
self.assertEquals(json.loads(result)['mime_type'].rstrip(), 'text/plain') self.assertEquals(json.loads(result)['mime_type'].rstrip(), 'text/plain')
def testBaseDisplayImageERP5Image(self): def testERP5ImageProcessor(self):
""" """
Test the fucntioning of Base_displayImage external method of erp5_data_notebook Test the fucntioning of the ERP5ImageProcessor and the custom system
BT5 for ERP5 image object as parameter and change display hook too.
""" """
self.image_module = self.portal.getDefaultModule('Image') self.image_module = self.portal.getDefaultModule('Image')
self.assertTrue(self.image_module is not None) self.assertTrue(self.image_module is not None)
# Create a new ERP5 image object # Create a new ERP5 image object
reference = 'testBase_displayImageReference' reference = 'testBase_displayImageReference5'
data_template = '<img src="data:application/unknown;base64,%s" /><br />'
data = 'qwertyuiopasdfghjklzxcvbnm<somerandomcharacterstosaveasimagedata>' data = 'qwertyuiopasdfghjklzxcvbnm<somerandomcharacterstosaveasimagedata>'
self.image_module.newContent( self.image_module.newContent(
portal_type='Image', portal_type='Image',
id='testBase_displayImageID', id='testBase_displayImageID5',
reference=reference, reference=reference,
data=data, data=data,
filename='test.png' filename='test.png'
...@@ -387,7 +390,7 @@ import sys ...@@ -387,7 +390,7 @@ import sys
# Call Base_displayImage from inside of Base_runJupyter # Call Base_displayImage from inside of Base_runJupyter
jupyter_code = """ jupyter_code = """
image = context.portal_catalog.getResultValue(portal_type='Image',reference='%s') image = context.portal_catalog.getResultValue(portal_type='Image',reference='%s')
context.Base_displayImage(image_object=image) context.Base_renderAsHtml(image)
"""%reference """%reference
local_variable_dict = {'imports' : {}, 'variables' : {}} local_variable_dict = {'imports' : {}, 'variables' : {}}
...@@ -396,7 +399,7 @@ context.Base_displayImage(image_object=image) ...@@ -396,7 +399,7 @@ context.Base_displayImage(image_object=image)
old_local_variable_dict=local_variable_dict old_local_variable_dict=local_variable_dict
) )
self.assertEquals(result['result_string'].rstrip(), base64.b64encode(data)) self.assertEquals(result['result_string'].rstrip(), data_template % base64.b64encode(data))
# Mime_type shouldn't be image/png just because of filename, instead it is # Mime_type shouldn't be image/png just because of filename, instead it is
# dependent on file and file data # dependent on file and file data
self.assertNotEqual(result['mime_type'], 'image/png') self.assertNotEqual(result['mime_type'], 'image/png')
...@@ -435,3 +438,32 @@ context.Base_displayImage(image_object=image) ...@@ -435,3 +438,32 @@ context.Base_displayImage(image_object=image)
python_expression=jupyter_code2 python_expression=jupyter_code2
) )
self.assertEquals(json.loads(result)['code_result'].rstrip(), 'sys') self.assertEquals(json.loads(result)['code_result'].rstrip(), 'sys')
def testPivotTableJsIntegration(self):
'''
This test ensures the PivotTableJs user interface is correctly integrated
into our Jupyter kernel.
'''
portal = self.portal
self.login('dev_user')
jupyter_code = '''
class DataFrameMock(object):
def to_csv(self):
return "column1, column2; 1, 2;"
my_df = DataFrameMock()
iframe = context.Base_erp5PivotTableUI(my_df, 'https://localhost:2202/erp5')
context.Base_renderAsHtml(iframe)
'''
reference = 'Test.Notebook.PivotTableJsIntegration %s' % time.time()
notebook = self._newNotebook(reference=reference)
result = portal.Base_executeJupyter(
reference=reference,
python_expression=jupyter_code
)
json_result = json.loads(result)
# The big hash in this string was previous calculated using the expect hash
# of the pivot table page's html.
pivottable_frame_display_path = 'Base_displayPivotTableFrame?key=853524757258b19805d13beb8c6bd284a7af4a974a96a3e5a4847885df069a74d3c8c1843f2bcc4d4bb3c7089194b57c90c14fe8dd0c776d84ce0868e19ac411'
self.assertTrue(pivottable_frame_display_path in json_result['code_result'])
...@@ -43,7 +43,9 @@ ...@@ -43,7 +43,9 @@
<item> <item>
<key> <string>text_content_warning_message</string> </key> <key> <string>text_content_warning_message</string> </key>
<value> <value>
<tuple/> <tuple>
<string>W:457, 4: Unused variable \'notebook\' (unused-variable)</string>
</tuple>
</value> </value>
</item> </item>
<item> <item>
......
...@@ -2,4 +2,5 @@ Interaction between Jupyter(IPython Notebook) and ERP5. ...@@ -2,4 +2,5 @@ Interaction between Jupyter(IPython Notebook) and ERP5.
!WARNING! !WARNING!
This business template is unsafe to install on a public server as one of the extensions uses eval and allows remote code execution. Proper security should be taken into account. This business template is unsafe to install on a public server as one of the extensions uses eval and allows remote code execution. Proper security should be taken into account.
This template includes a highly exprimental integration with PivotTableJs which doesn't follow ERP5 Javascript standards and will be refactored to use JIO and RenderJS.
!WARNING! !WARNING!
\ No newline at end of file
web_page_module/PivotTableJs_getMovementHistoryList
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment