Skip to content

erp5
erp5
Commit 98076a1b authored by Klaus Wölfel's avatar Klaus Wölfel
Browse files
Options

erp5_data_notebook: Add notebook conversion and execution

parent 220338f7
Show whitespace changes
Inline Side-by-side
Showing with 732 additions and 7 deletions
bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterConvert.py 0 → 100644
View file @ 98076a1b
import nbformat
import ssl
import re
from base64 import b64encode, b64decode
from nbconvert import HTMLExporter
from nbconvert.preprocessors import Preprocessor, ExecutePreprocessor
from xmlrpclib import ServerProxy
from urllib import urlencode
from urllib2 import urlopen
class PyMarkdownPreprocessor(Preprocessor):
"""
:mod:`nbconvert` Preprocessor for the python-markdown nbextension.
This :class:`~nbconvert.preprocessors.Preprocessor` replaces kernel code in
markdown cells with the results stored in the cell metadata.
"""
def replace_variables(self, source, variables):
"""
Replace {{variablename}} with stored value
"""
try:
replaced = re.sub(
"{{(.*?)}}", lambda m: variables.get(m.group(1), ''), source)
except TypeError:
replaced = source
return replaced
def preprocess_cell(self, cell, resources, index):
"""
Preprocess cell
Parameters
----------
cell : NotebookNode cell
Notebook cell being processed
resources : dictionary
Additional resources used in the conversion process. Allows
preprocessors to pass variables into the Jinja engine.
cell_index : int
Index of the cell being processed (see base.py)
"""
if cell.cell_type == "markdown":
if hasattr(cell['metadata'], 'variables'):
variables = cell['metadata']['variables']
if len(variables) > 0:
cell.source = self.replace_variables(
cell.source, variables)
return cell, resources
def to_html(self):
notebook = nbformat.reads(self.getTextContent(), as_version=4)
resources = {}
notebook = \
ExecutePreprocessor(kernel_name="erp5").preprocess(notebook, resources)[0]
notebook = PyMarkdownPreprocessor().preprocess(notebook, resources)[0]
html_exporter = HTMLExporter()
html_exporter.template_file = 'full'
body, resources = html_exporter.from_notebook_node(notebook)
return body
def cloudoooConvertFile(self, data, source_mimetype, destination_mimetype, zip=False, refresh=False, conversion_kw=None):
url = 'https://softinst78992.host.vifib.net/erp5/ERP5Site_htmlToPdf'
data_dict = {'data' : b64encode(data)}
data_dict.update(**conversion_kw)
return urlopen(url=url, data=urlencode(data_dict)).read()
proxy = ServerProxy(self.getPortalObject().portal_preferences.getPreferredDocumentConversionServerUrl(), allow_none=True)
return b64decode(proxy.convertFile(b64encode(data), source_mimetype, destination_mimetype, zip, refresh, conversion_kw or {}))
bt5/erp5_data_notebook/ExtensionTemplateItem/portal_components/extension.erp5.JupyterConvert.xml 0 → 100644
View file @ 98076a1b
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>JupyterConvert</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.JupyterConvert</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple>
<string>W: 65, 75: Redefining built-in \'zip\' (redefined-builtin)</string>
<string>W: 70, 2: Unreachable code (unreachable)</string>
<string>W: 2, 0: Unused import ssl (unused-import)</string>
</tuple>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
bt5/erp5_data_notebook/PortalTypeTemplateItem/portal_types/Data%20Notebook.xml
View file @ 98076a1b
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<ZopeData> <ZopeData>
<record id="1" aka="AAAAAAAAAAE="> <record id="1" aka="AAAAAAAAAAE=">
<pickle> <pickle>
<global name="Base Type" module="erp5.portal_type"/> <global name="JSON Type" module="erp5.portal_type"/>
</pickle> </pickle>
<pickle> <pickle>
<dictionary> <dictionary>
...@@ -18,10 +18,6 @@ ...@@ -18,10 +18,6 @@
<none/> <none/>
</value> </value>
</item> </item>
<item>
<key> <string>factory</string> </key>
<value> <string>addXMLObject</string> </value>
</item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>Data Notebook</string> </value> <value> <string>Data Notebook</string> </value>
...@@ -40,7 +36,7 @@ ...@@ -40,7 +36,7 @@
</item> </item>
<item> <item>
<key> <string>portal_type</string> </key> <key> <string>portal_type</string> </key>
<value> <string>Base Type</string> </value> <value> <string>JSON Type</string> </value>
</item> </item>
<item> <item>
<key> <string>searchable_text_property_id</string> </key> <key> <string>searchable_text_property_id</string> </key>
...@@ -54,6 +50,406 @@ ...@@ -54,6 +50,406 @@
</tuple> </tuple>
</value> </value>
</item> </item>
<item>
<key> <string>text_content</string> </key>
<value> <string>{\n
"$schema": "http://json-schema.org/draft-04/schema#",\n
"description": "Jupyter Notebook v4.2 JSON schema.",\n
"type": "object",\n
"additionalProperties": false,\n
"required": ["metadata", "nbformat_minor", "nbformat", "cells"],\n
"properties": {\n
"metadata": {\n
"description": "Notebook root-level metadata.",\n
"type": "object",\n
"additionalProperties": true,\n
"properties": {\n
"kernelspec": {\n
"description": "Kernel information.",\n
"type": "object",\n
"required": ["name", "display_name"],\n
"properties": {\n
"name": {\n
"description": "Name of the kernel specification.",\n
"type": "string"\n
},\n
"display_name": {\n
"description": "Name to display in UI.",\n
"type": "string"\n
}\n
}\n
},\n
"language_info": {\n
"description": "Kernel information.",\n
"type": "object",\n
"required": ["name"],\n
"properties": {\n
"name": {\n
"description": "The programming language which this kernel runs.",\n
"type": "string"\n
},\n
"codemirror_mode": {\n
"description": "The codemirror mode to use for code in this language.",\n
"oneOf": [\n
{"type": "string"},\n
{"type": "object"}\n
]\n
},\n
"file_extension": {\n
"description": "The file extension for files in this language.",\n
"type": "string"\n
},\n
"mimetype": {\n
"description": "The mimetype corresponding to files in this language.",\n
"type": "string"\n
},\n
"pygments_lexer": {\n
"description": "The pygments lexer to use for code in this language.",\n
"type": "string"\n
}\n
}\n
},\n
"orig_nbformat": {\n
"description": "Original notebook format (major number) before converting the notebook between versions. This should never be written to a file.",\n
"type": "integer",\n
"minimum": 1\n
},\n
"title": {\n
"description": "The title of the notebook document",\n
"type": "string"\n
},\n
"authors": {\n
"description": "The author(s) of the notebook document",\n
"type": "array",\n
"item": {\n
"type": "object",\n
"properties": {\n
"name": {\n
"type": "string"\n
}\n
},\n
"additionalProperties": true\n
}\n
}\n
}\n
},\n
"nbformat_minor": {\n
"description": "Notebook format (minor number). Incremented for backward compatible changes to the notebook format.",\n
"type": "integer",\n
"minimum": 0\n
},\n
"nbformat": {\n
"description": "Notebook format (major number). Incremented between backwards incompatible changes to the notebook format.",\n
"type": "integer",\n
"minimum": 4,\n
"maximum": 4\n
},\n
"cells": {\n
"description": "Array of cells of the current notebook.",\n
"type": "array",\n
"items": {"$ref": "#/definitions/cell"}\n
}\n
},\n
\n
"definitions": {\n
"cell": {\n
"type": "object",\n
"oneOf": [\n
{"$ref": "#/definitions/raw_cell"},\n
{"$ref": "#/definitions/markdown_cell"},\n
{"$ref": "#/definitions/code_cell"}\n
]\n
},\n
\n
"raw_cell": {\n
"description": "Notebook raw nbconvert cell.",\n
"type": "object",\n
"additionalProperties": false,\n
"required": ["cell_type", "metadata", "source"],\n
"properties": {\n
"cell_type": {\n
"description": "String identifying the type of cell.",\n
"enum": ["raw"]\n
},\n
"metadata": {\n
"description": "Cell-level metadata.",\n
"type": "object",\n
"additionalProperties": true,\n
"properties": {\n
"format": {\n
"description": "Raw cell metadata format for nbconvert.",\n
"type": "string"\n
},\n
"name": {"$ref": "#/definitions/misc/metadata_name"},\n
"tags": {"$ref": "#/definitions/misc/metadata_tags"}\n
}\n
},\n
"attachments": {"$ref": "#/definitions/misc/attachments"},\n
"source": {"$ref": "#/definitions/misc/source"}\n
}\n
},\n
\n
"markdown_cell": {\n
"description": "Notebook markdown cell.",\n
"type": "object",\n
"additionalProperties": false,\n
"required": ["cell_type", "metadata", "source"],\n
"properties": {\n
"cell_type": {\n
"description": "String identifying the type of cell.",\n
"enum": ["markdown"]\n
},\n
"metadata": {\n
"description": "Cell-level metadata.",\n
"type": "object",\n
"properties": {\n
"name": {"$ref": "#/definitions/misc/metadata_name"},\n
"tags": {"$ref": "#/definitions/misc/metadata_tags"}\n
},\n
"additionalProperties": true\n
},\n
"attachments": {"$ref": "#/definitions/misc/attachments"},\n
"source": {"$ref": "#/definitions/misc/source"}\n
}\n
},\n
\n
"code_cell": {\n
"description": "Notebook code cell.",\n
"type": "object",\n
"additionalProperties": false,\n
"required": ["cell_type", "metadata", "source", "outputs", "execution_count"],\n
"properties": {\n
"cell_type": {\n
"description": "String identifying the type of cell.",\n
"enum": ["code"]\n
},\n
"metadata": {\n
"description": "Cell-level metadata.",\n
"type": "object",\n
"additionalProperties": true,\n
"properties": {\n
"collapsed": {\n
"description": "Whether the cell is collapsed/expanded.",\n
"type": "boolean"\n
},\n
"scrolled": {\n
"description": "Whether the cell\'s output is scrolled, unscrolled, or autoscrolled.",\n
"enum": [true, false, "auto"]\n
},\n
"name": {"$ref": "#/definitions/misc/metadata_name"},\n
"tags": {"$ref": "#/definitions/misc/metadata_tags"}\n
}\n
},\n
"source": {"$ref": "#/definitions/misc/source"},\n
"outputs": {\n
"description": "Execution, display, or stream outputs.",\n
"type": "array",\n
"items": {"$ref": "#/definitions/output"}\n
},\n
"execution_count": {\n
"description": "The code cell\'s prompt number. Will be null if the cell has not been run.",\n
"type": ["integer", "null"],\n
"minimum": 0\n
}\n
}\n
},\n
\n
"unrecognized_cell": {\n
"description": "Unrecognized cell from a future minor-revision to the notebook format.",\n
"type": "object",\n
"additionalProperties": true,\n
"required": ["cell_type", "metadata"],\n
"properties": {\n
"cell_type": {\n
"description": "String identifying the type of cell.",\n
"not" : {\n
"enum": ["markdown", "code", "raw"]\n
}\n
},\n
"metadata": {\n
"description": "Cell-level metadata.",\n
"type": "object",\n
"properties": {\n
"name": {"$ref": "#/definitions/misc/metadata_name"},\n
"tags": {"$ref": "#/definitions/misc/metadata_tags"}\n
},\n
"additionalProperties": true\n
}\n
}\n
},\n
\n
"output": {\n
"type": "object",\n
"oneOf": [\n
{"$ref": "#/definitions/execute_result"},\n
{"$ref": "#/definitions/display_data"},\n
{"$ref": "#/definitions/stream"},\n
{"$ref": "#/definitions/error"}\n
]\n
},\n
\n
"execute_result": {\n
"description": "Result of executing a code cell.",\n
"type": "object",\n
"additionalProperties": false,\n
"required": ["output_type", "data", "metadata", "execution_count"],\n
"properties": {\n
"output_type": {\n
"description": "Type of cell output.",\n
"enum": ["execute_result"]\n
},\n
"execution_count": {\n
"description": "A result\'s prompt number.",\n
"type": ["integer", "null"],\n
"minimum": 0\n
},\n
"data": {"$ref": "#/definitions/misc/mimebundle"},\n
"metadata": {"$ref": "#/definitions/misc/output_metadata"}\n
}\n
},\n
\n
"display_data": {\n
"description": "Data displayed as a result of code cell execution.",\n
"type": "object",\n
"additionalProperties": false,\n
"required": ["output_type", "data", "metadata"],\n
"properties": {\n
"output_type": {\n
"description": "Type of cell output.",\n
"enum": ["display_data"]\n
},\n
"data": {"$ref": "#/definitions/misc/mimebundle"},\n
"metadata": {"$ref": "#/definitions/misc/output_metadata"}\n
}\n
},\n
\n
"stream": {\n
"description": "Stream output from a code cell.",\n
"type": "object",\n
"additionalProperties": false,\n
"required": ["output_type", "name", "text"],\n
"properties": {\n
"output_type": {\n
"description": "Type of cell output.",\n
"enum": ["stream"]\n
},\n
"name": {\n
"description": "The name of the stream (stdout, stderr).",\n
"type": "string"\n
},\n
"text": {\n
"description": "The stream\'s text output, represented as an array of strings.",\n
"$ref": "#/definitions/misc/multiline_string"\n
}\n
}\n
},\n
\n
"error": {\n
"description": "Output of an error that occurred during code cell execution.",\n
"type": "object",\n
"additionalProperties": false,\n
"required": ["output_type", "ename", "evalue", "traceback"],\n
"properties": {\n
"output_type": {\n
"description": "Type of cell output.",\n
"enum": ["error"]\n
},\n
"ename": {\n
"description": "The name of the error.",\n
"type": "string"\n
},\n
"evalue": {\n
"description": "The value, or message, of the error.",\n
"type": "string"\n
},\n
"traceback": {\n
"description": "The error\'s traceback, represented as an array of strings.",\n
"type": "array",\n
"items": {"type": "string"}\n
}\n
}\n
},\n
\n
"unrecognized_output": {\n
"description": "Unrecognized output from a future minor-revision to the notebook format.",\n
"type": "object",\n
"additionalProperties": true,\n
"required": ["output_type"],\n
"properties": {\n
"output_type": {\n
"description": "Type of cell output.",\n
"not": {\n
"enum": ["execute_result", "display_data", "stream", "error"]\n
}\n
}\n
}\n
},\n
\n
"misc": {\n
"metadata_name": {\n
"description": "The cell\'s name. If present, must be a non-empty string. Must be unique across all the cells of a given notebook.",\n
"type": "string",\n
"pattern": "^.+$"\n
},\n
"metadata_tags": {\n
"description": "The cell\'s tags. Tags must be unique, and must not contain commas.",\n
"type": "array",\n
"uniqueItems": true,\n
"items": {\n
"type": "string",\n
"pattern": "^[^,]+$"\n
}\n
},\n
"attachments": {\n
"description": "Media attachments (e.g. inline images), stored as mimebundle keyed by filename.",\n
"type": "object",\n
"patternProperties": {\n
".*": {\n
"description": "The attachment\'s data stored as a mimebundle.",\n
"$ref": "#/definitions/misc/mimebundle"\n
}\n
}\n
},\n
"source": {\n
"description": "Contents of the cell, represented as an array of lines.",\n
"$ref": "#/definitions/misc/multiline_string"\n
},\n
"execution_count": {\n
"description": "The code cell\'s prompt number. Will be null if the cell has not been run.",\n
"type": ["integer", "null"],\n
"minimum": 0\n
},\n
"mimebundle": {\n
"description": "A mime-type keyed dictionary of data",\n
"type": "object",\n
"additionalProperties": {\n
"description": "mimetype output (e.g. text/plain), represented as either an array of strings or a string.",\n
"$ref": "#/definitions/misc/multiline_string"\n
},\n
"patternProperties": {\n
"^application/(.*\\\\+)?json$": {\n
"description": "Mimetypes with JSON output, can be any type"\n
}\n
}\n
},\n
"output_metadata": {\n
"description": "Cell output metadata.",\n
"type": "object",\n
"additionalProperties": true\n
},\n
"multiline_string": {\n
"oneOf" : [\n
{"type": "string"},\n
{\n
"type": "array",\n
"items": {"type": "string"}\n
}\n
]\n
}\n
}\n
}\n
}</string> </value>
</item>
<item> <item>
<key> <string>type_class</string> </key> <key> <string>type_class</string> </key>
<value> <string>XMLObject</string> </value> <value> <string>XMLObject</string> </value>
......
bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/DataNotebook_toHtml.xml 0 → 100644
View file @ 98076a1b
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>to_html</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>JupyterConvert</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataNotebook_toHtml</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/DataNotebook_toPdf.py 0 → 100644
View file @ 98076a1b
from base64 import b64encode
portal = context.getPortalObject()
#web_page = portal.web_page_module.newContent(
# text_content = text_content,
# temp_object=1,
# title="Bericht")
web_page = context.web_page_module["6"]
text_content = context.DataNotebook_toHtml()
web_page.setTextContent(text_content)
html_data = web_page.WebPage_exportAsSingleFile(format="html", allow_script=True)
pdf_file = portal.Base_cloudoooConvertFile(html_data, "html", "pdf",
conversion_kw=dict(
encoding="utf8",
margin_top=10,
margin_bottom=10,
margin_left=10,
margin_right=10,
header_spacing=0,
header_html_data=b64encode(""),
footer_html_data=b64encode(""),
print_media_type=True,
javascript_delay=9000,
page_size="A4",
zoom=1,
dpi="75"
)
)
REQUEST = context.REQUEST
if REQUEST is not None:
REQUEST.RESPONSE.setHeader("Content-Type", "application/pdf")
REQUEST.response.setHeader('Content-Length', len(pdf_file))
#REQUEST.response.setHeader('Content-Disposition',
# 'attachment;filename="%s.pdf"' % context.getTitle())
return pdf_file
bt5/erp5_data_notebook/SkinTemplateItem/portal_skins/erp5_data_notebook/DataNotebook_toPdf.xml 0 → 100644
View file @ 98076a1b
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataNotebook_toPdf</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_data_notebook/bt/template_extension_id_list
View file @ 98076a1b
extension.erp5.JupyterCompile extension.erp5.JupyterCompile
extension.erp5.JupyterConvert
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7