Commit 02214c63 authored by Ivan Tyagov's avatar Ivan Tyagov

Add a generic implementation of transforming appended data in a "close to real time" way.

parent a81baf75
...@@ -49,7 +49,7 @@ ...@@ -49,7 +49,7 @@
</chain> </chain>
<chain> <chain>
<type>Data Stream</type> <type>Data Stream</type>
<workflow>edit_workflow, validation_workflow</workflow> <workflow>data_stream_interaction_workflow, edit_workflow, validation_workflow</workflow>
</chain> </chain>
<chain> <chain>
<type>Data Supply</type> <type>Data Supply</type>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>"""\n
Read tail of a Data Stream and aplly needed transformations.\n
This script is called every time we appendData to a Stream\n
using data_stream_interaction_workflow.\n
\n
The idea is to provide close to real time data transformations.\n
As transformation is quite specific we leave this script empty so developers\n
can hook in and add needed transformations.\n
\n
Interesting here ti to use OffsetIndex property sheet which allows to place\n
marker till where a Data Stream has been processed.\n
"""\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataStream_transformTail</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="InteractionWorkflowDefinition" module="Products.ERP5.InteractionWorkflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>creation_guard</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Data Stream interaction workflow</string> </value>
</item>
<item>
<key> <string>groups</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>data_stream_interaction_workflow</string> </value>
</item>
<item>
<key> <string>manager_bypass</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Data Stream interaction workflow</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Interaction" module="Products.ERP5.Interaction"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_mapping</string> </key>
<value>
<dictionary/>
</value>
</item>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>interactions</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="InteractionDefinition" module="Products.ERP5.Interaction"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>actbox_category</string> </key>
<value> <string>workflow</string> </value>
</item>
<item>
<key> <string>actbox_name</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>actbox_url</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>activate_script_name</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>after_script_name</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>before_commit_script_name</string> </key>
<value>
<list>
<string>DataStream_appendData</string>
</list>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>guard</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataStream_appendData</string> </value>
</item>
<item>
<key> <string>method_id</string> </key>
<value>
<list>
<string>_appendData</string>
</list>
</value>
</item>
<item>
<key> <string>once_per_transaction</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>portal_type_filter</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>portal_type_group_filter</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>script_name</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>temporary_document_disallowed</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>trigger_type</string> </key>
<value> <int>2</int> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Scripts" module="Products.DCWorkflow.Scripts"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_mapping</string> </key>
<value>
<dictionary/>
</value>
</item>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>scripts</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>"""\n
Handle appended data chunks.\n
"""\n
data_stream = state_change[\'object\']\n
\n
# call you own script to handle newly appended data which \n
# is not processed yet\n
data_stream.DataStream_transformTail()\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>state_change,**kw</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataStream_appendData</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Variables" module="Products.DCWorkflow.Variables"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_mapping</string> </key>
<value>
<dictionary/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>variables</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Worklists" module="Products.DCWorkflow.Worklists"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_mapping</string> </key>
<value>
<dictionary/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>worklists</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Wendelin is a Big Data platform based on ERP5. Wendelin is a Big Data platform based on ERP5.
More information can be found at http://www.wendelin.io/ More information can be found at http://www.wendelin.io/
\ No newline at end of file
...@@ -20,6 +20,7 @@ Data Release | edit_workflow ...@@ -20,6 +20,7 @@ Data Release | edit_workflow
Data Release | validation_workflow Data Release | validation_workflow
Data Set | edit_workflow Data Set | edit_workflow
Data Set | validation_workflow Data Set | validation_workflow
Data Stream | data_stream_interaction_workflow
Data Stream | edit_workflow Data Stream | edit_workflow
Data Stream | validation_workflow Data Stream | validation_workflow
Data Supply Line | edit_workflow Data Supply Line | edit_workflow
......
data_stream_interaction_workflow
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment