Skip to content

W
wendelin
Commit 20c49c81 authored by Ivan Tyagov's avatar Ivan Tyagov
Browse files
Options

Add ingestion dialog improvements. Use new data model so Ingestion Policy can be created quickly. 

Add test of this functionality.
parent 04a20e21
Hide whitespace changes
Inline Side-by-side
Showing with 289 additions and 40 deletions
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/IngestionPolicyTool_addIngestionPolicy.py
View file @ 20c49c81
"""
Create all required types for proper ingestion.
Create all required types for proper ingestion into Wendelin.
"""
from DateTime import DateTime
now = DateTime()
ingestion_policy = context.newContent( \
id = reference,
title = title,
portal_type ='Ingestion Policy',
reference = reference,
version = '001',
script_id = 'ERP5Site_handleDefaultFluentdIngestion')
script_id = 'IngestionPolicy_parseSimpleFluentdTag')
ingestion_policy.validate()
# create sensor
sensor = context.sensor_module.newContent( \
portal_type='Sensor',
reference = reference)
sensor.validate()
# create new Data Stream
data_stream = context.data_stream_module.newContent( \
portal_type='Data Stream', \
version = '001', \
reference=reference)
data_stream.validate()
use_category = context.restrictedTraverse("portal_categories/use/big_data/ingestion")
quantity_category = context.restrictedTraverse("portal_categories/quantity_unit/unit/piece")
data_operation = context.restrictedTraverse("data_operation_module/wendelin_1")
# create Data Product
data_product = context.data_product_module.newContent(
portal_type = "Data Product",
title = "Append to Data Stream",
reference = reference)
data_product.setUseValue(use_category)
data_product.setAggregatedPortalTypeList(["Data Stream"])
data_product.validate()
# create Data Supply
resource = context.restrictedTraverse('data_product_module/wendelin_4')
data_supply_kw = {'reference': reference,
data_supply_kw = {'title': title,
'reference': reference,
'version': '001',
'start_date': now,
'stop_date': now + 365}
data_supply_line_kw = {'resource_value': resource,
'source_value': sensor,
'destination_value': data_stream}
data_supply = ingestion_policy.PortalIngestionPolicy_addDataSupply( \
data_supply_kw, \
data_supply_line_kw)
data_array = context.data_array_module.newContent(
portal_type='Data Array',
reference = reference,
version = '001')
data_array.validate()
'effective_date': now,
'expiration_date': now + 365*10}
data_supply = context.data_supply_module.newContent( \
portal_type='Data Supply', **data_supply_kw)
data_supply.validate()
# add ingestion line
data_supply_line_kw = {'title': 'Ingest Data',
'reference': 'ingestion_operation',
'int_index': 1,
'quantity': 1.0}
data_supply_line = data_supply.newContent(portal_type='Data Supply Line', \
**data_supply_line_kw)
data_supply_line.setResourceValue(data_operation)
# add append to Data Stream line
data_supply_line_kw = {'title': 'Data Stream',
'reference': 'out_stream',
'int_index': 2,
'quantity': 1.0}
data_supply_line = data_supply.newContent(portal_type='Data Supply Line', \
**data_supply_line_kw)
data_supply_line.setResourceValue(data_product)
data_supply_line.setUseValue(use_category)
if batch_mode:
return ingestion_policy, data_supply, data_stream, data_array
return ingestion_policy, data_supply, data_product
else:
# UI case
ingestion_policy.Base_redirect(\
......
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/IngestionPolicyTool_addIngestionPolicy.xml
View file @ 20c49c81
......@@ -50,7 +50,7 @@
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>reference=None, batch_mode=0</string> </value>
<value> <string>reference=None, title=None, batch_mode=0</string> </value>
</item>
<item>
<key> <string>id</string> </key>
......
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/IngestionPolicyTool_viewAddIngestionPolicyActionDialog.xml
View file @ 20c49c81
......@@ -71,6 +71,8 @@
<key> <string>left</string> </key>
<value>
<list>
<string>your_label</string>
<string>your_title</string>
<string>your_reference</string>
</list>
</value>
......
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/IngestionPolicyTool_viewAddIngestionPolicyActionDialog/your_label.xml 0 → 100644
View file @ 20c49c81
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>default</string>
<string>editable</string>
<string>enabled</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>your_label</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>default</string> </key>
<value> <string>Please insert below the reference which you will use to ingest data into Wendelin instance. It should be unique inside the instance.</string> </value>
</item>
<item>
<key> <string>editable</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>enabled</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_label_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>required</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string encoding="cdata"><![CDATA[
&nbsp;
]]></string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/IngestionPolicyTool_viewAddIngestionPolicyActionDialog/your_title.xml 0 → 100644
View file @ 20c49c81
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>editable</string>
<string>enabled</string>
<string>required</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>your_title</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>editable</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>enabled</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_string_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>required</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Ingestion Title</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/TestTemplateItem/portal_components/test.erp5.testWendelin.py
View file @ 20c49c81
......@@ -27,7 +27,6 @@
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from wendelin.bigarray.array_zodb import ZBigArray
from DateTime import DateTime
from cStringIO import StringIO
import msgpack
import numpy as np
......@@ -58,11 +57,15 @@ class Test(ERP5TypeTestCase):
simulating input from fluentd.
"""
portal = self.portal
request = portal.REQUEST
ingestion_policy = portal.restrictedTraverse("portal_ingestion_policies/wendelin_1")
data_supply = portal.restrictedTraverse("data_supply_module/wendelin_1")
reference = 'wendelin-default-ingestion'
# add brand new ingestion
reference = getRandomString()
ingestion_policy, data_supply, data_product = portal.portal_ingestion_policies.IngestionPolicyTool_addIngestionPolicy(
reference = reference,
title = reference,
batch_mode=1)
self.tic()
number_string_list = []
for my_list in list(chunks(range(0, 100001), 10)):
number_string_list.append(','.join([str(x) for x in my_list]))
......
bt5/erp5_wendelin/TestTemplateItem/portal_components/test.erp5.testWendelin.xml
View file @ 20c49c81
......@@ -45,7 +45,9 @@
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
<tuple>
<string>W: 63, 35: Unused variable \'data_product\' (unused-variable)</string>
</tuple>
</value>
</item>
<item>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7