Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Jérome Perrin
wendelin
Commits
43f6f999
Commit
43f6f999
authored
Mar 02, 2021
by
Ivan Tyagov
Browse files
Options
Browse Files
Download
Plain Diff
Roque ebulk optimizations
See merge request
nexedi/wendelin!81
parents
f814135b
83b0372a
Changes
20
Hide whitespace changes
Inline
Side-by-side
Showing
20 changed files
with
1030 additions
and
34 deletions
+1030
-34
bt5/erp5_wendelin/ActionTemplateItem/portal_types/Data%20Set/data_streams.xml
...tionTemplateItem/portal_types/Data%20Set/data_streams.xml
+81
-0
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamCount.py
.../portal_skins/erp5_wendelin/DataSet_getDataStreamCount.py
+4
-0
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamCount.xml
...portal_skins/erp5_wendelin/DataSet_getDataStreamCount.xml
+1
-1
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamList.py
...m/portal_skins/erp5_wendelin/DataSet_getDataStreamList.py
+17
-0
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamList.xml
.../portal_skins/erp5_wendelin/DataSet_getDataStreamList.xml
+66
-0
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamRowList.py
...ortal_skins/erp5_wendelin/DataSet_getDataStreamRowList.py
+5
-0
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamRowList.xml
...rtal_skins/erp5_wendelin/DataSet_getDataStreamRowList.xml
+62
-0
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_viewDataStreams.xml
...em/portal_skins/erp5_wendelin/DataSet_viewDataStreams.xml
+138
-0
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_viewDataStreams/listbox.xml
...l_skins/erp5_wendelin/DataSet_viewDataStreams/listbox.xml
+164
-0
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_viewDataStreams/my_count_label.xml
.../erp5_wendelin/DataSet_viewDataStreams/my_count_label.xml
+154
-0
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_viewDataStreams/my_title.xml
..._skins/erp5_wendelin/DataSet_viewDataStreams/my_title.xml
+96
-0
bt5/erp5_wendelin/bt/template_action_path_list
bt5/erp5_wendelin/bt/template_action_path_list
+1
-0
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_checkMd5DataStreamList.py
...erp5_wendelin_data_lake/DataSet_checkMd5DataStreamList.py
+42
-0
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_checkMd5DataStreamList.xml
...rp5_wendelin_data_lake/DataSet_checkMd5DataStreamList.xml
+62
-0
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_getDataStreamList.py
...kins/erp5_wendelin_data_lake/DataSet_getDataStreamList.py
+0
-13
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamCount.py
...ns/erp5_wendelin_data_lake/ERP5Site_getDataStreamCount.py
+19
-0
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamCount.xml
...s/erp5_wendelin_data_lake/ERP5Site_getDataStreamCount.xml
+62
-0
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamList.py
...ins/erp5_wendelin_data_lake/ERP5Site_getDataStreamList.py
+28
-19
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamList.xml
...ns/erp5_wendelin_data_lake/ERP5Site_getDataStreamList.xml
+1
-1
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
...Item/portal_components/test.erp5.testDataLakeIngestion.py
+27
-0
No files found.
bt5/erp5_wendelin/ActionTemplateItem/portal_types/Data%20Set/data_streams.xml
0 → 100644
View file @
43f6f999
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"ActionInformation"
module=
"Products.CMFCore.ActionInformation"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
action
</string>
</key>
<value>
<persistent>
<string
encoding=
"base64"
>
AAAAAAAAAAI=
</string>
</persistent>
</value>
</item>
<item>
<key>
<string>
categories
</string>
</key>
<value>
<tuple>
<string>
action_type/object_view
</string>
</tuple>
</value>
</item>
<item>
<key>
<string>
category
</string>
</key>
<value>
<string>
object_view
</string>
</value>
</item>
<item>
<key>
<string>
condition
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
description
</string>
</key>
<value>
<none/>
</value>
</item>
<item>
<key>
<string>
icon
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
data_streams
</string>
</value>
</item>
<item>
<key>
<string>
permissions
</string>
</key>
<value>
<tuple>
<string>
View
</string>
</tuple>
</value>
</item>
<item>
<key>
<string>
priority
</string>
</key>
<value>
<float>
3.0
</float>
</value>
</item>
<item>
<key>
<string>
title
</string>
</key>
<value>
<string>
Data Streams
</string>
</value>
</item>
<item>
<key>
<string>
visible
</string>
</key>
<value>
<int>
1
</int>
</value>
</item>
</dictionary>
</pickle>
</record>
<record
id=
"2"
aka=
"AAAAAAAAAAI="
>
<pickle>
<global
name=
"Expression"
module=
"Products.CMFCore.Expression"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
text
</string>
</key>
<value>
<string>
string:${object_url}/DataSet_viewDataStreams
</string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamCount.py
0 → 100644
View file @
43f6f999
"""
Get count of Data Streams for context Data set.
"""
return
context
.
ERP5Site_getDataStreamCount
(
data_set_reference
=
context
.
getReference
())[
'result'
]
bt5/erp5_wendelin
_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_getDataStreamLis
t.xml
→
bt5/erp5_wendelin
/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamCoun
t.xml
View file @
43f6f999
...
@@ -54,7 +54,7 @@
...
@@ -54,7 +54,7 @@
</item>
</item>
<item>
<item>
<key>
<string>
id
</string>
</key>
<key>
<string>
id
</string>
</key>
<value>
<string>
DataSet_getDataStream
Lis
t
</string>
</value>
<value>
<string>
DataSet_getDataStream
Coun
t
</string>
</value>
</item>
</item>
</dictionary>
</dictionary>
</pickle>
</pickle>
...
...
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamList.py
0 → 100644
View file @
43f6f999
"""
This script will return all Data streams for Data set
"""
catalog_kw
=
{
'portal_type'
:
'Data Ingestion Line'
,
'aggregate_uid'
:
data_set_uid
,
'limit'
:
limit
,
}
data_ingestion_line_list
=
context
.
portal_catalog
(
**
catalog_kw
)
if
data_ingestion_line_list
:
data_ingestion_uid_list
=
[
x
.
uid
for
x
in
data_ingestion_line_list
]
catalog_kw
=
{
'portal_type'
:
'Data Stream'
,
'aggregate__related__uid'
:
data_ingestion_uid_list
,
'validation_state'
:
'validated'
,
'select_list'
:
[
'reference'
,
'relative_url'
,
'versioning.size'
,
'versioning.version'
],
}
return
context
.
getPortalObject
().
portal_catalog
(
**
catalog_kw
)
return
data_ingestion_line_list
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamList.xml
0 → 100644
View file @
43f6f999
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"PythonScript"
module=
"Products.PythonScripts.PythonScript"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
Script_magic
</string>
</key>
<value>
<int>
3
</int>
</value>
</item>
<item>
<key>
<string>
_bind_names
</string>
</key>
<value>
<object>
<klass>
<global
name=
"NameAssignments"
module=
"Shared.DC.Scripts.Bindings"
/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key>
<string>
_asgns
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
name_container
</string>
</key>
<value>
<string>
container
</string>
</value>
</item>
<item>
<key>
<string>
name_context
</string>
</key>
<value>
<string>
context
</string>
</value>
</item>
<item>
<key>
<string>
name_m_self
</string>
</key>
<value>
<string>
script
</string>
</value>
</item>
<item>
<key>
<string>
name_subpath
</string>
</key>
<value>
<string>
traverse_subpath
</string>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key>
<string>
_params
</string>
</key>
<value>
<string>
data_set_uid, limit=[]
</string>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
DataSet_getDataStreamList
</string>
</value>
</item>
<item>
<key>
<string>
title
</string>
</key>
<value>
<string></string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamRowList.py
0 → 100644
View file @
43f6f999
"""
Get list of Data Streams for context Data set.
"""
data_set_uid
=
context
.
getUid
()
return
context
.
DataSet_getDataStreamList
(
data_set_uid
,
limit
)
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_getDataStreamRowList.xml
0 → 100644
View file @
43f6f999
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"PythonScript"
module=
"Products.PythonScripts.PythonScript"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
Script_magic
</string>
</key>
<value>
<int>
3
</int>
</value>
</item>
<item>
<key>
<string>
_bind_names
</string>
</key>
<value>
<object>
<klass>
<global
name=
"NameAssignments"
module=
"Shared.DC.Scripts.Bindings"
/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key>
<string>
_asgns
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
name_container
</string>
</key>
<value>
<string>
container
</string>
</value>
</item>
<item>
<key>
<string>
name_context
</string>
</key>
<value>
<string>
context
</string>
</value>
</item>
<item>
<key>
<string>
name_m_self
</string>
</key>
<value>
<string>
script
</string>
</value>
</item>
<item>
<key>
<string>
name_subpath
</string>
</key>
<value>
<string>
traverse_subpath
</string>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key>
<string>
_params
</string>
</key>
<value>
<string>
list_start=0, list_lines=15, limit=1000, **kw
</string>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
DataSet_getDataStreamRowList
</string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_viewDataStreams.xml
0 → 100644
View file @
43f6f999
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"ERP5 Form"
module=
"erp5.portal_type"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
_objects
</string>
</key>
<value>
<tuple/>
</value>
</item>
<item>
<key>
<string>
action
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
action_title
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
description
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
edit_order
</string>
</key>
<value>
<list/>
</value>
</item>
<item>
<key>
<string>
encoding
</string>
</key>
<value>
<string>
UTF-8
</string>
</value>
</item>
<item>
<key>
<string>
enctype
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
group_list
</string>
</key>
<value>
<list>
<string>
left
</string>
<string>
right
</string>
<string>
center
</string>
<string>
bottom
</string>
<string>
hidden
</string>
</list>
</value>
</item>
<item>
<key>
<string>
groups
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
bottom
</string>
</key>
<value>
<list>
<string>
listbox
</string>
</list>
</value>
</item>
<item>
<key>
<string>
center
</string>
</key>
<value>
<list/>
</value>
</item>
<item>
<key>
<string>
hidden
</string>
</key>
<value>
<list/>
</value>
</item>
<item>
<key>
<string>
left
</string>
</key>
<value>
<list>
<string>
my_title
</string>
<string>
my_count_label
</string>
</list>
</value>
</item>
<item>
<key>
<string>
right
</string>
</key>
<value>
<list/>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
DataSet_viewDataStreams
</string>
</value>
</item>
<item>
<key>
<string>
method
</string>
</key>
<value>
<string>
POST
</string>
</value>
</item>
<item>
<key>
<string>
name
</string>
</key>
<value>
<string>
DataSet_viewDataStreams
</string>
</value>
</item>
<item>
<key>
<string>
pt
</string>
</key>
<value>
<string>
form_view
</string>
</value>
</item>
<item>
<key>
<string>
row_length
</string>
</key>
<value>
<int>
4
</int>
</value>
</item>
<item>
<key>
<string>
stored_encoding
</string>
</key>
<value>
<string>
UTF-8
</string>
</value>
</item>
<item>
<key>
<string>
title
</string>
</key>
<value>
<string>
View Data Streams
</string>
</value>
</item>
<item>
<key>
<string>
unicode_mode
</string>
</key>
<value>
<int>
0
</int>
</value>
</item>
<item>
<key>
<string>
update_action
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
update_action_title
</string>
</key>
<value>
<string></string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_viewDataStreams/listbox.xml
0 → 100644
View file @
43f6f999
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"ProxyField"
module=
"Products.ERP5Form.ProxyField"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
delegated_list
</string>
</key>
<value>
<list>
<string>
columns
</string>
<string>
editable
</string>
<string>
list_method
</string>
<string>
title
</string>
<string>
url_columns
</string>
</list>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
listbox
</string>
</value>
</item>
<item>
<key>
<string>
message_values
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
external_validator_failed
</string>
</key>
<value>
<string>
The input failed the external validator.
</string>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key>
<string>
overrides
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
field_id
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
form_id
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
target
</string>
</key>
<value>
<string></string>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key>
<string>
tales
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
editable
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
field_id
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
form_id
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
list_method
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
target
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
title
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
url_columns
</string>
</key>
<value>
<string></string>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key>
<string>
values
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
columns
</string>
</key>
<value>
<list>
<tuple>
<string>
title
</string>
<string>
Title
</string>
</tuple>
<tuple>
<string>
reference
</string>
<string>
Reference
</string>
</tuple>
<tuple>
<string>
size
</string>
<string>
Size
</string>
</tuple>
</list>
</value>
</item>
<item>
<key>
<string>
editable
</string>
</key>
<value>
<int>
0
</int>
</value>
</item>
<item>
<key>
<string>
field_id
</string>
</key>
<value>
<string>
my_view_mode_listbox
</string>
</value>
</item>
<item>
<key>
<string>
form_id
</string>
</key>
<value>
<string>
Base_viewFieldLibrary
</string>
</value>
</item>
<item>
<key>
<string>
list_method
</string>
</key>
<value>
<persistent>
<string
encoding=
"base64"
>
AAAAAAAAAAI=
</string>
</persistent>
</value>
</item>
<item>
<key>
<string>
target
</string>
</key>
<value>
<string>
Click to edit the target
</string>
</value>
</item>
<item>
<key>
<string>
title
</string>
</key>
<value>
<string>
View Data Streams
</string>
</value>
</item>
<item>
<key>
<string>
url_columns
</string>
</key>
<value>
<list/>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record
id=
"2"
aka=
"AAAAAAAAAAI="
>
<pickle>
<global
name=
"Method"
module=
"Products.Formulator.MethodField"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
method_name
</string>
</key>
<value>
<string>
DataSet_getDataStreamRowList
</string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_viewDataStreams/my_count_label.xml
0 → 100644
View file @
43f6f999
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"LabelField"
module=
"Products.Formulator.StandardFields"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
my_count_label
</string>
</value>
</item>
<item>
<key>
<string>
message_values
</string>
</key>
<value>
<dictionary/>
</value>
</item>
<item>
<key>
<string>
overrides
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
css_class
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
default
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
description
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
editable
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
enabled
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
extra
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
hidden
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
title
</string>
</key>
<value>
<string></string>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key>
<string>
tales
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
css_class
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
default
</string>
</key>
<value>
<persistent>
<string
encoding=
"base64"
>
AAAAAAAAAAI=
</string>
</persistent>
</value>
</item>
<item>
<key>
<string>
description
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
editable
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
enabled
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
extra
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
hidden
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
title
</string>
</key>
<value>
<string></string>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key>
<string>
values
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
css_class
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
default
</string>
</key>
<value>
<string>
lala
</string>
</value>
</item>
<item>
<key>
<string>
description
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
editable
</string>
</key>
<value>
<int>
0
</int>
</value>
</item>
<item>
<key>
<string>
enabled
</string>
</key>
<value>
<int>
1
</int>
</value>
</item>
<item>
<key>
<string>
extra
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
hidden
</string>
</key>
<value>
<int>
0
</int>
</value>
</item>
<item>
<key>
<string>
title
</string>
</key>
<value>
<string>
Total Data Streams Count
</string>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record
id=
"2"
aka=
"AAAAAAAAAAI="
>
<pickle>
<global
name=
"TALESMethod"
module=
"Products.Formulator.TALESField"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
_text
</string>
</key>
<value>
<string>
python: context.DataSet_getDataStreamCount()
</string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataSet_viewDataStreams/my_title.xml
0 → 100644
View file @
43f6f999
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"ProxyField"
module=
"Products.ERP5Form.ProxyField"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
delegated_list
</string>
</key>
<value>
<list>
<string>
editable
</string>
</list>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
my_title
</string>
</value>
</item>
<item>
<key>
<string>
message_values
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
external_validator_failed
</string>
</key>
<value>
<string>
The input failed the external validator.
</string>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key>
<string>
overrides
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
field_id
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
form_id
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
target
</string>
</key>
<value>
<string></string>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key>
<string>
tales
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
field_id
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
form_id
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
target
</string>
</key>
<value>
<string></string>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key>
<string>
values
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
editable
</string>
</key>
<value>
<int>
0
</int>
</value>
</item>
<item>
<key>
<string>
field_id
</string>
</key>
<value>
<string>
my_view_mode_title
</string>
</value>
</item>
<item>
<key>
<string>
form_id
</string>
</key>
<value>
<string>
Base_viewFieldLibrary
</string>
</value>
</item>
<item>
<key>
<string>
target
</string>
</key>
<value>
<string>
Click to edit the target
</string>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/bt/template_action_path_list
View file @
43f6f999
...
@@ -42,6 +42,7 @@ Data Release Module | view
...
@@ -42,6 +42,7 @@ Data Release Module | view
Data Release | predicate
Data Release | predicate
Data Release | view
Data Release | view
Data Set Module | view
Data Set Module | view
Data Set | data_streams
Data Set | predicate
Data Set | predicate
Data Set | view
Data Set | view
Data Stream Module | view
Data Stream Module | view
...
...
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_checkMd5DataStreamList.py
0 → 100644
View file @
43f6f999
"""
Script to check that a data set is properly uploaded
to Wendelin Data Lake.
How to use it: create a file_system_checksum file containing md5sum
values of all dataset files uploaded with the following format:
Format of is the same as md5sum's output:
<md5_sum> <filename.extension>
It can be generated in the original data set folder outside wendelin by doing md5sum * > output.txt
"""
import
os.path
data
=
str
(
context
.
file_system_checksum
).
strip
()
lines
=
data
.
split
(
"
\
n
"
)
print
"Total files = "
,
len
(
lines
)
print
check_result
=
True
for
line
in
lines
[:]:
md5_checksum
=
line
[:
32
].
strip
()
full_filename
=
line
[
32
:].
strip
()
filename
,
extension
=
os
.
path
.
splitext
(
full_filename
)
extension
=
extension
[
1
:]
reference
=
"%s/%s/%s"
%
(
data_set_reference
,
filename
,
extension
)
catalog_kw
=
{
"portal_type"
:
"Data Stream"
,
"reference"
:
reference
}
data_stream
=
context
.
portal_catalog
.
getResultValue
(
**
catalog_kw
)
if
data_stream
is
None
:
print
"[NOT FOUND]"
,
reference
check_result
=
False
else
:
is_upload_ok
=
(
data_stream
.
getVersion
()
==
md5_checksum
)
print
md5_checksum
,
filename
,
data_stream
is
not
None
,
is_upload_ok
if
not
is_upload_ok
:
check_result
=
False
print
if
check_result
:
print
"[OK] Data set correctly uploaded"
else
:
print
"[ERROR] Data set was not correctly uploaded"
return
printed
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_checkMd5DataStreamList.xml
0 → 100644
View file @
43f6f999
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"PythonScript"
module=
"Products.PythonScripts.PythonScript"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
Script_magic
</string>
</key>
<value>
<int>
3
</int>
</value>
</item>
<item>
<key>
<string>
_bind_names
</string>
</key>
<value>
<object>
<klass>
<global
name=
"NameAssignments"
module=
"Shared.DC.Scripts.Bindings"
/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key>
<string>
_asgns
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
name_container
</string>
</key>
<value>
<string>
container
</string>
</value>
</item>
<item>
<key>
<string>
name_context
</string>
</key>
<value>
<string>
context
</string>
</value>
</item>
<item>
<key>
<string>
name_m_self
</string>
</key>
<value>
<string>
script
</string>
</value>
</item>
<item>
<key>
<string>
name_subpath
</string>
</key>
<value>
<string>
traverse_subpath
</string>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key>
<string>
_params
</string>
</key>
<value>
<string>
data_set_reference=None
</string>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
DataSet_checkMd5DataStreamList
</string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_getDataStreamList.py
deleted
100644 → 0
View file @
f814135b
"""
Return list of Data Streams belonging to a Date Set.
Data Ingestion line aggregates both Data Set and Data Stream.
Note: This code is quite computationally costly (for Data Streams having thousands of iles) as it needs to:
1. Query MariaDB to find ingestion lines
2. Read from ZODB both Data Ingestion Lines and Data Streams (which itself can be big too)
"""
data_ingestion_line_list
=
context
.
portal_catalog
(
portal_type
=
"Data Ingestion Line"
,
aggregate_uid
=
context
.
getUid
())
return
[
x
.
getAggregateValue
(
portal_type
=
"Data Stream"
)
\
for
x
in
data_ingestion_line_list
]
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamCount.py
0 → 100644
View file @
43f6f999
"""
This script is called from ebulk client to get count of Data Streams for a Data set.
"""
from
erp5.component.module.Log
import
log
portal
=
context
.
getPortalObject
()
try
:
data_set
=
portal
.
data_set_module
.
get
(
data_set_reference
)
if
data_set
is
None
or
data_set
.
getReference
().
endswith
(
"_invalid"
):
return
{
"status_code"
:
0
,
"result"
:
0
}
except
Exception
as
e
:
log
(
"Unauthorized access to getDataStreamList: "
+
str
(
e
))
return
{
"status_code"
:
1
,
"error_message"
:
"401 - Unauthorized access. Please check your user credentials and try again."
}
data_set_uid
=
data_set
.
getUid
()
data_stream_list
=
context
.
DataSet_getDataStreamList
(
data_set_uid
)
return
{
"status_code"
:
0
,
"result"
:
len
(
data_stream_list
)
}
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamCount.xml
0 → 100644
View file @
43f6f999
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"PythonScript"
module=
"Products.PythonScripts.PythonScript"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
Script_magic
</string>
</key>
<value>
<int>
3
</int>
</value>
</item>
<item>
<key>
<string>
_bind_names
</string>
</key>
<value>
<object>
<klass>
<global
name=
"NameAssignments"
module=
"Shared.DC.Scripts.Bindings"
/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key>
<string>
_asgns
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
name_container
</string>
</key>
<value>
<string>
container
</string>
</value>
</item>
<item>
<key>
<string>
name_context
</string>
</key>
<value>
<string>
context
</string>
</value>
</item>
<item>
<key>
<string>
name_m_self
</string>
</key>
<value>
<string>
script
</string>
</value>
</item>
<item>
<key>
<string>
name_subpath
</string>
</key>
<value>
<string>
traverse_subpath
</string>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key>
<string>
_params
</string>
</key>
<value>
<string>
data_set_reference
</string>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
ERP5Site_getDataStreamCount
</string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamList.py
View file @
43f6f999
"""
"""
This script is called from ebulk client to get list of Data Streams for a Data set.
This script is called from ebulk client to get list of Data Streams for a Data set.
"""
"""
import
json
import
json
from
erp5.component.module.Log
import
log
from
erp5.component.module.Log
import
log
limit
=
[]
if
batch_size
:
limit
=
[
offset
,
batch_size
]
portal
=
context
.
getPortalObject
()
portal
=
context
.
getPortalObject
()
try
:
try
:
data_set
=
portal
.
data_set_module
.
get
(
data_set_reference
)
data_set
=
portal
.
data_set_module
.
get
(
data_set_reference
)
if
data_set
is
None
or
portal
.
ERP5Site_checkReferenceInvalidated
(
data_set
):
if
data_set
is
None
or
data_set
.
getReference
().
endswith
(
"_invalid"
):
return
{
"status_code"
:
0
,
"result"
:
[]
}
return
json
.
dumps
({
"status_code"
:
0
,
"result"
:
[]
})
except
Exception
as
e
:
# fails because unauthorized access
except
Exception
as
e
:
# fails because unauthorized access
log
(
"Unauthorized access to getDataStreamList: "
+
str
(
e
))
log
(
"Unauthorized access to getDataStreamList: "
+
str
(
e
))
return
{
"status_code"
:
1
,
"error_message"
:
"401 - Unauthorized access. Please check your user credentials and try again."
}
return
json
.
dumps
({
"status_code"
:
1
,
"error_message"
:
"401 - Unauthorized access. Please check your user credentials and try again."
})
data_set_uid
=
data_set
.
getUid
()
data_stream_list
=
context
.
DataSet_getDataStreamList
(
data_set_uid
,
limit
)
data_stream_dict
=
{}
data_stream_dict
=
{}
for
stream
in
data_set
.
DataSet_getDataStreamList
():
for
stream_brain
in
data_stream_list
:
if
stream
and
not
portal
.
ERP5Site_checkReferenceInvalidated
(
stream
)
and
stream
.
getValidationState
()
!=
"draft"
:
reference
=
stream_brain
.
reference
data_stream_info_dict
=
{
'id'
:
'data_stream_module/'
+
stream
.
getId
(),
version
=
stream_brain
.
version
'size'
:
stream
.
getSize
(),
size
=
stream_brain
.
size
'hash'
:
stream
.
getVersion
()
}
data_stream_id
=
stream_brain
.
relative_url
if
stream
.
getReference
()
in
data_stream_dict
:
data_stream_info_dict
=
{
'id'
:
data_stream_id
,
data_stream_dict
[
stream
.
getReference
()][
'data-stream-list'
].
append
(
data_stream_info_dict
)
'size'
:
size
,
data_stream_dict
[
stream
.
getReference
()][
'large-hash'
]
=
data_stream_dict
[
stream
.
getReference
()][
'large-hash'
]
+
str
(
stream
.
getVersion
())
'hash'
:
version
}
data_stream_dict
[
stream
.
getReference
()][
'full-size'
]
=
int
(
data_stream_dict
[
stream
.
getReference
()][
'full-size'
])
+
int
(
stream
.
getSize
())
if
reference
in
data_stream_dict
:
else
:
data_stream_dict
[
reference
][
'data-stream-list'
].
append
(
data_stream_info_dict
)
data_stream_dict
[
stream
.
getReference
()]
=
{
'data-stream-list'
:
[
data_stream_info_dict
],
data_stream_dict
[
reference
][
'large-hash'
]
=
data_stream_dict
[
reference
][
'large-hash'
]
+
str
(
version
)
'id'
:
'data_stream_module/'
+
stream
.
getId
(),
data_stream_dict
[
reference
][
'full-size'
]
=
int
(
data_stream_dict
[
reference
][
'full-size'
])
+
int
(
size
)
'reference'
:
stream
.
getReference
(),
else
:
'large-hash'
:
stream
.
getVersion
(),
data_stream_dict
[
reference
]
=
{
'data-stream-list'
:
[
data_stream_info_dict
],
'full-size'
:
stream
.
getSize
()
}
'id'
:
data_stream_id
,
'reference'
:
reference
,
'large-hash'
:
version
,
'full-size'
:
size
}
result_dict
=
{
'status_code'
:
0
,
'result'
:
data_stream_dict
.
values
()}
result_dict
=
{
'status_code'
:
0
,
'result'
:
data_stream_dict
.
values
()}
return
json
.
dumps
(
result_dict
)
return
json
.
dumps
(
result_dict
)
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamList.xml
View file @
43f6f999
...
@@ -50,7 +50,7 @@
...
@@ -50,7 +50,7 @@
</item>
</item>
<item>
<item>
<key>
<string>
_params
</string>
</key>
<key>
<string>
_params
</string>
</key>
<value>
<string>
data_set_reference
</string>
</value>
<value>
<string>
data_set_reference
, offset=0, batch_size=0, get_count=False
</string>
</value>
</item>
</item>
<item>
<item>
<key>
<string>
id
</string>
</key>
<key>
<string>
id
</string>
</key>
...
...
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
View file @
43f6f999
...
@@ -334,4 +334,31 @@ class TestDataIngestion(SecurityTestCase):
...
@@ -334,4 +334,31 @@ class TestDataIngestion(SecurityTestCase):
self
.
portal
.
portal_alarms
.
wendelin_handle_analysis
.
Alarm_handleAnalysis
()
self
.
portal
.
portal_alarms
.
wendelin_handle_analysis
.
Alarm_handleAnalysis
()
self
.
tic
()
self
.
tic
()
def
test_10_checkDataSetDataStreamRelation
(
self
):
"""
Data Set and its Data Streams are related through the corresponding Data Ingestion Lines
"""
# ingest a couple of files
reference
=
self
.
getRandomReference
()
self
.
ingest
(
"some-data-1"
,
reference
,
self
.
CSV
,
self
.
SINGLE_INGESTION_END
)
time
.
sleep
(
1
)
self
.
tic
()
reference
+=
"-2"
self
.
ingest
(
"some-data-2"
,
reference
,
self
.
CSV
,
self
.
SINGLE_INGESTION_END
)
time
.
sleep
(
1
)
self
.
tic
()
# get corresponding Data Streams by searching via Data Ingestion Lines of the Data Set
data_set
=
self
.
portal
.
data_set_module
.
get
(
self
.
REF_DATASET
)
data_ingestion_line_list
=
self
.
portal
.
portal_catalog
(
portal_type
=
'Data Ingestion Line'
,
aggregate_uid
=
data_set
.
getUid
())
data_ingestion_uid_list
=
[
x
.
getUid
()
for
x
in
data_ingestion_line_list
]
data_stream_list
=
self
.
portal
.
portal_catalog
(
portal_type
=
'Data Stream'
,
aggregate__related__uid
=
data_ingestion_uid_list
,
select_list
=
[
'reference'
,
'relative_url'
,
'versioning.size'
,
'versioning.version'
])
data_stream_list
=
[
x
.
getObject
()
for
x
in
data_stream_list
]
# assert that the list from the search is the same as DataSet_getDataStreamList
self
.
assertSameSet
(
data_stream_list
,
data_set
.
DataSet_getDataStreamList
())
# XXX: new test which simulates download / upload of Data Set and increase DS version
# XXX: new test which simulates download / upload of Data Set and increase DS version
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment