Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Léo-Paul Géneau
wendelin
Commits
b0be31a9
Commit
b0be31a9
authored
Mar 08, 2021
by
Roque
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
erp5_wendelin_data_lake_ingestion: drop obsolete script and use new name
parent
7357b3db
Changes
5
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
4 additions
and
79 deletions
+4
-79
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_getDataStreamList.py
...kins/erp5_wendelin_data_lake/DataSet_getDataStreamList.py
+0
-13
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_getDataStreamList.xml
...ins/erp5_wendelin_data_lake/DataSet_getDataStreamList.xml
+0
-62
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamCount.py
...ns/erp5_wendelin_data_lake/ERP5Site_getDataStreamCount.py
+1
-1
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamList.py
...ins/erp5_wendelin_data_lake/ERP5Site_getDataStreamList.py
+1
-1
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
...Item/portal_components/test.erp5.testDataLakeIngestion.py
+2
-2
No files found.
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_getDataStreamList.py
deleted
100644 → 0
View file @
7357b3db
"""
Return list of Data Streams belonging to a Date Set.
Data Ingestion line aggregates both Data Set and Data Stream.
Note: This code is quite computationally costly (for Data Streams having thousands of iles) as it needs to:
1. Query MariaDB to find ingestion lines
2. Read from ZODB both Data Ingestion Lines and Data Streams (which itself can be big too)
"""
data_ingestion_line_list
=
context
.
portal_catalog
(
portal_type
=
"Data Ingestion Line"
,
aggregate_uid
=
context
.
getUid
())
return
[
x
.
getAggregateValue
(
portal_type
=
"Data Stream"
)
\
for
x
in
data_ingestion_line_list
]
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataSet_getDataStreamList.xml
deleted
100644 → 0
View file @
7357b3db
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"PythonScript"
module=
"Products.PythonScripts.PythonScript"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
Script_magic
</string>
</key>
<value>
<int>
3
</int>
</value>
</item>
<item>
<key>
<string>
_bind_names
</string>
</key>
<value>
<object>
<klass>
<global
name=
"NameAssignments"
module=
"Shared.DC.Scripts.Bindings"
/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key>
<string>
_asgns
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
name_container
</string>
</key>
<value>
<string>
container
</string>
</value>
</item>
<item>
<key>
<string>
name_context
</string>
</key>
<value>
<string>
context
</string>
</value>
</item>
<item>
<key>
<string>
name_m_self
</string>
</key>
<value>
<string>
script
</string>
</value>
</item>
<item>
<key>
<string>
name_subpath
</string>
</key>
<value>
<string>
traverse_subpath
</string>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key>
<string>
_params
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
DataSet_getDataStreamList
</string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamCount.py
View file @
b0be31a9
...
@@ -14,6 +14,6 @@ except Exception as e:
...
@@ -14,6 +14,6 @@ except Exception as e:
return
{
"status_code"
:
1
,
"error_message"
:
"401 - Unauthorized access. Please check your user credentials and try again."
}
return
{
"status_code"
:
1
,
"error_message"
:
"401 - Unauthorized access. Please check your user credentials and try again."
}
data_set_uid
=
data_set
.
getUid
()
data_set_uid
=
data_set
.
getUid
()
data_stream_list
=
context
.
ERP5Site
_getDataStreamList
(
data_set_uid
)
data_stream_list
=
context
.
DataSet
_getDataStreamList
(
data_set_uid
)
return
{
"status_code"
:
0
,
"result"
:
len
(
data_stream_list
)
}
return
{
"status_code"
:
0
,
"result"
:
len
(
data_stream_list
)
}
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamList.py
View file @
b0be31a9
...
@@ -19,7 +19,7 @@ except Exception as e: # fails because unauthorized access
...
@@ -19,7 +19,7 @@ except Exception as e: # fails because unauthorized access
return
json
.
dumps
({
"status_code"
:
1
,
"error_message"
:
"401 - Unauthorized access. Please check your user credentials and try again."
})
return
json
.
dumps
({
"status_code"
:
1
,
"error_message"
:
"401 - Unauthorized access. Please check your user credentials and try again."
})
data_set_uid
=
data_set
.
getUid
()
data_set_uid
=
data_set
.
getUid
()
data_stream_list
=
context
.
ERP5Site
_getDataStreamList
(
data_set_uid
,
limit
)
data_stream_list
=
context
.
DataSet
_getDataStreamList
(
data_set_uid
,
limit
)
data_stream_dict
=
{}
data_stream_dict
=
{}
for
stream_brain
in
data_stream_list
:
for
stream_brain
in
data_stream_list
:
...
...
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
View file @
b0be31a9
...
@@ -227,7 +227,7 @@ class TestDataIngestion(SecurityTestCase):
...
@@ -227,7 +227,7 @@ class TestDataIngestion(SecurityTestCase):
self
.
tic
()
self
.
tic
()
# check data relation between Data Set and Data Streams work
# check data relation between Data Set and Data Streams work
self
.
assertSameSet
(
data_stream_list
,
data_set
.
DataSet_getDataStreamList
())
self
.
assertSameSet
(
data_stream_list
,
data_set
.
DataSet_getDataStreamList
(
data_set
.
getUid
()
))
# check data set and all Data Streams states
# check data set and all Data Streams states
self
.
assertEqual
(
'validated'
,
data_set
.
getValidationState
())
self
.
assertEqual
(
'validated'
,
data_set
.
getValidationState
())
...
@@ -359,6 +359,6 @@ class TestDataIngestion(SecurityTestCase):
...
@@ -359,6 +359,6 @@ class TestDataIngestion(SecurityTestCase):
select_list
=
[
'reference'
,
'relative_url'
,
'versioning.size'
,
'versioning.version'
])
select_list
=
[
'reference'
,
'relative_url'
,
'versioning.size'
,
'versioning.version'
])
data_stream_list
=
[
x
.
getObject
()
for
x
in
data_stream_list
]
data_stream_list
=
[
x
.
getObject
()
for
x
in
data_stream_list
]
# assert that the list from the search is the same as DataSet_getDataStreamList
# assert that the list from the search is the same as DataSet_getDataStreamList
self
.
assertSameSet
(
data_stream_list
,
data_set
.
DataSet_getDataStreamList
())
self
.
assertSameSet
(
data_stream_list
,
data_set
.
DataSet_getDataStreamList
(
data_set
.
getUid
()
))
# XXX: new test which simulates download / upload of Data Set and increase DS version
# XXX: new test which simulates download / upload of Data Set and increase DS version
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment