Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Léo-Paul Géneau
wendelin
Commits
7cf9b582
Commit
7cf9b582
authored
Apr 13, 2020
by
Ivan Tyagov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Clean up.
parent
39dde4fa
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
25 additions
and
39 deletions
+25
-39
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
...Item/portal_components/test.erp5.testDataLakeIngestion.py
+2
-19
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.xml
...tem/portal_components/test.erp5.testDataLakeIngestion.xml
+23
-20
No files found.
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
View file @
7cf9b582
...
...
@@ -6,7 +6,6 @@ import os
import
time
import
numpy
as
np
import
base64
from
datetime
import
datetime
class
TestDataIngestion
(
SecurityTestCase
):
...
...
@@ -16,23 +15,13 @@ class TestDataIngestion(SecurityTestCase):
PART_3
=
REFERENCE_SEPARATOR
+
"003"
EOF
=
REFERENCE_SEPARATOR
+
"EOF"
FIF
=
REFERENCE_SEPARATOR
+
"fif"
TXT
=
REFERENCE_SEPARATOR
+
"txt"
CSV
=
REFERENCE_SEPARATOR
+
"csv"
TSV
=
REFERENCE_SEPARATOR
+
"tsv"
GZ
=
REFERENCE_SEPARATOR
+
"gz"
NII
=
REFERENCE_SEPARATOR
+
"nii"
SIZE_HASH
=
REFERENCE_SEPARATOR
+
"fake-size"
+
REFERENCE_SEPARATOR
+
"fake-hash"
SINGLE_INGESTION_END
=
REFERENCE_SEPARATOR
RANDOM
=
REFERENCE_SEPARATOR
+
''
.
join
([
random
.
choice
(
string
.
ascii_letters
+
string
.
digits
)
for
_
in
xrange
(
3
)])
CHUNK_SIZE_TXT
=
50000
CHUNK_SIZE_CSV
=
25
REF_PREFIX
=
"fake-supplier"
+
REFERENCE_SEPARATOR
+
"fake-dataset"
+
REFERENCE_SEPARATOR
REF_SUPPLIER_PREFIX
=
"fake-supplier"
+
REFERENCE_SEPARATOR
INGESTION_SCRIPT
=
'HandleFifEmbulkIngestion'
INVALID
=
"_invalid"
NEW
=
"_NEW"
FALSE
=
"FALSE"
TRUE
=
"TRUE"
def
getTitle
(
self
):
return
"DataIngestionTest"
...
...
@@ -42,7 +31,7 @@ class TestDataIngestion(SecurityTestCase):
'erp5_wendelin'
,
'erp5_callables'
,
'erp5_core'
)
def
afterSetUp
(
self
):
self
.
context
=
self
.
portal
#.UnitTest_getContext()
self
.
context
=
self
.
portal
self
.
assertEqual
(
self
.
REFERENCE_SEPARATOR
,
self
.
portal
.
getIngestionReferenceDictionary
()[
"reference_separator"
])
self
.
assertEqual
(
self
.
INVALID
,
self
.
portal
.
getIngestionReferenceDictionary
()[
"invalid_suffix"
])
self
.
assertEqual
(
self
.
EOF
,
self
.
REFERENCE_SEPARATOR
+
self
.
portal
.
getIngestionReferenceDictionary
()[
"split_end_suffix"
])
...
...
@@ -69,12 +58,6 @@ class TestDataIngestion(SecurityTestCase):
for
i
in
xrange
(
0
,
len
(
l
),
n
):
yield
l
[
i
:
i
+
n
]
def
generateRawDataBytesAndArray
(
self
):
url
=
'data_stream_module/mne_sample_for_test'
sample_data_stream
=
self
.
context
.
restrictedTraverse
(
url
)
raw_data
,
array
,
json_data
=
self
.
context
.
generateRawData
(
sample_data_stream
)
return
raw_data
,
array
,
json_data
def
getIngestionPolicy
(
self
,
reference
,
ingestion_script
):
ingestion_policy
=
self
.
portal
.
portal_catalog
.
getResultValue
(
portal_type
=
'Ingestion Policy'
,
...
...
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.xml
View file @
7cf9b582
...
...
@@ -46,10 +46,9 @@
<key>
<string>
text_content_warning_message
</string>
</key>
<value>
<tuple>
<string>
W:125, 4: Unused variable \'ingestion_id\' (unused-variable)
</string>
<string>
W:132, 34: Unused variable \'i\' (unused-variable)
</string>
<string>
W:132, 76: Unused variable \'j\' (unused-variable)
</string>
<string>
W: 9, 0: Unused datetime imported from datetime (unused-import)
</string>
<string>
W:108, 4: Unused variable \'ingestion_id\' (unused-variable)
</string>
<string>
W:115, 34: Unused variable \'i\' (unused-variable)
</string>
<string>
W:115, 76: Unused variable \'j\' (unused-variable)
</string>
</tuple>
</value>
</item>
...
...
@@ -105,11 +104,13 @@
</record>
<record
id=
"4"
aka=
"AAAAAAAAAAQ="
>
<pickle>
<global
name=
"WorkflowHistoryList"
module=
"Products.ERP5Type.
patches.WorkflowTool
"
/>
<global
name=
"WorkflowHistoryList"
module=
"Products.ERP5Type.
Workflow
"
/>
</pickle>
<pickle>
<tuple>
<none/>
<dictionary>
<item>
<key>
<string>
_log
</string>
</key>
<value>
<list>
<dictionary>
<item>
...
...
@@ -122,7 +123,9 @@
</item>
</dictionary>
</list>
</tuple>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment