Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
3
Merge Requests
3
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
wendelin
Commits
ab6250a1
Commit
ab6250a1
authored
Apr 22, 2020
by
Ivan Tyagov
1
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Clean up.
parent
47712107
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
6 additions
and
14 deletions
+6
-14
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
...Item/portal_components/test.erp5.testDataLakeIngestion.py
+4
-11
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.xml
...tem/portal_components/test.erp5.testDataLakeIngestion.xml
+2
-3
No files found.
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
View file @
ab6250a1
...
@@ -3,7 +3,6 @@ import string
...
@@ -3,7 +3,6 @@ import string
import
random
import
random
import
csv
import
csv
import
os
import
os
import
time
import
numpy
as
np
import
numpy
as
np
import
base64
import
base64
...
@@ -31,7 +30,6 @@ class TestDataIngestion(SecurityTestCase):
...
@@ -31,7 +30,6 @@ class TestDataIngestion(SecurityTestCase):
self
.
assertEqual
(
self
.
INVALID
,
self
.
portal
.
getIngestionReferenceDictionary
()[
"invalid_suffix"
])
self
.
assertEqual
(
self
.
INVALID
,
self
.
portal
.
getIngestionReferenceDictionary
()[
"invalid_suffix"
])
self
.
assertEqual
(
self
.
EOF
,
self
.
REFERENCE_SEPARATOR
+
self
.
portal
.
getIngestionReferenceDictionary
()[
"split_end_suffix"
])
self
.
assertEqual
(
self
.
EOF
,
self
.
REFERENCE_SEPARATOR
+
self
.
portal
.
getIngestionReferenceDictionary
()[
"split_end_suffix"
])
self
.
assertEqual
(
self
.
PART_1
,
self
.
REFERENCE_SEPARATOR
+
self
.
portal
.
getIngestionReferenceDictionary
()[
"split_first_suffix"
])
self
.
assertEqual
(
self
.
PART_1
,
self
.
REFERENCE_SEPARATOR
+
self
.
portal
.
getIngestionReferenceDictionary
()[
"split_first_suffix"
])
# XXX: create default users
def
getRandomReference
(
self
):
def
getRandomReference
(
self
):
random_string
=
''
.
join
([
random
.
choice
(
string
.
ascii_letters
+
string
.
digits
)
for
_
in
xrange
(
10
)])
random_string
=
''
.
join
([
random
.
choice
(
string
.
ascii_letters
+
string
.
digits
)
for
_
in
xrange
(
10
)])
...
@@ -76,7 +74,6 @@ class TestDataIngestion(SecurityTestCase):
...
@@ -76,7 +74,6 @@ class TestDataIngestion(SecurityTestCase):
request
.
set
(
'data_chunk'
,
encoded_data_chunk
)
request
.
set
(
'data_chunk'
,
encoded_data_chunk
)
ingestion_policy
.
ingest
()
ingestion_policy
.
ingest
()
self
.
tic
()
self
.
tic
()
return
def
ingest
(
self
,
data_chunk
,
reference
,
extension
,
eof
):
def
ingest
(
self
,
data_chunk
,
reference
,
extension
,
eof
):
ingestion_reference
=
self
.
getIngestionReference
(
reference
,
extension
)
ingestion_reference
=
self
.
getIngestionReference
(
reference
,
extension
)
...
@@ -85,7 +82,7 @@ class TestDataIngestion(SecurityTestCase):
...
@@ -85,7 +82,7 @@ class TestDataIngestion(SecurityTestCase):
ingestion_policy
=
self
.
portal
.
portal_ingestion_policies
.
wendelin_embulk
ingestion_policy
=
self
.
portal
.
portal_ingestion_policies
.
wendelin_embulk
self
.
ingestRequest
(
ingestion_reference
,
eof
,
data_chunk
,
ingestion_policy
)
self
.
ingestRequest
(
ingestion_reference
,
eof
,
data_chunk
,
ingestion_policy
)
ingestion_id
,
ingestion_reference
=
self
.
sanitizeReference
(
ingestion_reference
)
_
,
ingestion_reference
=
self
.
sanitizeReference
(
ingestion_reference
)
return
ingestion_reference
return
ingestion_reference
...
@@ -151,19 +148,15 @@ class TestDataIngestion(SecurityTestCase):
...
@@ -151,19 +148,15 @@ class TestDataIngestion(SecurityTestCase):
reference
=
self
.
getRandomReference
()
reference
=
self
.
getRandomReference
()
ingestion_reference
=
self
.
ingest
(
data_chunk_1
,
reference
,
self
.
FIF
,
self
.
PART_1
)
ingestion_reference
=
self
.
ingest
(
data_chunk_1
,
reference
,
self
.
FIF
,
self
.
PART_1
)
time
.
sleep
(
1
)
self
.
tic
()
self
.
tic
()
ingestion_reference
=
self
.
ingest
(
data_chunk_2
,
reference
,
self
.
FIF
,
self
.
PART_2
)
ingestion_reference
=
self
.
ingest
(
data_chunk_2
,
reference
,
self
.
FIF
,
self
.
PART_2
)
time
.
sleep
(
1
)
self
.
tic
()
self
.
tic
()
ingestion_reference
=
self
.
ingest
(
data_chunk_3
,
reference
,
self
.
FIF
,
self
.
PART_3
)
ingestion_reference
=
self
.
ingest
(
data_chunk_3
,
reference
,
self
.
FIF
,
self
.
PART_3
)
time
.
sleep
(
1
)
self
.
tic
()
self
.
tic
()
ingestion_reference
=
self
.
ingest
(
data_chunk_4
,
reference
,
self
.
FIF
,
self
.
EOF
)
ingestion_reference
=
self
.
ingest
(
data_chunk_4
,
reference
,
self
.
FIF
,
self
.
EOF
)
time
.
sleep
(
1
)
self
.
tic
()
self
.
tic
()
# call explicitly alarm so all 4 Data Streams can be concatenated to one
# call explicitly alarm so all 4 Data Streams can be concatenated to one
...
...
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.xml
View file @
ab6250a1
...
@@ -46,9 +46,8 @@
...
@@ -46,9 +46,8 @@
<key>
<string>
text_content_warning_message
</string>
</key>
<key>
<string>
text_content_warning_message
</string>
</key>
<value>
<value>
<tuple>
<tuple>
<string>
W: 88, 4: Unused variable \'ingestion_id\' (unused-variable)
</string>
<string>
W: 92, 34: Unused variable \'i\' (unused-variable)
</string>
<string>
W: 95, 34: Unused variable \'i\' (unused-variable)
</string>
<string>
W: 92, 76: Unused variable \'j\' (unused-variable)
</string>
<string>
W: 95, 76: Unused variable \'j\' (unused-variable)
</string>
</tuple>
</tuple>
</value>
</value>
</item>
</item>
...
...
Ivan Tyagov
@Tyagov
mentioned in commit
9dc83906
·
Apr 23, 2020
mentioned in commit
9dc83906
mentioned in commit 9dc83906666d0c4c09746f34304cfe800999cf89
Toggle commit list
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment