Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Léo-Paul Géneau
wendelin
Commits
562b39c3
Commit
562b39c3
authored
Apr 30, 2020
by
Ivan Tyagov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Clean up.
parent
57c784cb
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
0 additions
and
200 deletions
+0
-200
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/Alarm_dataLakeHandleInvalids.py
...s/erp5_wendelin_data_lake/Alarm_dataLakeHandleInvalids.py
+0
-42
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/Alarm_dataLakeHandleInvalids.xml
.../erp5_wendelin_data_lake/Alarm_dataLakeHandleInvalids.xml
+0
-62
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_renameIngestion.py
...skins/erp5_wendelin_data_lake/ERP5Site_renameIngestion.py
+0
-34
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_renameIngestion.xml
...kins/erp5_wendelin_data_lake/ERP5Site_renameIngestion.xml
+0
-62
No files found.
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/Alarm_dataLakeHandleInvalids.py
deleted
100644 → 0
View file @
57c784cb
from
Products.ZSQLCatalog.SQLCatalog
import
Query
,
SimpleQuery
return
# This alarm was deprecated - kept for test
portal
=
context
.
getPortalObject
()
portal_catalog
=
portal
.
portal_catalog
# invalidate old (more than 5 hours) pending ingestions (e.g. split ingestions that were canceled/interrumped and no resumed)
from
DateTime
import
DateTime
now
=
DateTime
()
now_minus_max
=
now
-
1.0
/
24
/
60
*
9999
now_minus_5
=
now
-
1.0
/
24
/
60
*
60
*
5
catalog_kw
=
{
'creation_date'
:
{
'query'
:
(
now_minus_max
,
now_minus_5
),
'range'
:
'minmax'
},
'simulation_state'
:
'started'
,
'portal_type'
:
'Data Ingestion'
}
for
data_ingestion
in
portal_catalog
(
**
catalog_kw
):
# search related data ingestions that are not old yet (less than 5 hours)
catalog_kw
=
{
'creation_date'
:
{
'query'
:
(
now_minus_5
,
DateTime
()),
'range'
:
'minmax'
},
'simulation_state'
:
'started'
,
'portal_type'
:
'Data Ingestion'
,
'reference'
:
data_ingestion
.
getReference
()}
invalidate
=
True
if
len
(
portal_catalog
(
**
catalog_kw
))
>
0
:
invalidate
=
False
if
invalidate
:
# invalidate related Data Stream
kw_dict
=
{
"portal_type"
:
"Data Stream"
,
"id"
:
data_ingestion
.
getId
()}
for
data_stream
in
portal_catalog
(
**
kw_dict
):
if
not
data_stream
.
getReference
().
endswith
(
"_invalid"
):
data_stream
.
setReference
(
data_stream
.
getReference
()
+
"_invalid"
)
try
:
data_stream
.
invalidate
()
except
:
context
.
logEntry
(
"[WARNING] Could not invalidate data stream '%s', it was already invalidated or draft"
%
data_stream
.
getId
())
try
:
if
not
data_ingestion
.
getReference
().
endswith
(
"_invalid"
):
data_ingestion
.
setReference
(
data_ingestion
.
getReference
()
+
"_invalid"
)
data_ingestion
.
deliver
()
except
:
context
.
logEntry
(
"[WARNING] Could not invalidate/deliver data ingestion '%s', it was already invalidated/deliver"
%
data_ingestion
.
getId
())
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/Alarm_dataLakeHandleInvalids.xml
deleted
100644 → 0
View file @
57c784cb
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"PythonScript"
module=
"Products.PythonScripts.PythonScript"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
Script_magic
</string>
</key>
<value>
<int>
3
</int>
</value>
</item>
<item>
<key>
<string>
_bind_names
</string>
</key>
<value>
<object>
<klass>
<global
name=
"NameAssignments"
module=
"Shared.DC.Scripts.Bindings"
/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key>
<string>
_asgns
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
name_container
</string>
</key>
<value>
<string>
container
</string>
</value>
</item>
<item>
<key>
<string>
name_context
</string>
</key>
<value>
<string>
context
</string>
</value>
</item>
<item>
<key>
<string>
name_m_self
</string>
</key>
<value>
<string>
script
</string>
</value>
</item>
<item>
<key>
<string>
name_subpath
</string>
</key>
<value>
<string>
traverse_subpath
</string>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key>
<string>
_params
</string>
</key>
<value>
<string></string>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
Alarm_dataLakeHandleInvalids
</string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_renameIngestion.py
deleted
100644 → 0
View file @
57c784cb
portal
=
context
.
getPortalObject
()
portal_catalog
=
portal
.
portal_catalog
reference_separator
=
portal
.
getIngestionReferenceDictionary
()[
"reference_separator"
]
none_extension
=
portal
.
getIngestionReferenceDictionary
()[
"none_extension"
]
# check new reference
data_ingestions
=
portal_catalog
(
portal_type
=
"Data Ingestion"
,
reference
=
new_reference
)
if
len
(
data_ingestions
)
>
0
:
raise
"Error renaming: new reference '%s' already exists."
%
new_reference
# rename data ingestions
data_ingestions
=
portal_catalog
(
portal_type
=
"Data Ingestion"
,
reference
=
reference
)
if
len
(
data_ingestions
)
==
0
:
raise
"Error renaming: could not find any data ingestion with reference '%s'."
%
reference
data_ingestion_title
=
reference_separator
.
join
(
new_reference
.
split
(
reference_separator
)[
1
:
-
1
])
for
data_ingestion
in
data_ingestions
:
data_ingestion
.
setReference
(
new_reference
)
data_ingestion
.
setTitle
(
data_ingestion_title
)
extension
=
new_reference
.
split
(
reference_separator
)[
-
1
]
data_stream_title
=
"%s%s"
%
(
data_ingestion_title
,
"."
+
extension
if
extension
!=
none_extension
else
""
)
# rename data streams
data_streams
=
portal_catalog
(
portal_type
=
"Data Stream"
,
reference
=
reference
)
for
data_stream
in
data_streams
:
data_stream
.
setReference
(
new_reference
)
data_stream
.
setTitle
(
data_stream_title
)
# rename data analysis
data_analysises
=
portal_catalog
(
portal_type
=
"Data Analysis"
,
reference
=
reference
)
for
data_analysis
in
data_analysises
:
data_analysis
.
setReference
(
new_reference
)
# rename data arrays
data_arrays
=
portal_catalog
(
portal_type
=
"Data Array"
,
reference
=
reference
)
for
data_array
in
data_arrays
:
data_array
.
setReference
(
new_reference
)
bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_renameIngestion.xml
deleted
100644 → 0
View file @
57c784cb
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"PythonScript"
module=
"Products.PythonScripts.PythonScript"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
Script_magic
</string>
</key>
<value>
<int>
3
</int>
</value>
</item>
<item>
<key>
<string>
_bind_names
</string>
</key>
<value>
<object>
<klass>
<global
name=
"NameAssignments"
module=
"Shared.DC.Scripts.Bindings"
/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key>
<string>
_asgns
</string>
</key>
<value>
<dictionary>
<item>
<key>
<string>
name_container
</string>
</key>
<value>
<string>
container
</string>
</value>
</item>
<item>
<key>
<string>
name_context
</string>
</key>
<value>
<string>
context
</string>
</value>
</item>
<item>
<key>
<string>
name_m_self
</string>
</key>
<value>
<string>
script
</string>
</value>
</item>
<item>
<key>
<string>
name_subpath
</string>
</key>
<value>
<string>
traverse_subpath
</string>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key>
<string>
_params
</string>
</key>
<value>
<string>
reference, new_reference
</string>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
ERP5Site_renameIngestion
</string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment