Commit c2f51576 authored by Ivan Tyagov's avatar Ivan Tyagov Committed by Nikola Balog

No real need of this bt5 anymore here for generic Wendelin.

parent a616877d
# -*- coding: utf-8 -*-
"""
NEO consistency checks code.
"""
from ZODB import DB
from neo.client.Storage import Storage
import hashlib
def DataBucketStream_getChecksumListFromNEONodeListForKey(self, \
node_list, \
ca_file, \
cert_file, \
key_file, \
key, \
threshold):
"""
Directly connect to NEO backends and check checksums of this Data Bucket Stream for this key.
"""
checksum_list = []
# get directly checksum as we have access to data stream over self
data = self.getBucketByKey(key)
data = data[:threshold]
checksum = hashlib.sha256(data).hexdigest()
checksum_list.append(checksum)
for node in node_list:
kw = {'master_nodes': node[0],
'name': node[1],
'ca': ca_file,
'cert': cert_file,
'key': key_file}
# make a direct connection
stor = Storage(**kw)
db = DB(stor)
conn = db.open()
root = conn.root()
data_stream_id = self.getId()
data_stream = root['Application'].erp5.data_stream_module[data_stream_id]
data = data_stream.getBucketByKey(key)
data = data[:threshold]
conn.close()
db.close()
checksum = hashlib.sha256(data).hexdigest()
checksum_list.append(checksum)
return checksum_list
def DataStream_getChecksumListFromNEONodeListForStartStopOffset(self, \
node_list, \
ca_file, \
cert_file, \
key_file, \
start_offset, \
end_offset):
"""
Directly connect to NEO backends and check checksums of this Data Stream.
"""
checksum_list = []
# get directly checksum as we have access to data stream over self
chunk_list = self.readChunkList(start_offset, end_offset)
data = '\n'.join(chunk_list)
checksum = hashlib.sha256(data).hexdigest()
checksum_list.append(checksum)
for node in node_list:
kw = {'master_nodes': node[0],
'name': node[1],
'ca': ca_file,
'cert': cert_file,
'key': key_file}
# make a direct connection
stor = Storage(**kw)
db = DB(stor)
conn = db.open()
root = conn.root()
data_stream_id = self.getId()
data_stream = root['Application'].erp5.data_stream_module[data_stream_id]
chunk_list = data_stream.readChunkList(start_offset, end_offset)
data = '\n'.join(chunk_list)
conn.close()
db.close()
checksum = hashlib.sha256(data).hexdigest()
checksum_list.append(checksum)
return checksum_list
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>NEOAdministration</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.NEOAdministration</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Alarm" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>active_sense_method_id</string> </key>
<value> <string>ERP5Site_checkNEOCloneBackupConsistency</string> </value>
</item>
<item>
<key> <string>automatic_solve</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>enabled</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>promise_check_neo_clone_backup_consistency</string> </value>
</item>
<item>
<key> <string>periodicity_hour</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_hour_frequency</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>periodicity_minute</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_minute_frequency</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>periodicity_month</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_month_day</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_start_date</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1519862400.0</float>
<string>GMT</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>periodicity_week</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Alarm</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Check NEO clone backup consistency and produce a report</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Alarm" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>active_sense_method_id</string> </key>
<value> <string>ERP5Site_isNEOCloneBackupConsistent</string> </value>
</item>
<item>
<key> <string>alarm_notification_mode</string> </key>
<value>
<tuple>
<string>problem</string>
</tuple>
</value>
</item>
<item>
<key> <string>automatic_solve</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>destination/person_module/15</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>enabled</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>promise_is_neo_clone_backup_consistent</string> </value>
</item>
<item>
<key> <string>periodicity_hour</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_hour_frequency</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>periodicity_minute</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_minute_frequency</string> </key>
<value> <int>30</int> </value>
</item>
<item>
<key> <string>periodicity_month</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_month_day</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_start_date</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1519862400.0</float>
<string>GMT</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>periodicity_week</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Alarm</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Check if NEO clone backup is consistent</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Folder" module="OFS.Folder"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>erp5_neo_administration</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>DataBucketStream_getChecksumListFromNEONodeListForKey</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>NEOAdministration</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataBucketStream_getChecksumListFromNEONodeListForKey</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
"""
Check replication of Data Stream's or Data Bucket Stream's content amongst NEO clones.
"""
import random
portal_type = context.getPortalType()
if portal_type == 'Data Stream':
# sometimes we have HUGE streams so we need to check some randomly generated
# portion of a Data Stream
max_stop_offset = context.getSize()
start_offset = int(random.random() * max_stop_offset)
stop_offset = start_offset + threshold
if stop_offset > max_stop_offset:
stop_offset = max_stop_offset
checksum_list = context.DataStream_getChecksumListFromNEONodeListForStartStopOffset(
neo_node_list,
neo_cert_list[0],
neo_cert_list[1],
neo_cert_list[2],
start_offset,
stop_offset)
elif portal_type == 'Data Bucket Stream':
# choose a random bucket key as raw data source
key_list = context.getKeyList()
# choose random key
key = random.choice(key_list)
checksum_list = context.DataBucketStream_getChecksumListFromNEONodeListForKey(
neo_node_list,
neo_cert_list[0],
neo_cert_list[1],
neo_cert_list[2],
key,
threshold)
if len(set(checksum_list)) > 1:
# one of checksums didn't match
print "PROBLEM:", context.getRelativeUrl(), checksum_list
else:
print "OK:", context.getRelativeUrl(), checksum_list
return printed
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>neo_node_list, neo_cert_list, threshold=1048576</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataStream_checkIfNEOCloneBackupIsConsistent</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>DataStream_getChecksumListFromNEONodeListForStartStopOffset</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>NEOAdministration</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataStream_getChecksumListFromNEONodeListForStartStopOffset</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
"""
Check if a NEO backup works by checking a list of Data Streams against a NEO backup server.
Check consists of reading same portions of Data Stream from production and clone systems,
calculate hash and compare it.
Result of checks is gradually appended to an intermediat Active Process which is later examined for inconsistencies.
"""
threshold = context.ERP5Site_getPreferredThresholdSize()
neo_node_list, neo_cert_list = context.ERP5Site_getNEONodeListAndSSLCertificateLocation()
active_process = context.newActiveProcess()
# so we can distinguish from other Active Processes when needed!
active_process.setTitle("NEO_Clone_check")
# which Data Streams to check ...
data_stream_list = context.ERP5Site_getDataStreamListToCheck()
for data_stream in data_stream_list:
portal_type = data_stream.getPortalType()
if ((portal_type == "Data Stream" and data_stream.getSize() > 0) or
(portal_type == "Data Bucket Stream" and len(data_stream.getKeyList()) > 0)):
# Data Stream or Data Bucket Stream needs to have data ...
tag = '%s_consistency_check' %data_stream.getPath()
data_stream.activate(tag = tag,
priority = 100,
active_process = active_process.getPath()).DataStream_checkIfNEOCloneBackupIsConsistent(
neo_node_list,
neo_cert_list,
threshold)
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_checkNEOCloneBackupConsistency</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
"""
Return list of Data Streams which should be monitored for consistency.
This is customer specific.
"""
catalog_kw = dict(portal_type = 'Data Stream',
limit = 1000,
validation_state = 'validated')
return context.portal_catalog(**catalog_kw)
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_getDataStreamListToCheck</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
"""
This is hard coded for an instance.
"""
# NEO CLONE
CLONE1_NEO = ('[<IPV6_address_of_NEO_Clone>]:2051', '<name_of_neo_cluster>',)
etc_folder = '<path_to_folder_containing_ssl_certificates>'
node_list = [CLONE1_NEO]
ca_file = '%s/ca.crt' %etc_folder
cert_file = '%s/neo.crt' %etc_folder
key_file = '%s/neo.key' %etc_folder
return [node_list, [ca_file, cert_file, key_file]]
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_getNEONodeListAndSSLCertificateLocation</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
return 1024*1024*10 # 10MB is a good default but it can change from project to project
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_getPreferredThresholdSize</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
"""
Examine remote NEO clone lists results already saved in an Active Process.
"""
from Products.CMFActivity.ActiveResult import ActiveResult
active_result = ActiveResult()
# find latest Active Process
active_process = context.portal_catalog.getResultValue(
portal_type = 'Active Process',
title = 'NEO_Clone_check',
sort_on=(('modification_date', 'descending'),))
result_list = [x.getResult() for x in active_process.getResultList()]
context.log("Check NEO consistency using %s" %active_process.getPath())
is_consistent = 1
for result in result_list:
if result.startswith('PROBLEM:'):
is_consistent = 0
break
if not is_consistent:
severity = 1
summary = "NEO inconsistency report at %s" %active_process.getPath()
detail = result
else:
severity = 0
summary = "Nothing to do."
detail = ""
# save as a result from alarm
active_result.edit(
summary=summary,
severity=severity,
detail=detail)
context.newActiveProcess().postResult(active_result)
return 1
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_isNEOCloneBackupConsistent</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
2016-09-27 Ivan
* Initial
\ No newline at end of file
Administration scripts for NEO.
\ No newline at end of file
extension.erp5.NEOAdministration
\ No newline at end of file
portal_alarms/promise_check_neo_clone_backup_consistency
portal_alarms/promise_check_neo_clone_backup_consistency/**
portal_alarms/promise_is_neo_clone_backup_consistent
portal_alarms/promise_is_neo_clone_backup_consistent/**
\ No newline at end of file
erp5_neo_administration
\ No newline at end of file
erp5_neo_administration
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment