Commit f2b9d9ec authored by Rafael Monnerat's avatar Rafael Monnerat

slapos_erp5: Basic Disaster recovery scripts

    This helps sync ZODB and mariadb (considering mariadb is newer). This is useful when restore a clone or synchronize after restore the mariadb catalog.

   The most appropriated way is reindex the site, however in case of slapos master (with large amount of data) it is too long (make this solution more appealing.
parent c2f4d12f
Pipeline #12204 failed with stage
in 0 seconds
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Folder" module="OFS.Folder"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>slapos_disaster_recovery</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
portal = context.getPortalObject()
document_list = portal.portal_catalog(
limit=limit,
uid={'query': min_uid, 'range': 'nlt'},
sort_on=(('uid', 'ASC'),),
)
result_count = len(document_list)
if result_count:
if result_count == limit:
portal.portal_activities.activate(activity='SQLQueue').ERP5Site_checkDeletedDocumentList(document_list[-1].uid, limit, packet_size)
column_list = [(x.path, x.uid) for x in document_list]
for i in xrange(0, result_count, packet_size):
portal.portal_activities.activate(activity='SQLQueue').ERP5Site_unindexDeletedDocumentList(column_list[i:i+packet_size])
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>min_uid, limit, packet_size</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_checkDeletedDocumentList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
from Products.ZSQLCatalog.SQLCatalog import Query
portal = context.getPortalObject()
document_list = portal.portal_catalog(
limit=limit,
uid={'query': min_uid, 'range': 'nlt'},
indexation_timestamp=Query(**{'indexation_timestamp': (before, now), 'range': 'minngt'}),
sort_on=(('uid', 'ASC'),),
)
result_count = len(document_list)
if result_count:
if result_count == limit:
portal.portal_activities.activate(activity='SQLQueue').ERP5Site_checkLatestModifiedDocumentList(document_list[-1].uid, limit, packet_size, before, now)
column_list = [(x.path, x.uid) for x in document_list]
for i in xrange(0, result_count, packet_size):
portal.portal_activities.activate(activity='SQLQueue').ERP5Site_reindexOrUnindexDocumentList(column_list[i:i+packet_size])
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>min_uid, limit, packet_size, before, now</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_checkLatestModifiedDocumentList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
portal = context.getPortalObject()
now = DateTime()
before = now - int(7)
strfstring = '%Y-%m-%d %H:%M:%S'
portal.ERP5Site_checkLatestModifiedDocumentList(0, 1000, 100, before.strftime(strfstring), now.strftime(strfstring))
# Force reindexation of recently created document
# This expect module to use HBTree
for module_id in portal.objectIds(("ERP5 Folder",)):
if module_id.endswith("_module"):
portal[module_id].recurseCallMethod(
'recursiveReindexObject',
max_depth=1,
min_depth=1,
max_retry=0,
activity_count=100,
min_id=before.strftime("%Y%m%d"),
)
portal.ERP5Site_checkDeletedDocumentList(0, 1000, 100)
for module_id in [
'portal_preferences',
'portal_categories',
'portal_alarms',
'portal_simulation']+portal.objectIds(("ERP5 Folder",)):
portal[module_id].recurseCallMethod(
'immediateReindexObject',
min_depth=1,
max_depth=10000,
activate_kw=dict(
group_method_id='portal_catalog/catalogObjectList',
alternate_method_id='alternateReindexObject',
group_method_cost=1,
priority=6,
),
max_retry=0,
activity_count=100,
)
return "OK"
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_recoverFromRestoration</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
for path, uid in column_list:
try:
ob = context.restrictedTraverse(path)
except KeyError:
context.log("object not found", path)
context.portal_catalog.activate(activity='SQLQueue').uncatalog_object(uid=uid)
else:
ob.reindexObject()
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>column_list</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_reindexOrUnindexDocumentList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
for path, uid in column_list:
try:
_ = context.restrictedTraverse(path)
except KeyError:
context.log("object not found", path)
context.portal_catalog.activate(activity='SQLQueue').uncatalog_object(uid=uid)
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>column_list</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_unindexDeletedDocumentList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -224,6 +224,7 @@ erp5_wechat_secure_payment
erp5_workflow
erp5_xhtml_style
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -344,6 +345,7 @@ erp5_web_widget_library
erp5_wechat_secure_payment
erp5_workflow
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -464,6 +466,7 @@ erp5_web_widget_library
erp5_wechat_secure_payment
erp5_workflow
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -584,6 +587,7 @@ erp5_wechat_secure_payment
erp5_workflow
erp5_xhtml_style
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -705,6 +709,7 @@ erp5_wechat_secure_payment
erp5_workflow
erp5_xhtml_style
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -823,6 +828,7 @@ erp5_web_widget_library
erp5_wechat_secure_payment
erp5_workflow
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -942,6 +948,7 @@ erp5_wechat_secure_payment
erp5_workflow
erp5_xhtml_style
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -1060,6 +1067,7 @@ erp5_web_widget_library
erp5_wechat_secure_payment
erp5_workflow
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -1178,6 +1186,7 @@ erp5_web_widget_library
erp5_wechat_secure_payment
erp5_workflow
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -1297,6 +1306,7 @@ erp5_web_widget_library
erp5_wechat_secure_payment
erp5_workflow
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -1417,6 +1427,7 @@ erp5_wechat_secure_payment
erp5_workflow
erp5_xhtml_style
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -1537,6 +1548,7 @@ erp5_wechat_secure_payment
erp5_workflow
erp5_xhtml_style
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......@@ -1656,6 +1668,7 @@ erp5_web_widget_library
erp5_wechat_secure_payment
erp5_workflow
external_method
slapos_disaster_recovery
Images
"""
self.assertSameSkinSelection(skin_name, selection_string_list)
......
slapos_administration
slapos_base
slapos_core
slapos_disaster_recovery
slapos_erp5
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment