Skip to content

S
slapos.core
Commit f2b9d9ec authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

slapos_erp5: Basic Disaster recovery scripts 

    This helps sync ZODB and mariadb (considering mariadb is newer). This is useful when restore a clone or synchronize after restore the mariadb catalog.

   The most appropriated way is reindex the site, however in case of slapos master (with large amount of data) it is too long (make this solution more appealing.
parent c2f4d12f
Pipeline #12203 passed with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 443 additions and 0 deletions
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery.xml 0 → 100644
    View file @ f2b9d9ec
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="Folder" module="OFS.Folder"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>_objects</string> </key>
    <value>
    <tuple/>
    </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>slapos_disaster_recovery</string> </value>
    </item>
    <item>
    <key> <string>title</string> </key>
    <value> <string></string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery/ERP5Site_checkDeletedDocumentList.py 0 → 100644
    View file @ f2b9d9ec
    portal = context.getPortalObject()
    document_list = portal.portal_catalog(
    limit=limit,
    uid={'query': min_uid, 'range': 'nlt'},
    sort_on=(('uid', 'ASC'),),
    )
    result_count = len(document_list)
    if result_count:
    if result_count == limit:
    portal.portal_activities.activate(activity='SQLQueue').ERP5Site_checkDeletedDocumentList(document_list[-1].uid, limit, packet_size)
    column_list = [(x.path, x.uid) for x in document_list]
    for i in xrange(0, result_count, packet_size):
    portal.portal_activities.activate(activity='SQLQueue').ERP5Site_unindexDeletedDocumentList(column_list[i:i+packet_size])
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery/ERP5Site_checkDeletedDocumentList.xml 0 → 100644
    View file @ f2b9d9ec
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>Script_magic</string> </key>
    <value> <int>3</int> </value>
    </item>
    <item>
    <key> <string>_bind_names</string> </key>
    <value>
    <object>
    <klass>
    <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
    </klass>
    <tuple/>
    <state>
    <dictionary>
    <item>
    <key> <string>_asgns</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>name_container</string> </key>
    <value> <string>container</string> </value>
    </item>
    <item>
    <key> <string>name_context</string> </key>
    <value> <string>context</string> </value>
    </item>
    <item>
    <key> <string>name_m_self</string> </key>
    <value> <string>script</string> </value>
    </item>
    <item>
    <key> <string>name_subpath</string> </key>
    <value> <string>traverse_subpath</string> </value>
    </item>
    </dictionary>
    </value>
    </item>
    </dictionary>
    </state>
    </object>
    </value>
    </item>
    <item>
    <key> <string>_params</string> </key>
    <value> <string>min_uid, limit, packet_size</string> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>ERP5Site_checkDeletedDocumentList</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery/ERP5Site_checkLatestModifiedDocumentList.py 0 → 100644
    View file @ f2b9d9ec
    from Products.ZSQLCatalog.SQLCatalog import Query
    portal = context.getPortalObject()
    document_list = portal.portal_catalog(
    limit=limit,
    uid={'query': min_uid, 'range': 'nlt'},
    indexation_timestamp=Query(**{'indexation_timestamp': (before, now), 'range': 'minngt'}),
    sort_on=(('uid', 'ASC'),),
    )
    result_count = len(document_list)
    if result_count:
    if result_count == limit:
    portal.portal_activities.activate(activity='SQLQueue').ERP5Site_checkLatestModifiedDocumentList(document_list[-1].uid, limit, packet_size, before, now)
    column_list = [(x.path, x.uid) for x in document_list]
    for i in xrange(0, result_count, packet_size):
    portal.portal_activities.activate(activity='SQLQueue').ERP5Site_reindexOrUnindexDocumentList(column_list[i:i+packet_size])
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery/ERP5Site_checkLatestModifiedDocumentList.xml 0 → 100644
    View file @ f2b9d9ec
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>Script_magic</string> </key>
    <value> <int>3</int> </value>
    </item>
    <item>
    <key> <string>_bind_names</string> </key>
    <value>
    <object>
    <klass>
    <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
    </klass>
    <tuple/>
    <state>
    <dictionary>
    <item>
    <key> <string>_asgns</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>name_container</string> </key>
    <value> <string>container</string> </value>
    </item>
    <item>
    <key> <string>name_context</string> </key>
    <value> <string>context</string> </value>
    </item>
    <item>
    <key> <string>name_m_self</string> </key>
    <value> <string>script</string> </value>
    </item>
    <item>
    <key> <string>name_subpath</string> </key>
    <value> <string>traverse_subpath</string> </value>
    </item>
    </dictionary>
    </value>
    </item>
    </dictionary>
    </state>
    </object>
    </value>
    </item>
    <item>
    <key> <string>_params</string> </key>
    <value> <string>min_uid, limit, packet_size, before, now</string> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>ERP5Site_checkLatestModifiedDocumentList</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery/ERP5Site_recoverFromRestoration.py 0 → 100644
    View file @ f2b9d9ec
    portal = context.getPortalObject()
    now = DateTime()
    before = now - int(7)
    strfstring = '%Y-%m-%d %H:%M:%S'
    portal.ERP5Site_checkLatestModifiedDocumentList(0, 1000, 100, before.strftime(strfstring), now.strftime(strfstring))
    # Force reindexation of recently created document
    # This expect module to use HBTree
    for module_id in portal.objectIds(("ERP5 Folder",)):
    if module_id.endswith("_module"):
    portal[module_id].recurseCallMethod(
    'recursiveReindexObject',
    max_depth=1,
    min_depth=1,
    max_retry=0,
    activity_count=100,
    min_id=before.strftime("%Y%m%d"),
    )
    portal.ERP5Site_checkDeletedDocumentList(0, 1000, 100)
    for module_id in [
    'portal_preferences',
    'portal_categories',
    'portal_alarms',
    'portal_simulation']+portal.objectIds(("ERP5 Folder",)):
    portal[module_id].recurseCallMethod(
    'immediateReindexObject',
    min_depth=1,
    max_depth=10000,
    activate_kw=dict(
    group_method_id='portal_catalog/catalogObjectList',
    alternate_method_id='alternateReindexObject',
    group_method_cost=1,
    priority=6,
    ),
    max_retry=0,
    activity_count=100,
    )
    return "OK"
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery/ERP5Site_recoverFromRestoration.xml 0 → 100644
    View file @ f2b9d9ec
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>Script_magic</string> </key>
    <value> <int>3</int> </value>
    </item>
    <item>
    <key> <string>_bind_names</string> </key>
    <value>
    <object>
    <klass>
    <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
    </klass>
    <tuple/>
    <state>
    <dictionary>
    <item>
    <key> <string>_asgns</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>name_container</string> </key>
    <value> <string>container</string> </value>
    </item>
    <item>
    <key> <string>name_context</string> </key>
    <value> <string>context</string> </value>
    </item>
    <item>
    <key> <string>name_m_self</string> </key>
    <value> <string>script</string> </value>
    </item>
    <item>
    <key> <string>name_subpath</string> </key>
    <value> <string>traverse_subpath</string> </value>
    </item>
    </dictionary>
    </value>
    </item>
    </dictionary>
    </state>
    </object>
    </value>
    </item>
    <item>
    <key> <string>_params</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>ERP5Site_recoverFromRestoration</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery/ERP5Site_reindexOrUnindexDocumentList.py 0 → 100644
    View file @ f2b9d9ec
    for path, uid in column_list:
    try:
    ob = context.restrictedTraverse(path)
    except KeyError:
    context.log("object not found", path)
    context.portal_catalog.activate(activity='SQLQueue').uncatalog_object(uid=uid)
    else:
    ob.reindexObject()
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery/ERP5Site_reindexOrUnindexDocumentList.xml 0 → 100644
    View file @ f2b9d9ec
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>Script_magic</string> </key>
    <value> <int>3</int> </value>
    </item>
    <item>
    <key> <string>_bind_names</string> </key>
    <value>
    <object>
    <klass>
    <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
    </klass>
    <tuple/>
    <state>
    <dictionary>
    <item>
    <key> <string>_asgns</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>name_container</string> </key>
    <value> <string>container</string> </value>
    </item>
    <item>
    <key> <string>name_context</string> </key>
    <value> <string>context</string> </value>
    </item>
    <item>
    <key> <string>name_m_self</string> </key>
    <value> <string>script</string> </value>
    </item>
    <item>
    <key> <string>name_subpath</string> </key>
    <value> <string>traverse_subpath</string> </value>
    </item>
    </dictionary>
    </value>
    </item>
    </dictionary>
    </state>
    </object>
    </value>
    </item>
    <item>
    <key> <string>_params</string> </key>
    <value> <string>column_list</string> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>ERP5Site_reindexOrUnindexDocumentList</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery/ERP5Site_unindexDeletedDocumentList.py 0 → 100644
    View file @ f2b9d9ec
    for path, uid in column_list:
    try:
    _ = context.restrictedTraverse(path)
    except KeyError:
    context.log("object not found", path)
    context.portal_catalog.activate(activity='SQLQueue').uncatalog_object(uid=uid)
    master/bt5/slapos_erp5/SkinTemplateItem/portal_skins/slapos_disaster_recovery/ERP5Site_unindexDeletedDocumentList.xml 0 → 100644
    View file @ f2b9d9ec
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>Script_magic</string> </key>
    <value> <int>3</int> </value>
    </item>
    <item>
    <key> <string>_bind_names</string> </key>
    <value>
    <object>
    <klass>
    <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
    </klass>
    <tuple/>
    <state>
    <dictionary>
    <item>
    <key> <string>_asgns</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>name_container</string> </key>
    <value> <string>container</string> </value>
    </item>
    <item>
    <key> <string>name_context</string> </key>
    <value> <string>context</string> </value>
    </item>
    <item>
    <key> <string>name_m_self</string> </key>
    <value> <string>script</string> </value>
    </item>
    <item>
    <key> <string>name_subpath</string> </key>
    <value> <string>traverse_subpath</string> </value>
    </item>
    </dictionary>
    </value>
    </item>
    </dictionary>
    </state>
    </object>
    </value>
    </item>
    <item>
    <key> <string>_params</string> </key>
    <value> <string>column_list</string> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>ERP5Site_unindexDeletedDocumentList</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_erp5/TestTemplateItem/portal_components/test.erp5.testSlapOSERP5SkinSelection.py
    View file @ f2b9d9ec
    ......@@ -224,6 +224,7 @@ erp5_wechat_secure_payment
    erp5_workflow
    erp5_xhtml_style
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -344,6 +345,7 @@ erp5_web_widget_library
    erp5_wechat_secure_payment
    erp5_workflow
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -464,6 +466,7 @@ erp5_web_widget_library
    erp5_wechat_secure_payment
    erp5_workflow
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -584,6 +587,7 @@ erp5_wechat_secure_payment
    erp5_workflow
    erp5_xhtml_style
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -705,6 +709,7 @@ erp5_wechat_secure_payment
    erp5_workflow
    erp5_xhtml_style
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -823,6 +828,7 @@ erp5_web_widget_library
    erp5_wechat_secure_payment
    erp5_workflow
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -942,6 +948,7 @@ erp5_wechat_secure_payment
    erp5_workflow
    erp5_xhtml_style
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -1060,6 +1067,7 @@ erp5_web_widget_library
    erp5_wechat_secure_payment
    erp5_workflow
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -1178,6 +1186,7 @@ erp5_web_widget_library
    erp5_wechat_secure_payment
    erp5_workflow
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -1297,6 +1306,7 @@ erp5_web_widget_library
    erp5_wechat_secure_payment
    erp5_workflow
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -1417,6 +1427,7 @@ erp5_wechat_secure_payment
    erp5_workflow
    erp5_xhtml_style
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -1537,6 +1548,7 @@ erp5_wechat_secure_payment
    erp5_workflow
    erp5_xhtml_style
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......@@ -1656,6 +1668,7 @@ erp5_web_widget_library
    erp5_wechat_secure_payment
    erp5_workflow
    external_method
    slapos_disaster_recovery
    Images
    """
    self.assertSameSkinSelection(skin_name, selection_string_list)
    ......
    master/bt5/slapos_erp5/bt/template_skin_id_list
    View file @ f2b9d9ec
    slapos_administration
    slapos_base
    slapos_core
    slapos_disaster_recovery
    slapos_erp5
    \ No newline at end of file
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7