Skip to content

erp5
erp5
Commit f296d380 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki
Browse files
Options

erp5_mysql_innodb_catalog: record deleted object paths. 

so that we can identify recently deleted objects, that are important to resynchronise catalog and ZODB in case of a catastrophe.
parent d6f1aa37
Hide whitespace changes
Inline Side-by-side
Showing with 258 additions and 4 deletions
bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testERP5Core.py
View file @ f296d380
...@@ -35,6 +35,7 @@ import urllib ...@@ -35,6 +35,7 @@ import urllib
import lxml.html import lxml.html
from AccessControl.SecurityManagement import newSecurityManager from AccessControl.SecurityManagement import newSecurityManager
from DateTime import DateTime
from Testing import ZopeTestCase from Testing import ZopeTestCase
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
...@@ -770,3 +771,20 @@ class TestERP5Core(ERP5TypeTestCase, ZopeTestCase.Functional): ...@@ -770,3 +771,20 @@ class TestERP5Core(ERP5TypeTestCase, ZopeTestCase.Functional):
uid=old.getUid(), uid=old.getUid(),
) )
], [old_indexation_timestamp]) ], [old_indexation_timestamp])
def test_deleted_catalog(self):
query = self.portal.erp5_sql_connection().query
query('DELETE FROM deleted_catalog')
person_list = [
self.portal.person_module.newContent(portal_type='Person')
for _ in range(10)
]
self.tic()
self.portal.person_module.manage_delObjects(
ids=[e.getId() for e in person_list],
)
# check if deleted_catalog is immediately updated after deletion.
self.assertEqual(
len(self.portal.z_get_deleted_path_list(timestamp=DateTime() - 1)),
len(person_list),
)
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/z_get_deleted_path_list.sql 0 → 100644
View file @ f296d380
SELECT uid, path FROM deleted_catalog WHERE <dtml-sqltest column="deletion_timestamp" expr="timestamp" op="ge" type="datetime">
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/z_get_deleted_path_list.xml 0 → 100644
View file @ f296d380
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL Method" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>timestamp</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_get_deleted_path_list</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_deleted_catalog.catalog_keys.xml 0 → 100644
View file @ f296d380
<catalog_method>
<item key="sql_clear_catalog" type="int">
<value>1</value>
</item>
</catalog_method>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_deleted_catalog.xml 0 → 100644
View file @ f296d380
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL Method" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_drop_deleted_catalog</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_deleted_catalog.catalog_keys.xml 0 → 100644
View file @ f296d380
<catalog_method>
<item key="sql_clear_catalog" type="int">
<value>1</value>
</item>
</catalog_method>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_deleted_catalog.sql 0 → 100644
View file @ f296d380
<dtml-comment>
* deletion_timestanmp is needed both to know when a deletion happened (when restoring consistency) and to forget old-enough entries.
* path is needed to locate the document once the ZODB has been truncated before the deletion transaction.
* uid is needed to detect if the document we do retrieve is the one which was deleted, or another one.
* an index with deletion_timestanmp as first column is needed for good query performance.
* a primary key is good for good deletion performance.
* there is no natural primary key in this table, because a given path may have multiple uids / a given uid may have multiple paths over time.
</dtml-comment>
CREATE TABLE `deleted_catalog` (
`uid` BIGINT UNSIGNED NOT NULL,
`path` varchar(255) NOT NULL,
`deletion_timestamp` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`deletion_timestamp`, `path`, `uid`)
) ENGINE=InnoDB;
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_deleted_catalog.xml 0 → 100644
View file @ f296d380
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL Method" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_create_deleted_catalog</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_delete_uid.sql
View file @ f296d380
...@@ -3,6 +3,16 @@ REPLACE INTO ...@@ -3,6 +3,16 @@ REPLACE INTO
VALUES VALUES
(<dtml-sqlvar uid type="int">, 'deleted','',NULL,'',NULL) (<dtml-sqlvar uid type="int">, 'deleted','',NULL,'',NULL)
<dtml-var sql_delimiter> <dtml-var sql_delimiter>
<dtml-if expr="'deleted_catalog' in portal_catalog.getSQLCatalog().getTableIds()">
<dtml-comment>
Here we use REPLACE instead of INSERT to avoid duplicate entries.
</dtml-comment>
REPLACE INTO
deleted_catalog (uid, path)
VALUES
(<dtml-sqlvar uid type="int">, <dtml-sqlvar path type="string">)
<dtml-var sql_delimiter>
</dtml-if>
<dtml-comment> <dtml-comment>
Note on "UPDATE stock" query: this query preserve transactionality of movement Note on "UPDATE stock" query: this query preserve transactionality of movement
deletion: all movements deleted in a single transaction will be either deletion: all movements deleted in a single transaction will be either
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_delete_uid.xml
View file @ f296d380
...@@ -14,7 +14,8 @@ ...@@ -14,7 +14,8 @@
</item> </item>
<item> <item>
<key> <string>arguments_src</string> </key> <key> <string>arguments_src</string> </key>
<value> <string>uid</string> </value> <value> <string>uid\n
path</string> </value>
</item> </item>
<item> <item>
<key> <string>cache_time_</string> </key> <key> <string>cache_time_</string> </key>
......
product/ERP5Type/CopySupport.py
View file @ f296d380
...@@ -354,7 +354,7 @@ class CopyContainer: ...@@ -354,7 +354,7 @@ class CopyContainer:
if s is None: ob._p_deactivate() if s is None: ob._p_deactivate()
security.declareProtected(Permissions.ModifyPortalContent, 'unindexObject') security.declareProtected(Permissions.ModifyPortalContent, 'unindexObject')
def unindexObject(self, path=None): def unindexObject(self):
""" """
Unindex the object from the portal catalog. Unindex the object from the portal catalog.
""" """
...@@ -372,7 +372,7 @@ class CopyContainer: ...@@ -372,7 +372,7 @@ class CopyContainer:
# Set the path as deleted, sql wich generate no locks # Set the path as deleted, sql wich generate no locks
# Set also many columns in order to make sure lines # Set also many columns in order to make sure lines
# marked as deleted will not be selected # marked as deleted will not be selected
catalog.beforeUnindexObject(None,path=path,uid=uid) catalog.beforeUnindexObject(None, path=self.getPath(), uid=uid)
# Then start activity in order to remove lines in catalog, # Then start activity in order to remove lines in catalog,
# sql wich generate locks # sql wich generate locks
# - serialization_tag is used in order to prevent unindexation to # - serialization_tag is used in order to prevent unindexation to
......
product/ZSQLCatalog/SQLCatalog.py
View file @ f296d380
...@@ -1488,7 +1488,10 @@ class Catalog(Folder, ...@@ -1488,7 +1488,10 @@ class Catalog(Folder,
LOG('ZSQLCatalog.beforeUncatalogObject', INFO, 'The sql_catalog_delete_uid method is not defined') LOG('ZSQLCatalog.beforeUncatalogObject', INFO, 'The sql_catalog_delete_uid method is not defined')
return self.uncatalogObject(path=path,uid=uid) return self.uncatalogObject(path=path,uid=uid)
method = self._getOb(method_name) method = self._getOb(method_name)
method(uid = uid) try:
method(uid=uid, path=path)
except ValueError: # BBB
method(uid=uid)
def getSqlUncatalogObjectList(self): def getSqlUncatalogObjectList(self):
return self.sql_uncatalog_object return self.sql_uncatalog_object
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7