Skip to content

erp5
erp5
Commit 69aefdff authored by Vincent Pelletier's avatar Vincent Pelletier
3 Browse files
Options

ZSQLCatalog: Drop support for "reserved" path. 

ZSQLCatalog now requires having portal_ids.
Keep support for clearing existing ones.

Any existing "reserved" row encountered while indexing documents will be
treated as a uid being attributed to more than one path.
parent 0d21e66d
Show whitespace changes
Inline Side-by-side
Showing with 22 additions and 263 deletions
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_produce_reserved_uid_list.catalog_keys.xml deleted 100644 → 0
View file @ 0d21e66d
<catalog_method>
<item key="sql_catalog_produce_reserved" type="int">
<value>1</value>
</item>
</catalog_method>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_produce_reserved_uid_list.sql deleted 100644 → 0
View file @ 0d21e66d
INSERT INTO
catalog (id, path)
VALUES
<dtml-in "_.range(count - 1)">
( <dtml-sqlvar instance_id type="string"> , 'reserved' ) ,
</dtml-in>
( <dtml-sqlvar instance_id type="string"> , 'reserved' )
<dtml-var sql_delimiter>
SELECT
uid
FROM
catalog
WHERE
path = 'reserved'
AND
id = <dtml-sqlvar instance_id type="string">
LIMIT
10000
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_produce_reserved_uid_list.xml deleted 100644 → 0
View file @ 0d21e66d
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL Method" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>count:int\r\n
instance_id</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_produce_reserved_uid_list</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_reserve_uid.catalog_keys.xml deleted 100644 → 0
View file @ 0d21e66d
<catalog_method>
<item key="sql_catalog_reserve_uid" type="int">
<value>1</value>
</item>
</catalog_method>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_reserve_uid.sql deleted 100644 → 0
View file @ 0d21e66d
REPLACE INTO
catalog (uid, id, path)
VALUES
<dtml-in uid>
(<dtml-sqlvar sequence-item type="int">, 'used', 'reserved')
<dtml-if sequence-end><dtml-else>,</dtml-if>
</dtml-in>
\ No newline at end of file
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_reserve_uid.xml deleted 100644 → 0
View file @ 0d21e66d
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL Method" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid:list</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_reserve_uid</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/template_catalog_method_id_list
View file @ 69aefdff
...@@ -79,7 +79,6 @@ erp5_mysql_innodb/z_getitem_by_path ...@@ -79,7 +79,6 @@ erp5_mysql_innodb/z_getitem_by_path
erp5_mysql_innodb/z_getitem_by_uid erp5_mysql_innodb/z_getitem_by_uid
erp5_mysql_innodb/z_portal_ids_commit erp5_mysql_innodb/z_portal_ids_commit
erp5_mysql_innodb/z_portal_ids_generate_id erp5_mysql_innodb/z_portal_ids_generate_id
erp5_mysql_innodb/z_produce_reserved_uid_list
erp5_mysql_innodb/z_read_recorded_object_list erp5_mysql_innodb/z_read_recorded_object_list
erp5_mysql_innodb/z_record_object_list erp5_mysql_innodb/z_record_object_list
erp5_mysql_innodb/z_related_base_category_id erp5_mysql_innodb/z_related_base_category_id
...@@ -130,7 +129,6 @@ erp5_mysql_innodb/z_related_translated_validation_state ...@@ -130,7 +129,6 @@ erp5_mysql_innodb/z_related_translated_validation_state
erp5_mysql_innodb/z_related_translated_validation_state_title erp5_mysql_innodb/z_related_translated_validation_state_title
erp5_mysql_innodb/z_related_uid erp5_mysql_innodb/z_related_uid
erp5_mysql_innodb/z_related_uid_or_parent_uid erp5_mysql_innodb/z_related_uid_or_parent_uid
erp5_mysql_innodb/z_reserve_uid
erp5_mysql_innodb/z_search_results erp5_mysql_innodb/z_search_results
erp5_mysql_innodb/z_search_security erp5_mysql_innodb/z_search_security
erp5_mysql_innodb/z_select_optimizer_switch erp5_mysql_innodb/z_select_optimizer_switch
......
product/ZSQLCatalog/SQLCatalog.py
View file @ 69aefdff
...@@ -379,21 +379,11 @@ class Catalog(Folder, ...@@ -379,21 +379,11 @@ class Catalog(Folder,
'mode' : 'w' }, 'mode' : 'w' },
# Z SQL Methods # Z SQL Methods
{ 'id' : 'sql_catalog_produce_reserved',
'description' : 'A method to produce new uid values in advance',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_clear_reserved', { 'id' : 'sql_catalog_clear_reserved',
'description' : 'A method to clear reserved uid values', 'description' : 'A method to clear reserved uid values',
'type' : 'selection', 'type' : 'selection',
'select_variable' : 'getCatalogMethodIds', 'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' }, 'mode' : 'w' },
{ 'id' : 'sql_catalog_reserve_uid',
'description' : 'A method to reserve a uid value',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_delete_uid', { 'id' : 'sql_catalog_delete_uid',
'description' : 'A method to delete a uid value', 'description' : 'A method to delete a uid value',
'type' : 'selection', 'type' : 'selection',
...@@ -587,10 +577,8 @@ class Catalog(Folder, ...@@ -587,10 +577,8 @@ class Catalog(Folder,
) )
sql_catalog_produce_reserved = ''
sql_catalog_delete_uid = '' sql_catalog_delete_uid = ''
sql_catalog_clear_reserved = '' sql_catalog_clear_reserved = ''
sql_catalog_reserve_uid = ''
sql_catalog_object_list = () sql_catalog_object_list = ()
sql_uncatalog_object = () sql_uncatalog_object = ()
sql_clear_catalog = () sql_clear_catalog = ()
...@@ -754,7 +742,6 @@ class Catalog(Folder, ...@@ -754,7 +742,6 @@ class Catalog(Folder,
if not security_uid: if not security_uid:
getTransactionalVariable().pop('getSecurityUidDictAndRoleColumnDict', getTransactionalVariable().pop('getSecurityUidDictAndRoleColumnDict',
None) None)
id_tool = getattr(self.getPortalObject(), 'portal_ids', None)
# We must keep compatibility with existing sites # We must keep compatibility with existing sites
security_uid = getattr(self, 'security_uid_index', None) security_uid = getattr(self, 'security_uid_index', None)
if security_uid is None: if security_uid is None:
...@@ -762,16 +749,13 @@ class Catalog(Folder, ...@@ -762,16 +749,13 @@ class Catalog(Folder,
# At some point, it was a Length # At some point, it was a Length
elif isinstance(security_uid, Length): elif isinstance(security_uid, Length):
security_uid = security_uid() security_uid = security_uid()
# If the id_tool is there, it is better to use it, it allows security_uid = int(
# to create many new security uids by the same time self.getPortalObject().portal_ids.generateNewId(
# because with this tool we are sure that we will have 2 different id_generator='uid',
# uids if two instances are doing this code in the same time id_group='security_uid_index',
security_uid += 1 default=security_uid,
if id_tool is not None: ),
security_uid = int(id_tool.generateNewId(id_generator='uid', )
id_group='security_uid_index', default=security_uid))
else:
self.security_uid_index = security_uid
self.security_uid_dict[key] = security_uid self.security_uid_dict[key] = security_uid
local_roles_group_id_to_security_uid_mapping[local_roles_group_id]\ local_roles_group_id_to_security_uid_mapping[local_roles_group_id]\
...@@ -864,33 +848,10 @@ class Catalog(Folder, ...@@ -864,33 +848,10 @@ class Catalog(Folder,
raise raise
# Reserved uids have been removed. # Reserved uids have been removed.
self._clearReserved() self._clearReserved()
id_tool = getattr(self.getPortalObject(), 'portal_ids', None)
if id_tool is None:
# Add a dummy item so that SQLCatalog will not use existing uids again.
self.insertMaxUid()
self._clearSecurityCache() self._clearSecurityCache()
self._clearSubjectCache() self._clearSubjectCache()
self._clearCaches() self._clearCaches()
def getSqlCatalogReservedUid(self):
return self.sql_catalog_reserve_uid
security.declarePrivate('insertMaxUid')
def insertMaxUid(self):
"""
Add a dummy item so that SQLCatalog will not use existing uids again.
"""
if self._max_uid is not None and self._max_uid() != 0:
method_id = self.getSqlCatalogReservedUid()
method = self._getOb(method_id)
self._max_uid.change(1)
method(uid = [self._max_uid()])
def getSqlCatalogClearReserved(self):
return self.sql_catalog_clear_reserved
def _clearReserved(self): def _clearReserved(self):
""" """
Clears reserved uids Clears reserved uids
...@@ -1103,9 +1064,6 @@ class Catalog(Folder, ...@@ -1103,9 +1064,6 @@ class Catalog(Folder,
uid_buffer_dict[thread_key] = UidBuffer() uid_buffer_dict[thread_key] = UidBuffer()
return uid_buffer_dict[thread_key] return uid_buffer_dict[thread_key]
def getSqlCatalogProduceReserved(self):
return self.sql_catalog_produce_reserved
# the cataloging API # the cataloging API
security.declarePrivate('produceUid') security.declarePrivate('produceUid')
def produceUid(self): def produceUid(self):
...@@ -1120,29 +1078,20 @@ class Catalog(Folder, ...@@ -1120,29 +1078,20 @@ class Catalog(Folder,
uid_buffer = self.getUIDBuffer(force_new_buffer=force_new_buffer) uid_buffer = self.getUIDBuffer(force_new_buffer=force_new_buffer)
global_clear_reserved_time = self._last_clear_reserved_time global_clear_reserved_time = self._last_clear_reserved_time
if len(uid_buffer) == 0: if len(uid_buffer) == 0:
id_tool = getattr(self.getPortalObject(), 'portal_ids', None)
if id_tool is not None:
if self._max_uid is None: if self._max_uid is None:
self._max_uid = Length(1) self._max_uid = Length(1)
uid_list = id_tool.generateNewIdList(id_generator='uid', id_group='catalog_uid',
id_count=UID_BUFFER_SIZE, default=self._max_uid())
# TODO: if this method is kept and former uid allocation code is # TODO: if this method is kept and former uid allocation code is
# discarded, self._max_uid duplicates work done by portal_ids: it # discarded, self._max_uid duplicates work done by portal_ids: it
# already keeps track of the highest allocated number for all id # already keeps track of the highest allocated number for all id
# generator groups. # generator groups.
else: uid_buffer.extend(
method_id = self.getSqlCatalogProduceReserved() self.getPortalObject().portal_ids.generateNewIdList(
method = self._getOb(method_id) id_generator='uid',
# Generate an instance id randomly. Note that there is a small possibility that this id_group='catalog_uid',
# would conflict with others. id_count=UID_BUFFER_SIZE,
random_factor_list = [time.time(), os.getpid(), os.times()] default=self._max_uid(),
try: ),
random_factor_list.append(os.getloadavg()) )
except (OSError, AttributeError): # AttributeError is required under cygwin
pass
instance_id = md5(str(random_factor_list)).hexdigest()
uid_list = [x.uid for x in method(count = UID_BUFFER_SIZE, instance_id = instance_id) if x.uid != 0]
uid_buffer.extend(uid_list)
security.declarePrivate('isIndexable') security.declarePrivate('isIndexable')
def isIndexable(self): def isIndexable(self):
...@@ -1432,23 +1381,7 @@ class Catalog(Folder, ...@@ -1432,23 +1381,7 @@ class Catalog(Folder,
catalog_path = uid_path_dict.get(uid) catalog_path = uid_path_dict.get(uid)
else: else:
catalog_path = self.getPathForUid(uid) catalog_path = self.getPathForUid(uid)
if catalog_path == "reserved": if catalog_path == 'deleted':
with global_reserved_uid_lock:
uid_buffer = self.getUIDBuffer()
if uid_buffer is not None:
# This is the case where:
# 1. The object got an uid.
# 2. The catalog was cleared.
# 3. The catalog produced the same reserved uid.
# 4. The object was reindexed.
# In this case, the uid is not reserved any longer, but
# SQLCatalog believes that it is still reserved. So it is
# necessary to remove the uid from the list explicitly.
try:
uid_buffer.remove(uid)
except ValueError:
pass
elif catalog_path == 'deleted':
# Two possible cases: # Two possible cases:
# - Reindexed object's path changed (ie, it or at least one of its # - Reindexed object's path changed (ie, it or at least one of its
# parents was renamed) but unindexObject was not called yet. # parents was renamed) but unindexObject was not called yet.
......
product/ZSQLCatalog/ZSQLCatalog.py
View file @ 69aefdff
...@@ -372,12 +372,6 @@ class ZCatalog(Folder, Persistent, Implicit): ...@@ -372,12 +372,6 @@ class ZCatalog(Folder, Persistent, Implicit):
""" """
if self.getDefaultSqlCatalogId() == source_sql_catalog_id: if self.getDefaultSqlCatalogId() == source_sql_catalog_id:
self._setDefaultSqlCatalogId(destination_sql_catalog_id) self._setDefaultSqlCatalogId(destination_sql_catalog_id)
id_tool = getattr(self.getPortalObject(), 'portal_ids', None)
if id_tool is None:
# Insert the latest generated uid.
# This must be done just before swaping the catalogs in case there were
# generated uids since destination catalog was created.
self[destination_sql_catalog_id].insertMaxUid()
LOG('_exchangeDatabases skin_selection_dict:',0,skin_selection_dict) LOG('_exchangeDatabases skin_selection_dict:',0,skin_selection_dict)
if skin_selection_dict is not None: if skin_selection_dict is not None:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7