Commit d7fca2a4 authored by Tatuya Kamada's avatar Tatuya Kamada

[IdTool] Use OOBTree instead of PersistentMapping in IdTool.

Use OOBTree instead of PersistentMapping. Because it is more disk-efficent,
it will reduce the number of conflicts, improve the performance.
parent eca30de0
...@@ -29,13 +29,13 @@ ...@@ -29,13 +29,13 @@
import zope.interface import zope.interface
from Acquisition import aq_base from Acquisition import aq_base
from AccessControl import ClassSecurityInfo from AccessControl import ClassSecurityInfo
from Products.ERP5Type.Globals import PersistentMapping
from Products.ERP5Type import Permissions, PropertySheet, interfaces from Products.ERP5Type import Permissions, PropertySheet, interfaces
from Products.ERP5Type.Utils import ScalarMaxConflictResolver from Products.ERP5Type.Utils import ScalarMaxConflictResolver
from Products.ERP5.Document.IdGenerator import IdGenerator from Products.ERP5.Document.IdGenerator import IdGenerator
from _mysql_exceptions import ProgrammingError from _mysql_exceptions import ProgrammingError
from MySQLdb.constants.ER import NO_SUCH_TABLE from MySQLdb.constants.ER import NO_SUCH_TABLE
from zLOG import LOG, INFO from zLOG import LOG, INFO
from BTrees.OOBTree import OOBTree
class SQLNonContinuousIncreasingIdGenerator(IdGenerator): class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
""" """
...@@ -135,7 +135,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator): ...@@ -135,7 +135,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
id_group_done.append(id_group) id_group_done.append(id_group)
# save the last ids which not exist in sql # save the last ids which not exist in sql
for id_group in (set(self.last_max_id_dict) - set(id_group_done)): for id_group in (set(self.last_max_id_dict.keys()) - set(id_group_done)):
set_last_id_method(id_group=id_group, set_last_id_method(id_group=id_group,
last_id=self.last_max_id_dict[id_group].value) last_id=self.last_max_id_dict[id_group].value)
...@@ -168,7 +168,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator): ...@@ -168,7 +168,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
LOG('initialize SQL Generator', INFO, 'Id Generator: %s' % (self,)) LOG('initialize SQL Generator', INFO, 'Id Generator: %s' % (self,))
# Check the dictionnary # Check the dictionnary
if self.last_max_id_dict is None: if self.last_max_id_dict is None:
self.last_max_id_dict = PersistentMapping() self.last_max_id_dict = OOBTree()
# Create table portal_ids if not exists # Create table portal_ids if not exists
portal = self.getPortalObject() portal = self.getPortalObject()
try: try:
...@@ -219,7 +219,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator): ...@@ -219,7 +219,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
added here) added here)
""" """
# Remove dictionary # Remove dictionary
self.last_max_id_dict = PersistentMapping() self.last_max_id_dict = OOBTree()
# Remove and recreate portal_ids table # Remove and recreate portal_ids table
portal = self.getPortalObject() portal = self.getPortalObject()
portal.IdTool_zDropTable() portal.IdTool_zDropTable()
...@@ -263,9 +263,24 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator): ...@@ -263,9 +263,24 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
# Update persistent dict # Update persistent dict
if self.getStoredInZodb(): if self.getStoredInZodb():
if self.last_max_id_dict is None: if self.last_max_id_dict is None:
self.last_max_id_dict = PersistentMapping() self.last_max_id_dict = OOBTree()
self.last_max_id_dict.update(new_id_dict) self.last_max_id_dict.update(new_id_dict)
security.declareProtected(Permissions.ModifyPortalContent,
'rebuildGeneratorIdDict')
def rebuildGeneratorIdDict(self):
"""
Rebuild generator id dict from SQL table.
This is usefull when we are migrating the dict structure, or cleanly
rebuild the dict from sql table. This method is opposite of
rebuildSqlTable().
"""
if not self.getStoredInZodb():
raise RuntimeError('Please set \"stored in zodb\" flag before rebuild.')
id_dict = self.exportGeneratorIdDict()
self.importGeneratorIdDict(id_dict=id_dict, clear=True)
security.declareProtected(Permissions.ModifyPortalContent, security.declareProtected(Permissions.ModifyPortalContent,
'rebuildSqlTable') 'rebuildSqlTable')
def rebuildSqlTable(self): def rebuildSqlTable(self):
...@@ -319,7 +334,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator): ...@@ -319,7 +334,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
portal = self.getPortalObject() portal = self.getPortalObject()
last_max_id_dict = self.last_max_id_dict last_max_id_dict = self.last_max_id_dict
if last_max_id_dict is None: if last_max_id_dict is None:
self.last_max_id_dict = last_max_id_dict = PersistentMapping() self.last_max_id_dict = last_max_id_dict = OOBTree()
last_id_group = None last_id_group = None
for line in portal.IdTool_zGetValueList(id_group=id_group): for line in portal.IdTool_zGetValueList(id_group=id_group):
last_id_group = id_group = line[0] last_id_group = id_group = line[0]
......
...@@ -28,9 +28,9 @@ ...@@ -28,9 +28,9 @@
import zope.interface import zope.interface
from AccessControl import ClassSecurityInfo from AccessControl import ClassSecurityInfo
from Products.ERP5Type.Globals import PersistentMapping
from Products.ERP5Type import Permissions, interfaces from Products.ERP5Type import Permissions, interfaces
from Products.ERP5.Document.IdGenerator import IdGenerator from Products.ERP5.Document.IdGenerator import IdGenerator
from BTrees.OOBTree import OOBTree
from zLOG import LOG, INFO from zLOG import LOG, INFO
...@@ -97,7 +97,7 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator): ...@@ -97,7 +97,7 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator):
""" """
LOG('initialize ZODB Generator', INFO, 'Id Generator: %s' % (self,)) LOG('initialize ZODB Generator', INFO, 'Id Generator: %s' % (self,))
if getattr(self, 'last_id_dict', None) is None: if getattr(self, 'last_id_dict', None) is None:
self.last_id_dict = PersistentMapping() self.last_id_dict = OOBTree()
# XXX compatiblity code below, dump the old dictionnaries # XXX compatiblity code below, dump the old dictionnaries
portal_ids = getattr(self, 'portal_ids', None) portal_ids = getattr(self, 'portal_ids', None)
...@@ -124,7 +124,7 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator): ...@@ -124,7 +124,7 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator):
added here) added here)
""" """
# Remove dictionary # Remove dictionary
self.last_id_dict = PersistentMapping() self.last_id_dict = OOBTree()
security.declareProtected(Permissions.ModifyPortalContent, security.declareProtected(Permissions.ModifyPortalContent,
'exportGeneratorIdDict') 'exportGeneratorIdDict')
...@@ -149,3 +149,17 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator): ...@@ -149,3 +149,17 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator):
if not isinstance(value, int): if not isinstance(value, int):
raise TypeError, 'the value given in dictionary is not a integer' raise TypeError, 'the value given in dictionary is not a integer'
self.last_id_dict.update(id_dict) self.last_id_dict.update(id_dict)
security.declareProtected(Permissions.ModifyPortalContent,
'rebuildGeneratorIdDict')
def rebuildGeneratorIdDict(self):
"""
Rebuild generator id dict.
In fact, export it, clear it and import it into new dict.
This is mostly intendted to use when we are migrating the id dict
structure.
"""
id_dict = self.exportGeneratorIdDict()
self.importGeneratorIdDict(id_dict=id_dict, clear=True)
...@@ -33,6 +33,7 @@ import unittest ...@@ -33,6 +33,7 @@ import unittest
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.tests.utils import createZODBPythonScript from Products.ERP5Type.tests.utils import createZODBPythonScript
from _mysql_exceptions import ProgrammingError from _mysql_exceptions import ProgrammingError
from BTrees.OOBTree import OOBTree
class TestIdTool(ERP5TypeTestCase): class TestIdTool(ERP5TypeTestCase):
...@@ -154,7 +155,7 @@ class TestIdTool(ERP5TypeTestCase): ...@@ -154,7 +155,7 @@ class TestIdTool(ERP5TypeTestCase):
zodb_generator = self.getLastGenerator('test_application_zodb') zodb_generator = self.getLastGenerator('test_application_zodb')
zodb_portal_type = 'ZODB Continuous Increasing Id Generator' zodb_portal_type = 'ZODB Continuous Increasing Id Generator'
self.assertEquals(zodb_generator.getPortalType(), zodb_portal_type) self.assertEquals(zodb_generator.getPortalType(), zodb_portal_type)
self.assertEqual(getattr(zodb_generator, 'last_id_dict', {}), {}) self.assertEqual(len(zodb_generator.last_id_dict), 0)
# generate ids # generate ids
self.checkGenerateNewId('test_application_zodb') self.checkGenerateNewId('test_application_zodb')
# check zodb dict # check zodb dict
...@@ -171,7 +172,11 @@ class TestIdTool(ERP5TypeTestCase): ...@@ -171,7 +172,11 @@ class TestIdTool(ERP5TypeTestCase):
sql_generator = self.getLastGenerator('test_application_sql') sql_generator = self.getLastGenerator('test_application_sql')
sql_portal_type = 'SQL Non Continuous Increasing Id Generator' sql_portal_type = 'SQL Non Continuous Increasing Id Generator'
self.assertEquals(sql_generator.getPortalType(), sql_portal_type) self.assertEquals(sql_generator.getPortalType(), sql_portal_type)
self.assertEquals(getattr(sql_generator, 'last_max_id_dict', {}), {}) # This assertEquals() make sure that last_max_id_dict property is empty.
# Note that keys(), values() and items() methods of OOBTree do not return
# a list of all the items. The methods return a lazy evaluated object.
# len() method on OOBTree can handle properly even in the situation.
self.assertEquals(len(sql_generator.last_max_id_dict), 0)
# retrieve method to recovery the last id in the database # retrieve method to recovery the last id in the database
last_id_method = getattr(self.portal, 'IdTool_zGetLastId', None) last_id_method = getattr(self.portal, 'IdTool_zGetLastId', None)
self.assertNotEquals(last_id_method, None) self.assertNotEquals(last_id_method, None)
...@@ -189,7 +194,7 @@ class TestIdTool(ERP5TypeTestCase): ...@@ -189,7 +194,7 @@ class TestIdTool(ERP5TypeTestCase):
self.assertEquals(sql_generator.last_max_id_dict['c02'].value, 0) self.assertEquals(sql_generator.last_max_id_dict['c02'].value, 0)
self.assertEquals(sql_generator.last_max_id_dict['d02'].value, 21) self.assertEquals(sql_generator.last_max_id_dict['d02'].value, 21)
else: else:
self.assertEquals(getattr(sql_generator, 'last_max_id_dict', {}), {}) self.assertEquals(len(sql_generator.last_max_id_dict), 0)
def test_02b_generateNewIdWithSQLGeneratorWithoutStorageZODB(self): def test_02b_generateNewIdWithSQLGeneratorWithoutStorageZODB(self):
""" """
......
...@@ -27,12 +27,17 @@ ...@@ -27,12 +27,17 @@
# #
############################################################################## ##############################################################################
import unittest
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.Globals import PersistentMapping from Products.ERP5Type.Globals import PersistentMapping
from Products.ERP5Type.Utils import ScalarMaxConflictResolver
from BTrees.Length import Length from BTrees.Length import Length
from BTrees.OOBTree import OOBTree
from zLOG import LOG from zLOG import LOG
class TestIdTool(ERP5TypeTestCase):
class TestIdToolUpgrade(ERP5TypeTestCase):
""" """
Automatic upgrade of id tool is really sensible to any change. Therefore, Automatic upgrade of id tool is really sensible to any change. Therefore,
make sure that the upgrade is still working even if there changes. make sure that the upgrade is still working even if there changes.
...@@ -46,6 +51,66 @@ class TestIdTool(ERP5TypeTestCase): ...@@ -46,6 +51,66 @@ class TestIdTool(ERP5TypeTestCase):
""" """
return "Test Id Tool Upgrade" return "Test Id Tool Upgrade"
def afterSetUp(self):
self.login()
self.portal = self.getPortal()
self.id_tool = self.portal.portal_ids
self.id_tool.initializeGenerator(all=True)
self.createGenerators()
self.tic()
def beforeTearDown(self):
self.id_tool.clearGenerator(all=True)
def createGenerators(self):
"""
Initialize some generators for the tests
"""
self.application_sql_generator = self.id_tool.newContent(\
portal_type='Application Id Generator',
reference='test_application_sql',
version='001')
self.conceptual_sql_generator = self.id_tool.newContent(\
portal_type='Conceptual Id Generator',
reference='test_non_continuous_increasing',
version='001')
self.sql_generator = self.id_tool.newContent(\
portal_type='SQL Non Continuous Increasing Id Generator',
reference='test_sql_non_continuous_increasing',
version='001')
self.application_sql_generator.setSpecialiseValue(\
self.conceptual_sql_generator)
self.conceptual_sql_generator.setSpecialiseValue(self.sql_generator)
self.application_zodb_generator = self.id_tool.newContent(\
portal_type='Application Id Generator',
reference='test_application_zodb',
version='001')
self.conceptual_zodb_generator = self.id_tool.newContent(\
portal_type='Conceptual Id Generator',
reference='test_continuous_increasing',
version='001')
self.zodb_generator = self.id_tool.newContent(\
portal_type='ZODB Continuous Increasing Id Generator',
reference='test_zodb_continuous_increasing',
version='001')
self.application_zodb_generator.setSpecialiseValue(\
self.conceptual_zodb_generator)
self.conceptual_zodb_generator.setSpecialiseValue(self.zodb_generator)
def getLastGenerator(self, id_generator):
"""
Return Last Id Generator
"""
document_generator = self.id_tool.searchFolder(reference=id_generator)[0]
application_generator = document_generator.getLatestVersionValue()
conceptual_generator = application_generator.getSpecialiseValue()\
.getLatestVersionValue()
last_generator = conceptual_generator.getSpecialiseValue()\
.getLatestVersionValue()
return last_generator
def testUpgradeIdToolDicts(self): def testUpgradeIdToolDicts(self):
# With old erp5_core, we have no generators, no IdTool_* zsql methods, # With old erp5_core, we have no generators, no IdTool_* zsql methods,
# and we have a dictionary stored on id tool # and we have a dictionary stored on id tool
...@@ -123,3 +188,132 @@ class TestIdTool(ERP5TypeTestCase): ...@@ -123,3 +188,132 @@ class TestIdTool(ERP5TypeTestCase):
generator = generator_list[0] generator = generator_list[0]
self.assertEquals(generator.last_id_dict['foo'], 4) self.assertEquals(generator.last_id_dict['foo'], 4)
self.assertEquals(generator.last_id_dict["('bar', 'baz')"], 3) self.assertEquals(generator.last_id_dict["('bar', 'baz')"], 3)
def _setUpLastMaxIdDict(self, id_generator_reference):
def countup(id_generator, id_group, until):
for i in xrange(until + 1):
self.id_tool.generateNewId(id_generator=id_generator_reference,
id_group=id_group)
countup(id_generator_reference, 'A-01', 2)
countup(id_generator_reference, 'B-01', 1)
var_id = 'C-%04d'
for x in xrange(self.a_lot_of_key):
countup(id_generator_reference, var_id % x, 0)
def _getLastIdDictName(self, id_generator):
portal_type = id_generator.getPortalType()
if portal_type == 'SQL Non Continuous Increasing Id Generator':
return 'last_max_id_dict'
elif portal_type == 'ZODB Continuous Increasing Id Generator':
return 'last_id_dict'
else:
raise RuntimeError("not expected to test the generator :%s" % portal_type)
def _getLastIdDict(self, id_generator):
last_id_dict_name = self._getLastIdDictName(id_generator)
return getattr(id_generator, last_id_dict_name)
def _setLastIdDict(self, id_generator, value):
last_id_dict_name = self._getLastIdDictName(id_generator)
setattr(id_generator, last_id_dict_name, value)
def _getValueFromLastIdDict(self, last_id_dict, key):
value = last_id_dict[key]
if isinstance(value, int):
# in ZODB Id Generator it is stored in int
return value
elif isinstance(value, ScalarMaxConflictResolver):
return value.value
else:
raise RuntimeError('not expected to test the value: %s' % value)
def _assertIdGeneratorLastMaxIdDict(self, id_generator):
last_id_dict = self._getLastIdDict(id_generator)
self.assertEquals(2, self._getValueFromLastIdDict(last_id_dict, 'A-01'))
self.assertEquals(1, self._getValueFromLastIdDict(last_id_dict, 'B-01'))
for x in xrange(self.a_lot_of_key):
key = 'C-%04d' % x
self.assertEquals(0, self._getValueFromLastIdDict(last_id_dict, key))
# 1(A-01) + 1(B-01) + a_lot_of_key(C-*)
number_of_group_id = self.a_lot_of_key + 2
self.assertEqual(number_of_group_id,
len(id_generator.exportGeneratorIdDict()))
self.assertEqual(number_of_group_id, len(last_id_dict))
def _checkDataStructureMigration(self, id_generator):
""" First, simulate previous data structure which is using
PersisntentMapping as the storage, then migrate to OOBTree.
Then, migrate the id generator again from OOBTree to OOBtree
just to be sure."""
id_generator_reference = id_generator.getReference()
reference_portal_type_dict = {
'test_sql_non_continuous_increasing':'SQL Non Continuous ' \
'Increasing Id Generator',
'test_zodb_continuous_increasing':'ZODB Continuous ' \
'Increasing Id Generator'
}
try:
portal_type = reference_portal_type_dict[id_generator_reference]
self.assertEquals(id_generator.getPortalType(), portal_type)
except:
raise ValueError("reference is not valid: %s" % id_generator_reference)
self._setLastIdDict(id_generator, PersistentMapping()) # simulate previous
last_id_dict = self._getLastIdDict(id_generator)
# setUp the data for migration test
self._setUpLastMaxIdDict(id_generator_reference)
# test migration: PersistentMapping to OOBTree
self.assertTrue(isinstance(last_id_dict, PersistentMapping))
self._assertIdGeneratorLastMaxIdDict(id_generator)
id_generator.rebuildGeneratorIdDict() # migrate the dict
self._assertIdGeneratorLastMaxIdDict(id_generator)
# test migration: OOBTree to OOBTree. this changes nothing, just to be sure
last_id_dict = self._getLastIdDict(id_generator)
self.assertTrue(isinstance(last_id_dict, OOBTree))
self._assertIdGeneratorLastMaxIdDict(id_generator)
id_generator.rebuildGeneratorIdDict() # migrate the dict
self._assertIdGeneratorLastMaxIdDict(id_generator)
# test migration: SQL to OOBTree
if id_generator.getPortalType() == \
'SQL Non Continuous Increasing Id Generator':
self._setLastIdDict(id_generator, OOBTree()) # set empty one
last_id_dict = self._getLastIdDict(id_generator)
assert(len(last_id_dict), 0) # 0 because it is empty
self.assertTrue(isinstance(last_id_dict, OOBTree))
# migrate the dict totally from sql table in this case
id_generator.rebuildGeneratorIdDict()
self._assertIdGeneratorLastMaxIdDict(id_generator)
def testRebuildIdDictFromPersistentMappingToOOBTree(self):
"""
Check migration is working
"""
# this is the amount of keys that is creating in this test
self.a_lot_of_key = 1010
# check sql id generator migration
id_generator_reference = 'test_application_sql'
id_generator = self.getLastGenerator(id_generator_reference)
id_generator.setStoredInZodb(True)
id_generator.clearGenerator() # clear stored data
self._checkDataStructureMigration(id_generator)
# check zodb id generator migration
id_generator_reference = 'test_application_zodb'
id_generator = self.getLastGenerator(id_generator_reference)
id_generator.clearGenerator() # clear stored data
self._checkDataStructureMigration(id_generator)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestIdToolUpgrade))
return suite
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment