Commit d2189e27 authored by Tatuya Kamada's avatar Tatuya Kamada

Make id dict updating more scalable.

- Avoid to create too much OIDs when updateLastMaxIdDictFromTable
- Add a parameter 'id_group' into IdTool_zGetValueList for the
  scalability of updateLastMaxIdDictFromTable().
  This parameter is intended to use with CMFActivity
  (see TestIdTool.test_08_updateLastMaxIdDictFromTable).
parent 4ba4d26f
......@@ -121,12 +121,11 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
Update the portal ids table with the data of persistent dictionary
"""
portal = self.getPortalObject()
get_value_list = portal.IdTool_zGetValueList
set_last_id_method = portal.IdTool_zSetLastId
id_group_done = []
# Save the last id of persistent dict if it is higher that
# the last id stored in the sql table
for line in get_value_list().dictionaries():
for line in self._getValueListFromTable():
id_group = line['id_group']
last_id = line['last_id']
if self.last_max_id_dict.has_key(id_group) and \
......@@ -237,9 +236,8 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
if self.getStoredInZodb():
self._updateSqlTable()
# Return values from sql
get_value_list = portal.IdTool_zGetValueList
return dict([(line['id_group'],int(line['last_id'])) for line in
get_value_list().dictionaries()])
self._getValueListFromTable()])
security.declareProtected(Permissions.ModifyPortalContent,
'importGeneratorIdDict')
......@@ -283,26 +281,53 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
portal.IdTool_zCreateEmptyTable()
self._updateSqlTable()
def _getValueListFromTable(self):
"""
get all the records of portal_ids table
returns list of id_dict. like [{'id_group', 'last_id'},..]
TODO: This method which is used in _updateSqlTable() still is not
scalable when portal_ids has a large amount of records.
If split into several transaction is acceptable, you can scale
it like updateLastMaxIdDictFromTable() do with the id_group parameter.
"""
portal = self.getPortalObject()
value_dict_list = []
id_group = None
while True:
record_list = portal.IdTool_zGetValueList(
id_group=id_group).dictionaries()
value_dict_list.extend(record_list)
if record_list:
id_group = record_list[-1]['id_group']
else:
break
return value_dict_list
security.declareProtected(Permissions.ModifyPortalContent,
'updateLastMaxIdDictFromTable')
def updateLastMaxIdDictFromTable(self):
def updateLastMaxIdDictFromTable(self, id_group=None):
"""
Update the Persistent id_dict from portal_ids table.
Update the Persistent id_dict from portal_ids table
in steps of the max_rows quantity of IdTool_getValueList ZSQL Method.
The quantity is currently configured 1000. This means update 1000
keys as the max in one call.
Returns the last id_group value that is updated in the call.
Warning: IdToool_zGetValueList ZSQL method retrieve all the records of
portal_ids table. So this method neither does not scale well if you have
millions id_group.
-- id_group: update the id_dict from this value by alphabetial sort
"""
portal = self.getPortalObject()
get_value_list = portal.IdTool_zGetValueList
new_id_dict = dict()
for line in get_value_list().dictionaries():
id_group = line['id_group']
last_id = line['last_id']
if isinstance(last_id, int) or isinstance(last_id, long):
new_id_dict[id_group] = ScalarMaxConflictResolver(last_id)
last_max_id_dict = self.last_max_id_dict
if last_max_id_dict is None:
self.last_max_id_dict = last_max_id_dict = PersistentMapping()
last_id_group = None
for line in portal.IdTool_zGetValueList(id_group=id_group):
last_id_group = id_group = line[0]
last_id = line[1]
try:
scalar = last_max_id_dict[id_group]
except KeyError:
last_max_id_dict[id_group] = ScalarMaxConflictResolver(last_id)
else:
raise TypeError, 'the value in the dictionary given is not a number'
if self.last_max_id_dict is None:
self.last_max_id_dict = PersistentMapping()
self.last_max_id_dict.update(new_id_dict)
scalar.set(last_id)
return last_id_group
......@@ -6,6 +6,49 @@
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>name</string> </key>
<value> <string>id_group</string> </value>
</item>
<item>
<key> <string>null</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>t</string> </value>
</item>
<item>
<key> <string>width</string> </key>
<value> <int>84</int> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>name</string> </key>
<value> <string>last_id</string> </value>
</item>
<item>
<key> <string>null</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>l</string> </value>
</item>
<item>
<key> <string>width</string> </key>
<value> <int>7</int> </value>
</item>
</dictionary>
</list>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
......@@ -14,7 +57,7 @@
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
<value> <string>id_group</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
......@@ -46,11 +89,17 @@
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>0</int> </value>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string>select id_group, last_id from portal_ids</string> </value>
<value> <string encoding="cdata"><![CDATA[
select id_group, last_id from portal_ids \n
<dtml-if id_group>where id_group > "<dtml-var id_group>"</dtml-if>\n
order by id_group
]]></string> </value>
</item>
<item>
<key> <string>title</string> </key>
......
......@@ -31,6 +31,7 @@
import unittest
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.tests.utils import createZODBPythonScript
from _mysql_exceptions import ProgrammingError
class TestIdTool(ERP5TypeTestCase):
......@@ -330,6 +331,61 @@ class TestIdTool(ERP5TypeTestCase):
id_dict = self.getLastGenerator(id_generator).exportGeneratorIdDict()
self.assertEquals(id_dict['07'], 6)
def test_08_updateLastMaxIdDictFromTable(self):
"""
Check that it can update id_dict persistent object from portal_ids table.
Also, check that it can update more than 1000 keys.
"""
id_generator = 'test_application_sql'
sql_generator = self.getLastGenerator(id_generator)
sql_generator.setStoredInZodb(False)
self.assertEquals(0, self.id_tool.generateNewId(id_generator=id_generator,
id_group='A-08'))
self.assertEquals(1, self.id_tool.generateNewId(id_generator=id_generator,
id_group='A-08'))
self.assertEquals(2, self.id_tool.generateNewId(id_generator=id_generator,
id_group='A-08'))
self.assertEquals(0, self.id_tool.generateNewId(id_generator=id_generator,
id_group='B-08'))
self.assertEquals(1, self.id_tool.generateNewId(id_generator=id_generator,
id_group='B-08'))
A_LOT_OF_KEY = 2500
var_id = 'C-%04d'
for x in xrange(A_LOT_OF_KEY):
self.assertEquals(0, self.id_tool.generateNewId(id_generator=id_generator,
id_group=var_id % x))
# test before update
self.assertEquals(None, sql_generator.last_max_id_dict.get('A-08'))
self.assertEquals(None, sql_generator.last_max_id_dict.get('B-08'))
for x in xrange(A_LOT_OF_KEY):
self.assertEquals(None, sql_generator.last_max_id_dict.get(var_id % x))
createZODBPythonScript(
self.portal.portal_skins.custom,
'IdTool_updateLastMaxId',
'last_id_group=None',
r"""
id_tool = context.getPortalObject().portal_ids
sql_generator = id_tool.searchFolder(
reference='test_sql_non_continuous_increasing')[0].getLatestVersionValue()
new_last_id_group = sql_generator.updateLastMaxIdDictFromTable(last_id_group)
if new_last_id_group is not None:
getattr(context.activate(), script.id)(new_last_id_group)
""")
# update dict from sql with activities
sql_generator.IdTool_updateLastMaxId()
self.tic()
# asserts
self.assertEquals(2, sql_generator.last_max_id_dict['A-08'].value)
self.assertEquals(1, sql_generator.last_max_id_dict['B-08'].value)
for x in xrange(A_LOT_OF_KEY):
self.assertEquals(0, sql_generator.last_max_id_dict[var_id % x].value)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestIdTool))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment