Make id dict updating more scalable.

- Avoid to create too much OIDs when updateLastMaxIdDictFromTable
- Add a parameter 'id_group' into IdTool_zGetValueList for the
  scalability of updateLastMaxIdDictFromTable().
  This parameter is intended to use with CMFActivity
  (see TestIdTool.test_08_updateLastMaxIdDictFromTable).

product/ERP5/Document/SQLNonContinuousIncreasingIdGenerator.py

@@ -121,12 +121,11 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
       Update the portal ids table with the data of persistent dictionary
     """
     portal = self.getPortalObject()
-    get_value_list = portal.IdTool_zGetValueList
     set_last_id_method = portal.IdTool_zSetLastId
     id_group_done = []
     # Save the last id of persistent dict if it is higher that
     # the last id stored in the sql table
-    for line in get_value_list().dictionaries():
+    for line in self._getValueListFromTable():
       id_group = line['id_group']
       last_id = line['last_id']
       if self.last_max_id_dict.has_key(id_group) and \
@@ -237,9 +236,8 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
     if self.getStoredInZodb(): 
       self._updateSqlTable()
     # Return values from sql 
-    get_value_list = portal.IdTool_zGetValueList
     return dict([(line['id_group'],int(line['last_id'])) for line in
-      get_value_list().dictionaries()])
+      self._getValueListFromTable()])
 
   security.declareProtected(Permissions.ModifyPortalContent,
       'importGeneratorIdDict')
@@ -283,26 +281,53 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
     portal.IdTool_zCreateEmptyTable()
     self._updateSqlTable()
 
+  def _getValueListFromTable(self):
+    """
+      get all the records of portal_ids table
+      returns list of id_dict. like [{'id_group', 'last_id'},..]
+
+      TODO: This method which is used in _updateSqlTable() still is not
+      scalable when portal_ids has a large amount of records.
+      If split into several transaction is acceptable, you can scale
+      it like updateLastMaxIdDictFromTable() do with the id_group parameter.
+    """
+    portal = self.getPortalObject()
+    value_dict_list = []
+    id_group = None
+    while True:
+      record_list = portal.IdTool_zGetValueList(
+                       id_group=id_group).dictionaries()
+      value_dict_list.extend(record_list)
+      if record_list:
+        id_group = record_list[-1]['id_group']
+      else:
+        break
+    return value_dict_list
+
   security.declareProtected(Permissions.ModifyPortalContent,
        'updateLastMaxIdDictFromTable')
-  def updateLastMaxIdDictFromTable(self):
+  def updateLastMaxIdDictFromTable(self, id_group=None):
     """
-      Update the Persistent id_dict from portal_ids table.
+      Update the Persistent id_dict from portal_ids table
+      in steps of the max_rows quantity of IdTool_getValueList ZSQL Method.
+      The quantity is currently configured 1000. This means update 1000
+      keys as the max in one call.
+      Returns the last id_group value that is updated in the call.
 
-      Warning: IdToool_zGetValueList ZSQL method retrieve all the records of
-      portal_ids table. So this method neither does not scale well if you have
-      millions id_group.
+    -- id_group: update the id_dict from this value by alphabetial sort
     """
     portal = self.getPortalObject()
-    get_value_list = portal.IdTool_zGetValueList
-    new_id_dict = dict()
-    for line in get_value_list().dictionaries():
-      id_group = line['id_group']
-      last_id = line['last_id']
-      if isinstance(last_id, int) or isinstance(last_id, long):
-        new_id_dict[id_group] = ScalarMaxConflictResolver(last_id)
+    last_max_id_dict = self.last_max_id_dict
+    if last_max_id_dict is None:
+      self.last_max_id_dict = last_max_id_dict = PersistentMapping()
+    last_id_group = None
+    for line in portal.IdTool_zGetValueList(id_group=id_group):
+      last_id_group = id_group = line[0]
+      last_id = line[1]
+      try:
+        scalar = last_max_id_dict[id_group]
+      except KeyError:
+        last_max_id_dict[id_group] = ScalarMaxConflictResolver(last_id)
       else:
-        raise TypeError, 'the value in the dictionary given is not a number'
-    if self.last_max_id_dict is None:
-      self.last_max_id_dict = PersistentMapping()
-    self.last_max_id_dict.update(new_id_dict)
+        scalar.set(last_id)
+    return last_id_group

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/IdTool_zGetValueList.xml

@@ -6,6 +6,49 @@
     </pickle>
     <pickle>
       <dictionary>
+        <item>
+            <key> <string>_col</string> </key>
+            <value>
+              <list>
+                <dictionary>
+                  <item>
+                      <key> <string>name</string> </key>
+                      <value> <string>id_group</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>null</string> </key>
+                      <value> <int>0</int> </value>
+                  </item>
+                  <item>
+                      <key> <string>type</string> </key>
+                      <value> <string>t</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>width</string> </key>
+                      <value> <int>84</int> </value>
+                  </item>
+                </dictionary>
+                <dictionary>
+                  <item>
+                      <key> <string>name</string> </key>
+                      <value> <string>last_id</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>null</string> </key>
+                      <value> <int>1</int> </value>
+                  </item>
+                  <item>
+                      <key> <string>type</string> </key>
+                      <value> <string>l</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>width</string> </key>
+                      <value> <int>7</int> </value>
+                  </item>
+                </dictionary>
+              </list>
+            </value>
+        </item>
         <item>
             <key> <string>allow_simple_one_argument_traversal</string> </key>
             <value>
@@ -14,7 +57,7 @@
         </item>
         <item>
             <key> <string>arguments_src</string> </key>
-            <value> <string></string> </value>
+            <value> <string>id_group</string> </value>
         </item>
         <item>
             <key> <string>cache_time_</string> </key>
@@ -46,11 +89,17 @@
         </item>
         <item>
             <key> <string>max_rows_</string> </key>
-            <value> <int>0</int> </value>
+            <value> <int>1000</int> </value>
         </item>
         <item>
             <key> <string>src</string> </key>
-            <value> <string>select id_group, last_id from portal_ids</string> </value>
+            <value> <string encoding="cdata"><![CDATA[
+
+select id_group, last_id from portal_ids \n
+  <dtml-if id_group>where id_group > "<dtml-var id_group>"</dtml-if>\n
+  order by id_group
+
+]]></string> </value>
         </item>
         <item>
             <key> <string>title</string> </key>

product/ERP5/bootstrap/erp5_core/bt/revision

-41043
\ No newline at end of file
+41044

product/ERP5/tests/testIdTool.py

@@ -31,6 +31,7 @@
 import unittest
 
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
+from Products.ERP5Type.tests.utils import createZODBPythonScript
 from _mysql_exceptions import ProgrammingError
 
 class TestIdTool(ERP5TypeTestCase):
@@ -330,6 +331,61 @@ class TestIdTool(ERP5TypeTestCase):
     id_dict = self.getLastGenerator(id_generator).exportGeneratorIdDict()
     self.assertEquals(id_dict['07'], 6)
 
+  def test_08_updateLastMaxIdDictFromTable(self):
+    """
+     Check that it can update id_dict persistent object from portal_ids table.
+     Also, check that it can update more than 1000 keys.
+    """
+    id_generator = 'test_application_sql'
+    sql_generator = self.getLastGenerator(id_generator)
+    sql_generator.setStoredInZodb(False)
+    self.assertEquals(0, self.id_tool.generateNewId(id_generator=id_generator,
+                                                    id_group='A-08'))
+    self.assertEquals(1, self.id_tool.generateNewId(id_generator=id_generator,
+                                                    id_group='A-08'))
+    self.assertEquals(2, self.id_tool.generateNewId(id_generator=id_generator,
+                                                    id_group='A-08'))
+    self.assertEquals(0, self.id_tool.generateNewId(id_generator=id_generator,
+                                                    id_group='B-08'))
+    self.assertEquals(1, self.id_tool.generateNewId(id_generator=id_generator,
+                                                    id_group='B-08'))
+
+    A_LOT_OF_KEY = 2500
+    var_id = 'C-%04d'
+    for x in xrange(A_LOT_OF_KEY):
+      self.assertEquals(0, self.id_tool.generateNewId(id_generator=id_generator,
+                                                      id_group=var_id % x))
+
+    # test before update
+    self.assertEquals(None, sql_generator.last_max_id_dict.get('A-08'))
+    self.assertEquals(None, sql_generator.last_max_id_dict.get('B-08'))
+    for x in xrange(A_LOT_OF_KEY):
+      self.assertEquals(None, sql_generator.last_max_id_dict.get(var_id % x))
+    createZODBPythonScript(
+      self.portal.portal_skins.custom,
+      'IdTool_updateLastMaxId',
+      'last_id_group=None',
+r"""
+id_tool = context.getPortalObject().portal_ids
+sql_generator = id_tool.searchFolder(
+  reference='test_sql_non_continuous_increasing')[0].getLatestVersionValue()
+
+new_last_id_group = sql_generator.updateLastMaxIdDictFromTable(last_id_group)
+if new_last_id_group is not None:
+  getattr(context.activate(), script.id)(new_last_id_group)
+""")
+
+    # update dict from sql with activities
+    sql_generator.IdTool_updateLastMaxId()
+
+    self.tic()
+
+    # asserts
+    self.assertEquals(2, sql_generator.last_max_id_dict['A-08'].value)
+    self.assertEquals(1, sql_generator.last_max_id_dict['B-08'].value)
+    for x in xrange(A_LOT_OF_KEY):
+      self.assertEquals(0, sql_generator.last_max_id_dict[var_id % x].value)
+
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestIdTool))