ERP5Type.Base: Implement decentralised in-ZODB id generator.

Provide a way to generate ids on specific documents when they make sense.
Reduces the risk of conflicts that portal_ids's in-ZODB id generator
creates.

product/ERP5/tests/testIdTool.py

@@ -423,6 +423,44 @@ if new_last_id_group is not None:
     for x in xrange(A_LOT_OF_KEY):
       self.assertEqual(0, sql_generator.last_max_id_dict[var_id % x].value)
 
+  def test_decentralised_ZODB_id_generator(self):
+    """
+    Check decentralised ID generator API, and migrating to it from portal_ids.
+    """
+    portal = self.portal
+    portal_ids = portal.portal_ids
+    # Note: in a real use-case this would be a terrible choice for an id
+    # generator container. This is just a test, so it should be fine here.
+    container = portal.portal_simulation
+    old_id_group = str((
+      'test_decentralised_ZODB_id_generator',
+      container.getPath(),
+    ))
+    new_id_group = 'test_decentralised_ZODB_id_generator'
+    latest_id_old_generator, = portal_ids.generateNewIdList(
+      id_group=old_id_group,
+      id_generator='zodb_continuous_increasing',
+      default=5,
+    )
+    first_id_new_generator, = container.generateIdList(
+      group=new_id_group,
+      onMissing=lambda: portal_ids.generateNewIdList(
+        id_group=old_id_group,
+        id_generator='zodb_continuous_increasing',
+        default=5,
+        poison=True,
+      )[0],
+    )
+    # migration is smealess
+    self.assertEqual(latest_id_old_generator + 1, first_id_new_generator)
+    # old generator is poisoned
+    self.assertRaises(
+        TypeError,
+        portal_ids.generateNewIdList,
+        id_group=old_id_group,
+        id_generator='zodb_continuous_increasing',
+    )
+
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestIdTool))

product/ERP5Type/Base.py

@@ -33,6 +33,7 @@ import warnings
 import types
 import thread, threading
 
+from BTrees.OOBTree import OOBTree
 from Products.ERP5Type.Globals import InitializeClass, DTMLFile
 from AccessControl import ClassSecurityInfo
 from AccessControl.Permission import pname, Permission
@@ -44,6 +45,7 @@ from DateTime import DateTime
 import OFS.History
 from OFS.SimpleItem import SimpleItem
 from OFS.PropertyManager import PropertyManager
+from persistent import Persistent
 from persistent.TimeStamp import TimeStamp
 from zExceptions import NotFound, Unauthorized
 
@@ -106,6 +108,18 @@ from zLOG import LOG, INFO, ERROR, WARNING
 
 _MARKER = []
 
+class PersistentContainer(Persistent):
+    """
+    Hold a value, making it persistent independently from its container, and
+    allowing in-place modification (useful for immutable).
+    Does not do any magic, so code using this is well aware of what is the
+    container and what is its content, and does not risk leaking one when
+    intending to provide the other.
+    """
+    __slots__ = ('value', )
+    def __init__(self, value):
+        self.value = value
+
 global registered_workflow_method_set
 wildcard_interaction_method_id_match = re.compile(r'[[.?*+{(\\]').search
 workflow_method_registry = [] # XXX A set() would be better but would require a hash in WorkflowMethod class
@@ -735,6 +749,8 @@ class Base( CopyContainer,
   aq_method_generating = []
   aq_portal_type = {}
   aq_related_generated = 0
+  # Only generateIdList may look at this property. Anything else is unsafe.
+  _id_generator_state = None
 
   # Declarative security - in ERP5 we use AccessContentsInformation to
   # define the right of accessing content properties as opposed
@@ -3464,6 +3480,74 @@ class Base( CopyContainer,
 
     return new_document
 
+  security.declareProtected(Permissions.ModifyPortalContent, 'generateIdList')
+  def generateIdList(self, group, count=1, default=1, onMissing=None, poison=False):
+    """
+    Manages multiple independent sequences of unique numbers.
+    Each sequence guarantees the unicity of each produced value within that
+    sequence and for <self> instance, and is monotonously increasing by 1 for
+    each generated id.
+
+    group (string):
+      Identifies the sequence to use.
+    count (int):
+      How many identifiers to generate.
+    default (int):
+      If the sequence for given <group> did not already exist, initialise it at
+      this for the first generated value.
+    onMissing (callable):
+      If provided, called when requested sequence is missing, "default" is
+      ignored and the value returned by this function is used instead.
+      Allows seamless migration from another id generator *if* that id
+      generator is able to "poison the land" (see next option).
+    poison (bool):
+      If True, return the next id in requested sequence, and permanently break
+      that sequence's state, so that no new id may be successfuly generated
+      from it. Useful to ensure seamless migration away from this generator,
+      without risking a (few) late generation from happening after migration
+      code already moved sequence's state elsewhere.
+      Once a sequence has been poisoned, attempting to generate a new value
+      from it will raise an exception (exception type may vary).
+      "count" must be 1, otherwise a ValueError is raised.
+
+    Conflicts & guarantees:
+    - If multiple transactions modify the same sequence, ALL BUT ONE get a
+      ConflictError. This is by design, to achieve per-sequence unicity.
+    - If multiple transactions modify different sequences, NONE will get a
+      ConflictError (each sequence state is a separate persistent object).
+    - If multiple transactions create new sequences, SOME may get a
+      ConflictError. This is a limitation of the chosen data structure.
+      It is expected that group creation is a rare event, very unlikely to
+      happen concurrently in multiple transactions on the same object.
+    """
+    if not isinstance(group, basestring):
+      raise TypeError('group must be a string')
+    if not isinstance(default, (int, long)):
+      raise TypeError('default must be an integer')
+    if not isinstance(count, (int, long)):
+      raise TypeError('count must be an integer')
+    if count < 0:
+      raise ValueError('count cannot be negative')
+    if poison and count != 1:
+      raise ValueError('sequence generator poisoning requires count=1')
+    if count == 0:
+      return []
+    id_generator_state = self._id_generator_state
+    if id_generator_state is None:
+      id_generator_state = self._id_generator_state = OOBTree()
+    try:
+      next_id = id_generator_state[group].value
+    except KeyError:
+      if onMissing is not None:
+        default = onMissing()
+        if not isinstance(default, (int, long)):
+          raise TypeError('onMissing must return an integer')
+      id_generator_state[group] = PersistentContainer(default)
+      next_id = default
+    new_next_id = None if poison else next_id + count
+    id_generator_state[group].value = new_next_id
+    return range(next_id, new_next_id)
+
 InitializeClass(Base)
 
 from Products.CMFCore.interfaces import IContentish