ERP5Type.Base: Implement decentralised in-ZODB id generator.

Provide a way to generate ids on specific documents when they make sense.
Reduces the risk of conflicts that portal_ids's in-ZODB id generator
creates.
Reset generator state on any copy (both copy/pase and clone), but not on
move (cut/paste and setId).

product/ERP5/interfaces/id_generator.py

@@ -34,7 +34,7 @@ class IIdGenerator(Interface):
   Rounding tool interface
   """
 
-  def generateNewId(id_group=None, default=None):
+  def generateNewId(id_group=None, default=None, poison=False):
     """
     Generate the next id in the sequence of ids of a particular group
 
@@ -51,9 +51,15 @@ class IIdGenerator(Interface):
        If the default value is incompatible with the generator,
        ValueError will be raised.
 
+    poison (bool)
+      If True, return the next id in requested sequence, and permanently break
+      that sequence's state, so that no new id may be successfuly generated
+      from it. Useful to ensure seamless migration away from this generator,
+      without risking a (few) late generation from happening after migration
+      code already moved sequence's state elsewhere.
     """
 
-  def generateNewIdList(id_group=None, default=None, id_count=1):
+  def generateNewIdList(id_group=None, default=None, id_count=1, poison=False):
     """
     Generate a list of next ids in the sequence of ids of a particular group
 
@@ -75,6 +81,13 @@ class IIdGenerator(Interface):
        method should take as parameter the previously generated
        id (optional). By default, ids are managed like integers and
        are increased one by one
+
+    poison (bool)
+      If True, return the next id in requested sequence, and permanently break
+      that sequence's state, so that no new id may be successfuly generated
+      from it. Useful to ensure seamless migration away from this generator,
+      without risking a (few) late generation from happening after migration
+      code already moved sequence's state elsewhere.
     """
 
   def initializeGenerator():

product/ERP5/interfaces/id_tool.py

@@ -34,7 +34,7 @@ class IIdTool(Interface):
   Id Tool interface
   """
 
-  def generateNewId(id_group=None, default=None, id_generator=None):
+  def generateNewId(id_group=None, default=None, id_generator=None, poison=False):
     """
     Generate the next id in the sequence of ids of a particular group
 
@@ -56,6 +56,13 @@ class IIdTool(Interface):
        reference. This is not mandatory, a default generator will exist.
        Only id generator of type application can be selected.
 
+    poison (bool)
+      If True, return the next id in requested sequence, and permanently break
+      that sequence's state, so that no new id may be successfuly generated
+      from it. Useful to ensure seamless migration away from this generator,
+      without risking a (few) late generation from happening after migration
+      code already moved sequence's state elsewhere.
+
     Example :
        my_new_id = portal_ids.generateNewId(id_group='sale_invoice',
                               default=100)
@@ -63,7 +70,7 @@ class IIdTool(Interface):
     """
 
   def generateNewIdList(id_group=None, default=None, id_count=1,
-      id_generator=None):
+      id_generator=None, poison=False):
     """
     Generate a list of next ids in the sequence of ids of a particular group
 
@@ -85,6 +92,13 @@ class IIdTool(Interface):
        reference. This is not mandatory, a default generator will exist.
        Only id generator of type application can be selected.
 
+    poison (bool)
+      If True, return the next id in requested sequence, and permanently break
+      that sequence's state, so that no new id may be successfuly generated
+      from it. Useful to ensure seamless migration away from this generator,
+      without risking a (few) late generation from happening after migration
+      code already moved sequence's state elsewhere.
+
     Example :
        my_new_id_list = portal_ids.generateNewIdList(id_group='sale_invoice',
                               default=100, id_count=3)

product/ERP5/tests/testIdTool.py

@@ -423,6 +423,66 @@ if new_last_id_group is not None:
     for x in xrange(A_LOT_OF_KEY):
       self.assertEqual(0, sql_generator.last_max_id_dict[var_id % x].value)
 
+  def test_decentralised_ZODB_id_generator(self):
+    """
+    Check decentralised ID generator API, and migrating to it from portal_ids.
+    """
+    portal = self.portal
+    portal_ids = portal.portal_ids
+    container_container = portal.portal_simulation
+    container = container_container.newContent()
+    old_id_group = str((
+      'test_decentralised_ZODB_id_generator',
+      container.getPath(),
+    ))
+    new_id_group = 'test_decentralised_ZODB_id_generator'
+    latest_id_old_generator, = portal_ids.generateNewIdList(
+      id_group=old_id_group,
+      id_generator='zodb_continuous_increasing',
+      default=5,
+    )
+    # Test-internal sanity check
+    assert latest_id_old_generator == 5
+    first_id_new_generator, = container.generateIdList(
+      group=new_id_group,
+      onMissing=lambda: portal_ids.generateNewIdList(
+        id_group=old_id_group,
+        id_generator='zodb_continuous_increasing',
+        default=5,
+        poison=True,
+      )[0],
+    )
+    # migration is smealess
+    self.assertEqual(latest_id_old_generator + 1, first_id_new_generator)
+    # old generator is poisoned
+    self.assertRaises(
+      TypeError,
+      portal_ids.generateNewIdList,
+      id_group=old_id_group,
+      id_generator='zodb_continuous_increasing',
+    )
+    # copying container clears sequence state
+    copied_container_id_dict, = container_container.manage_pasteObjects(
+      container_container.manage_copyObjects([container.getId()]),
+    )
+    self.assertEqual(container_container[copied_container_id_dict['new_id']
+      ].generateIdList(new_id_group), [1])
+    # cloning container clears sequence state
+    self.assertEqual(container.Base_createCloneDocument(batch_mode=1,
+      ).generateIdList(new_id_group), [1])
+    # renaming container does not clear sequence state
+    container.setId(container.getId() + '_new_name')
+    self.assertEqual(
+      container.generateIdList(new_id_group),
+      [first_id_new_generator + 1],
+    )
+    # cutting container does not clear sequence state
+    cut_container_id_dict, = container_container.manage_pasteObjects(
+      container_container.manage_cutObjects([container.getId()]),
+    )
+    self.assertEqual(container_container[cut_container_id_dict['new_id']
+      ].generateIdList(new_id_group), [first_id_new_generator + 2])
+
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestIdTool))

product/ERP5Type/Base.py

@@ -33,6 +33,7 @@ import warnings
 import types
 import thread, threading
 
+from BTrees.OOBTree import OOBTree
 from Products.ERP5Type.Globals import InitializeClass, DTMLFile
 from AccessControl import ClassSecurityInfo
 from AccessControl.Permission import pname, Permission
@@ -44,6 +45,7 @@ from DateTime import DateTime
 import OFS.History
 from OFS.SimpleItem import SimpleItem
 from OFS.PropertyManager import PropertyManager
+from persistent import Persistent
 from persistent.TimeStamp import TimeStamp
 from zExceptions import NotFound, Unauthorized
 
@@ -106,6 +108,24 @@ from zLOG import LOG, INFO, ERROR, WARNING
 
 _MARKER = []
 
+class PersistentContainer(Persistent):
+    """
+    Hold a value, making it persistent independently from its container, and
+    allowing in-place modification (useful for immutable).
+    Does not do any magic, so code using this is well aware of what is the
+    container and what is its content, and does not risk leaking one when
+    intending to provide the other.
+    """
+    __slots__ = ('value', )
+    def __init__(self, value):
+        self.value = value
+
+    def __getstate__(self):
+        return self.value
+
+    def __setstate__(self, state):
+        self.value = state
+
 global registered_workflow_method_set
 wildcard_interaction_method_id_match = re.compile(r'[[.?*+{(\\]').search
 workflow_method_registry = [] # XXX A set() would be better but would require a hash in WorkflowMethod class
@@ -735,6 +755,8 @@ class Base( CopyContainer,
   aq_method_generating = []
   aq_portal_type = {}
   aq_related_generated = 0
+  # Only generateIdList may look at this property. Anything else is unsafe.
+  _id_generator_state = None
 
   # Declarative security - in ERP5 we use AccessContentsInformation to
   # define the right of accessing content properties as opposed
@@ -3464,6 +3486,84 @@ class Base( CopyContainer,
 
     return new_document
 
+  def _postCopy(self, container, op=0):
+    super(Base, self)._postCopy(container, op=op)
+    if op == 0: # copy (not cut)
+      # We are the copy of another document (either cloned or copy/pasted),
+      # forget id generator state.
+      try:
+        del self._id_generator_state
+      except AttributeError:
+        pass
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'generateIdList')
+  def generateIdList(self, group, count=1, default=1, onMissing=None, poison=False):
+    """
+    Manages multiple independent sequences of unique numbers.
+    Each sequence guarantees the unicity of each produced value within that
+    sequence and for <self> instance, and is monotonously increasing by 1 for
+    each generated id.
+
+    group (string):
+      Identifies the sequence to use.
+    count (int):
+      How many identifiers to generate.
+    default (int):
+      If the sequence for given <group> did not already exist, initialise it at
+      this for the first generated value.
+    onMissing (callable):
+      If provided, called when requested sequence is missing, "default" is
+      ignored and the value returned by this function is used instead.
+      Allows seamless migration from another id generator *if* that id
+      generator is able to "poison the land" (see next option).
+    poison (bool):
+      If True, return the next id in requested sequence, and permanently break
+      that sequence's state, so that no new id may be successfuly generated
+      from it. Useful to ensure seamless migration away from this generator,
+      without risking a (few) late generation from happening after migration
+      code already moved sequence's state elsewhere.
+      Once a sequence has been poisoned, attempting to generate a new value
+      from it will raise an exception (exception type may vary).
+      "count" must be 1, otherwise a ValueError is raised.
+
+    Conflicts & guarantees:
+    - If multiple transactions modify the same sequence, ALL BUT ONE get a
+      ConflictError. This is by design, to achieve per-sequence unicity.
+    - If multiple transactions modify different sequences, NONE will get a
+      ConflictError (each sequence state is a separate persistent object).
+    - If multiple transactions create new sequences, SOME may get a
+      ConflictError. This is a limitation of the chosen data structure.
+      It is expected that group creation is a rare event, very unlikely to
+      happen concurrently in multiple transactions on the same object.
+    """
+    if not isinstance(group, basestring):
+      raise TypeError('group must be a string')
+    if not isinstance(default, (int, long)):
+      raise TypeError('default must be an integer')
+    if not isinstance(count, (int, long)):
+      raise TypeError('count must be an integer')
+    if count < 0:
+      raise ValueError('count cannot be negative')
+    if poison and count != 1:
+      raise ValueError('sequence generator poisoning requires count=1')
+    if count == 0:
+      return []
+    id_generator_state = self._id_generator_state
+    if id_generator_state is None:
+      id_generator_state = self._id_generator_state = OOBTree()
+    try:
+      next_id = id_generator_state[group].value
+    except KeyError:
+      if onMissing is not None:
+        default = onMissing()
+        if not isinstance(default, (int, long)):
+          raise TypeError('onMissing must return an integer')
+      id_generator_state[group] = PersistentContainer(default)
+      next_id = default
+    new_next_id = None if poison else next_id + count
+    id_generator_state[group].value = new_next_id
+    return range(next_id, new_next_id)
+
 InitializeClass(Base)
 
 from Products.CMFCore.interfaces import IContentish