Commit 4f2f9487 by Vincent Pelletier

ERP5Type.Base: Implement decentralised in-ZODB id generator.

Provide a way to generate ids on specific documents when they make sense.
Reduces the risk of conflicts that portal_ids's in-ZODB id generator
creates.
Reset generator state on any copy (both copy/pase and clone), but not on
move (cut/paste and setId).
parent c5cd1820
......@@ -34,7 +34,7 @@ class IIdGenerator(Interface):
Rounding tool interface
"""
def generateNewId(id_group=None, default=None):
def generateNewId(id_group=None, default=None, poison=False):
"""
Generate the next id in the sequence of ids of a particular group
......@@ -51,9 +51,15 @@ class IIdGenerator(Interface):
If the default value is incompatible with the generator,
ValueError will be raised.
poison (bool)
If True, return the next id in requested sequence, and permanently break
that sequence's state, so that no new id may be successfuly generated
from it. Useful to ensure seamless migration away from this generator,
without risking a (few) late generation from happening after migration
code already moved sequence's state elsewhere.
"""
def generateNewIdList(id_group=None, default=None, id_count=1):
def generateNewIdList(id_group=None, default=None, id_count=1, poison=False):
"""
Generate a list of next ids in the sequence of ids of a particular group
......@@ -75,6 +81,13 @@ class IIdGenerator(Interface):
method should take as parameter the previously generated
id (optional). By default, ids are managed like integers and
are increased one by one
poison (bool)
If True, return the next id in requested sequence, and permanently break
that sequence's state, so that no new id may be successfuly generated
from it. Useful to ensure seamless migration away from this generator,
without risking a (few) late generation from happening after migration
code already moved sequence's state elsewhere.
"""
def initializeGenerator():
......
......@@ -34,7 +34,7 @@ class IIdTool(Interface):
Id Tool interface
"""
def generateNewId(id_group=None, default=None, id_generator=None):
def generateNewId(id_group=None, default=None, id_generator=None, poison=False):
"""
Generate the next id in the sequence of ids of a particular group
......@@ -56,6 +56,13 @@ class IIdTool(Interface):
reference. This is not mandatory, a default generator will exist.
Only id generator of type application can be selected.
poison (bool)
If True, return the next id in requested sequence, and permanently break
that sequence's state, so that no new id may be successfuly generated
from it. Useful to ensure seamless migration away from this generator,
without risking a (few) late generation from happening after migration
code already moved sequence's state elsewhere.
Example :
my_new_id = portal_ids.generateNewId(id_group='sale_invoice',
default=100)
......@@ -63,7 +70,7 @@ class IIdTool(Interface):
"""
def generateNewIdList(id_group=None, default=None, id_count=1,
id_generator=None):
id_generator=None, poison=False):
"""
Generate a list of next ids in the sequence of ids of a particular group
......@@ -85,6 +92,13 @@ class IIdTool(Interface):
reference. This is not mandatory, a default generator will exist.
Only id generator of type application can be selected.
poison (bool)
If True, return the next id in requested sequence, and permanently break
that sequence's state, so that no new id may be successfuly generated
from it. Useful to ensure seamless migration away from this generator,
without risking a (few) late generation from happening after migration
code already moved sequence's state elsewhere.
Example :
my_new_id_list = portal_ids.generateNewIdList(id_group='sale_invoice',
default=100, id_count=3)
......
......@@ -423,6 +423,66 @@ if new_last_id_group is not None:
for x in xrange(A_LOT_OF_KEY):
self.assertEqual(0, sql_generator.last_max_id_dict[var_id % x].value)
def test_decentralised_ZODB_id_generator(self):
"""
Check decentralised ID generator API, and migrating to it from portal_ids.
"""
portal = self.portal
portal_ids = portal.portal_ids
container_container = portal.portal_simulation
container = container_container.newContent()
old_id_group = str((
'test_decentralised_ZODB_id_generator',
container.getPath(),
))
new_id_group = 'test_decentralised_ZODB_id_generator'
latest_id_old_generator, = portal_ids.generateNewIdList(
id_group=old_id_group,
id_generator='zodb_continuous_increasing',
default=5,
)
# Test-internal sanity check
assert latest_id_old_generator == 5
first_id_new_generator, = container.generateIdList(
group=new_id_group,
onMissing=lambda: portal_ids.generateNewIdList(
id_group=old_id_group,
id_generator='zodb_continuous_increasing',
default=5,
poison=True,
)[0],
)
# migration is smealess
self.assertEqual(latest_id_old_generator + 1, first_id_new_generator)
# old generator is poisoned
self.assertRaises(
TypeError,
portal_ids.generateNewIdList,
id_group=old_id_group,
id_generator='zodb_continuous_increasing',
)
# copying container clears sequence state
copied_container_id_dict, = container_container.manage_pasteObjects(
container_container.manage_copyObjects([container.getId()]),
)
self.assertEqual(container_container[copied_container_id_dict['new_id']
].generateIdList(new_id_group), [1])
# cloning container clears sequence state
self.assertEqual(container.Base_createCloneDocument(batch_mode=1,
).generateIdList(new_id_group), [1])
# renaming container does not clear sequence state
container.setId(container.getId() + '_new_name')
self.assertEqual(
container.generateIdList(new_id_group),
[first_id_new_generator + 1],
)
# cutting container does not clear sequence state
cut_container_id_dict, = container_container.manage_pasteObjects(
container_container.manage_cutObjects([container.getId()]),
)
self.assertEqual(container_container[cut_container_id_dict['new_id']
].generateIdList(new_id_group), [first_id_new_generator + 2])
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestIdTool))
......
......@@ -33,6 +33,7 @@ import warnings
import types
import thread, threading
from BTrees.OOBTree import OOBTree
from Products.ERP5Type.Globals import InitializeClass, DTMLFile
from AccessControl import ClassSecurityInfo
from AccessControl.Permission import pname, Permission
......@@ -44,6 +45,7 @@ from DateTime import DateTime
import OFS.History
from OFS.SimpleItem import SimpleItem
from OFS.PropertyManager import PropertyManager
from persistent import Persistent
from persistent.TimeStamp import TimeStamp
from zExceptions import NotFound, Unauthorized
......@@ -106,6 +108,24 @@ from zLOG import LOG, INFO, ERROR, WARNING
_MARKER = []
class PersistentContainer(Persistent):
"""
Hold a value, making it persistent independently from its container, and
allowing in-place modification (useful for immutable).
Does not do any magic, so code using this is well aware of what is the
container and what is its content, and does not risk leaking one when
intending to provide the other.
"""
__slots__ = ('value', )
def __init__(self, value):
self.value = value
def __getstate__(self):
return self.value
def __setstate__(self, state):
self.value = state
global registered_workflow_method_set
wildcard_interaction_method_id_match = re.compile(r'[[.?*+{(\\]').search
workflow_method_registry = [] # XXX A set() would be better but would require a hash in WorkflowMethod class
......@@ -735,6 +755,8 @@ class Base( CopyContainer,
aq_method_generating = []
aq_portal_type = {}
aq_related_generated = 0
# Only generateIdList may look at this property. Anything else is unsafe.
_id_generator_state = None
# Declarative security - in ERP5 we use AccessContentsInformation to
# define the right of accessing content properties as opposed
......@@ -3464,6 +3486,84 @@ class Base( CopyContainer,
return new_document
def _postCopy(self, container, op=0):
super(Base, self)._postCopy(container, op=op)
if op == 0: # copy (not cut)
# We are the copy of another document (either cloned or copy/pasted),
# forget id generator state.
try:
del self._id_generator_state
except AttributeError:
pass
security.declareProtected(Permissions.AccessContentsInformation, 'generateIdList')
def generateIdList(self, group, count=1, default=1, onMissing=None, poison=False):
"""
Manages multiple independent sequences of unique numbers.
Each sequence guarantees the unicity of each produced value within that
sequence and for <self> instance, and is monotonously increasing by 1 for
each generated id.
group (string):
Identifies the sequence to use.
count (int):
How many identifiers to generate.
default (int):
If the sequence for given <group> did not already exist, initialise it at
this for the first generated value.
onMissing (callable):
If provided, called when requested sequence is missing, "default" is
ignored and the value returned by this function is used instead.
Allows seamless migration from another id generator *if* that id
generator is able to "poison the land" (see next option).
poison (bool):
If True, return the next id in requested sequence, and permanently break
that sequence's state, so that no new id may be successfuly generated
from it. Useful to ensure seamless migration away from this generator,
without risking a (few) late generation from happening after migration
code already moved sequence's state elsewhere.
Once a sequence has been poisoned, attempting to generate a new value
from it will raise an exception (exception type may vary).
"count" must be 1, otherwise a ValueError is raised.
Conflicts & guarantees:
- If multiple transactions modify the same sequence, ALL BUT ONE get a
ConflictError. This is by design, to achieve per-sequence unicity.
- If multiple transactions modify different sequences, NONE will get a
ConflictError (each sequence state is a separate persistent object).
- If multiple transactions create new sequences, SOME may get a
ConflictError. This is a limitation of the chosen data structure.
It is expected that group creation is a rare event, very unlikely to
happen concurrently in multiple transactions on the same object.
"""
if not isinstance(group, basestring):
raise TypeError('group must be a string')
if not isinstance(default, (int, long)):
raise TypeError('default must be an integer')
if not isinstance(count, (int, long)):
raise TypeError('count must be an integer')
if count < 0:
raise ValueError('count cannot be negative')
if poison and count != 1:
raise ValueError('sequence generator poisoning requires count=1')
if count == 0:
return []
id_generator_state = self._id_generator_state
if id_generator_state is None:
id_generator_state = self._id_generator_state = OOBTree()
try:
next_id = id_generator_state[group].value
except KeyError:
if onMissing is not None:
default = onMissing()
if not isinstance(default, (int, long)):
raise TypeError('onMissing must return an integer')
id_generator_state[group] = PersistentContainer(default)
next_id = default
new_next_id = None if poison else next_id + count
id_generator_state[group].value = new_next_id
return range(next_id, new_next_id)
InitializeClass(Base)
from Products.CMFCore.interfaces import IContentish
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment