ERP5Type,CopySupport: Allow immediately indexing new subobjects

"immediate", in this context, means "during the same transaction".
Normally, indexation always happens in a transaction different from the
one which did the indexation-inducing action (modifying a property,
creating a document, explicitely requesting indexation). This is because
SQL and object databases do not have the same approach to conflict
resolution: in SQL, the last one wins, and ordering happens based on locks.
In ZODB, conflict resolution is stricter in that to modify an object
a transaction must have started with the same revision of that object as
the one which is current at the time it is trying to commit. As both
databases must be kept consistent, one interpretation must be enforced
onto the other: the ZODB interpretation. So delayed indexation, plus
careful activity sequencing (serialization_tag) is required.

But in very specific cases, it is actually safe to index a document
immediately: when creating that document. This is because the only
conflict which may then happen is if two transaction produce the same
path, and ZODB will prevent the transaction from committing altogether,
preventing any conflict resolution from happening. Pasting a document
falls into this category as well, for the same reason.

In turn, this feature removes the need to call "immediate" reindexation
methods, allowing to restrict their availability later and preventing
API misuse and catalog consistency compromission.

Two variants of "immediate" indexation are available:
- internal to the method which creates considered document
- delayed to a caller-controller, but mandatory, point later in current
  transaction, by using a context (in python sense) manager object.

product/ERP5Type/CopySupport.py

@@ -12,6 +12,7 @@
 #
 ##############################################################################
 
+from functools import partial
 from OFS import Moniker
 from zExceptions import BadRequest
 from AccessControl import ClassSecurityInfo, getSecurityManager
@@ -27,6 +28,7 @@ from Products.ERP5Type import Permissions
 from Acquisition import aq_base, aq_inner, aq_parent
 from Products.ERP5Type.Accessor.Constant import PropertyGetter as ConstantGetter
 from Products.ERP5Type.Globals import PersistentMapping, MessageDialog
+from Products.ERP5Type.ImmediateReindexContextManager import ImmediateReindexContextManager
 from Products.ERP5Type.Utils import get_request
 from Products.ERP5Type.Message import translateString
 from Products.CMFCore.WorkflowCore import WorkflowException
@@ -422,16 +424,17 @@ class CopyContainer:
                                          path_item_list=previous_path,
                                          new_id=self.id)
 
-  def _duplicate(self, cp, reindex_kw=None):
+  def _duplicate(self, cp, reindex_kw=None, immediate_reindex=False):
     _, result = self.__duplicate(
       cp,
       duplicate=True,
       is_indexable=None,
       reindex_kw=reindex_kw,
+      immediate_reindex=immediate_reindex,
     )
     return result
 
-  def __duplicate(self, cp, duplicate, is_indexable, reindex_kw):
+  def __duplicate(self, cp, duplicate, is_indexable, reindex_kw, immediate_reindex):
     try:
       cp = _cb_decode(cp)
     except:
@@ -506,6 +509,16 @@ class CopyContainer:
       if not set_owner:
         # try to make ownership implicit if possible
         new_ob.manage_changeOwnershipType(explicit=0)
+      method = new_ob.immediateReindexObject
+      if reindex_kw is not None:
+        method = partial(method, **reindex_kw)
+      if isinstance(immediate_reindex, ImmediateReindexContextManager):
+        immediate_reindex.append(method)
+      elif immediate_reindex:
+        # Immediately reindexing document that we just pasted is safe, as no
+        # other transaction can by definition see it, so there cannot be a race
+        # condition leading to stale catalog content.
+        method()
     return op, result
 
   def _postDuplicate(self):
@@ -534,7 +547,7 @@ class CopyContainer:
     self.isIndexable = ConstantGetter('isIndexable', value=False)
     self.__recurse('_setNonIndexable')
 
-  def manage_pasteObjects(self, cb_copy_data=None, is_indexable=None, reindex_kw=None, REQUEST=None):
+  def manage_pasteObjects(self, cb_copy_data=None, is_indexable=None, reindex_kw=None, immediate_reindex=False, REQUEST=None):
     """Paste previously copied objects into the current object.
 
     If calling manage_pasteObjects from python code, pass the result of a
@@ -543,6 +556,18 @@ class CopyContainer:
 
     If is_indexable is False, we will avoid indexing the pasted objects and
     subobjects
+
+    immediate_reindex (bool)
+      Immediately (=during current transaction) reindex created document, so
+      it is possible to find it in catalog before transaction ends.
+      Note: this does not apply to subobjects which may be created during
+      pasting. Only the topmost object will be immediately reindexed. Any
+      subobject will be reindexed later, using activities.
+
+      If a ImmediateReindexContextManager instance is given, a context (in
+      python sense) must have been entered with it, and indexation will
+      occur when that context is exited, allowing further changes before
+      first indexation (ex: workflow state change, property change).
     """
     cp = None
     if cb_copy_data is not None:
@@ -556,6 +581,7 @@ class CopyContainer:
       duplicate=False,
       is_indexable=is_indexable,
       reindex_kw=reindex_kw,
+      immediate_reindex=immediate_reindex,
     )
     if REQUEST is None:
       return result

product/ERP5Type/ERP5Type.py

@@ -21,6 +21,7 @@
 #
 ##############################################################################
 
+from functools import partial
 import zope.interface
 from Products.ERP5Type.Globals import InitializeClass
 from AccessControl import ClassSecurityInfo, getSecurityManager
@@ -34,6 +35,7 @@ from Products.ERP5Type import interfaces, Constraint, Permissions, PropertySheet
 from Products.ERP5Type.Base import getClassPropertyList
 from Products.ERP5Type.UnrestrictedMethod import UnrestrictedMethod
 from Products.ERP5Type.Utils import deprecated, createExpressionContext
+from Products.ERP5Type.ImmediateReindexContextManager import ImmediateReindexContextManager
 from Products.ERP5Type.XMLObject import XMLObject
 from Products.ERP5Type.Cache import CachingMethod
 from Products.ERP5Type.dynamic.accessor_holder import getPropertySheetValueList, \
@@ -42,6 +44,7 @@ from Products.ERP5Type.dynamic.accessor_holder import getPropertySheetValueList,
 ERP5TYPE_SECURITY_GROUP_ID_GENERATION_SCRIPT = 'ERP5Type_asSecurityGroupId'
 
 from TranslationProviderBase import TranslationProviderBase
+from Products.ERP5Type.Accessor.Constant import PropertyGetter as ConstantGetter
 from Products.ERP5Type.Accessor.Translation import TRANSLATION_DOMAIN_CONTENT_TRANSLATION
 from zLOG import LOG, ERROR
 from Products.CMFCore.exceptions import zExceptions_Unauthorized
@@ -359,11 +362,25 @@ class ERP5TypeInformation(XMLObject,
     def constructInstance(self, container, id, created_by_builder=0,
                           temp_object=0, compute_local_role=None,
                           notify_workflow=True, is_indexable=None,
-                          activate_kw=None, reindex_kw=None, **kw):
+                          activate_kw=None, reindex_kw=None,
+                          immediate_reindex=False, **kw):
       """
       Build a "bare" instance of the appropriate type in
       'container', using 'id' as its id.
       Call the init_script for the portal_type.
+
+      immediate_reindex (bool, ImmediateReindexContextManager)
+        Immediately (=during current transaction) reindex created document, so
+        it is possible to find it in catalog before transaction ends.
+        Note: this does not apply to subobjects which may be created during
+        document construction. Only the topmost object will be immediately
+        reindexed. Any subobject will be reindexed later, using activities.
+
+        If a ImmediateReindexContextManager instance is given, a context (in
+        python sense) must have been entered with it, and indexation will
+        occur when that context is exited, allowing further changes before
+        first indexation (ex: workflow state change, property change).
+
       Returns the object.
       """
       if compute_local_role is None:
@@ -426,6 +443,17 @@ class ERP5TypeInformation(XMLObject,
       if kw:
         ob._edit(force_update=1, **kw)
 
+      if not temp_object:
+        method = ob.immediateReindexObject
+        if reindex_kw is not None:
+          method = partial(method, **reindex_kw)
+        if isinstance(immediate_reindex, ImmediateReindexContextManager):
+          immediate_reindex.append(method)
+        elif immediate_reindex:
+          # Immediately reindexing document that we just created is safe, as no
+          # other transaction can by definition see it, so there cannot be a race
+          # condition leading to stale catalog content.
+          method()
       return ob
 
     def _getPropertyHolder(self):

product/ERP5Type/ImmediateReindexContextManager.py

+#############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from AccessControl import ClassSecurityInfo
+from Products.ERP5Type.Globals import InitializeClass
+
+class ImmediateReindexContextManager(object):
+  """
+  Immediately reindex given object(s) upon leaving context.
+
+  Pass an instance of this class as "immediate_reindex" argument on methods
+  having one to delay indexation a bit (ex: to let you change object state,
+  change some peroperties).
+
+  Example usage:
+  from Products.ERP5Type.ImmediateReindexContextManager import ImmediateReindexContextManager
+  with ImmediateReindexContextManager() as immediate_reindex_context_manager:
+    document = context.newContent(
+      immediate_reindex=immediate_reindex_context_manager,
+      ...
+    )
+    document.confirm()
+  # document will be indexed as already confirmed
+  """
+  security = ClassSecurityInfo()
+
+  def __init__(self):
+    # Detect and tolerate (but track) context nesting.
+    self.__context_stack = []
+
+  def __enter__(self):
+    self.__context_stack.append([])
+    return self
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    for method in self.__context_stack.pop():
+      method()
+
+  # Note: if you want to reuse this class, pay extra attention to security.
+  # It is critical that:
+  # - the class can be imported and instanciated from restricted python
+  # - "append" method cannot be called from anywhere but products
+  # - ImmediateReindexContextManager stay a class on its own (even if it
+  #   should become an empty subclass), only used for indexation-related
+  #   methods upon document creation.
+  # Otherwise, misuse will happen.
+  security.declarePrivate('append')
+  def append(self, method):
+    """
+    Queue indexation method for execution upon context exit.
+
+    May only be called by places which just bound document into its container,
+    like constructInstance or object paste handler.
+    DO NOT CALL THIS ANYWHERE ELSE !
+    """
+    try:
+      self.__context_stack[-1].append(method)
+    except IndexError:
+      raise RuntimeError(
+        'ImmediateReindexContextManager must be entered '
+        '(see "with" statement) before it can be used',
+      )
+
+InitializeClass(ImmediateReindexContextManager)

product/ERP5Type/__init__.py

@@ -175,6 +175,7 @@ allow_module('Products.ERP5Type.Error')
 allow_module('Products.ERP5Type.Errors')
 allow_module('Products.ERP5Type.JSONEncoder')
 allow_module('Products.ERP5Type.Log')
+allow_module('Products.ERP5Type.ImmediateReindexContextManager')
 ModuleSecurityInfo('Products.ERP5Type.JSON').declarePublic('dumps', 'loads')
 ModuleSecurityInfo('Products.ERP5Type.Constraint').declarePublic('PropertyTypeValidity')
 ModuleSecurityInfo('Products.ERP5Type.DiffUtils').declarePublic('DiffFile')