This big change optimizes the scheduling of active objects,

and fix some bugs.

The basic idea is to track a dependency graph to find executable
messages quickly. This makes the activity system far more efficient,
when you have many inter-dependent messages queued in the tables.

Also, this obsoletes the time shifting in the schedulers,
as executable messages can be found in a more efficient manner.
So the activity parameter "at_date" should work expectedly.

Now the API of validate methods in Activities return a
list of message objects instead of a boolean value. Such
a list contains messages that are depended upon by a given
message.

The validate method in Message accepts a new optional
parameter, check_order_validation, to indicate whether
order validation should be performed. The default behavior
has not changed.

getDependentMessageList is added to ActivityTool, Queue
and Message. This method collects dependent message for
a given message from all activities.

There are some other subtle changes. Look at the diffs for
more details.


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@14039 20353a03-c40f-0410-a6d1-a30d3c3de9de

product/CMFActivity/Activity/Queue.py

@@ -26,12 +26,17 @@
 #
 ##############################################################################
 
-import pickle, sys
-from Acquisition import aq_base
+import cPickle, sys
 from DateTime import DateTime
-from Products.CMFActivity.ActivityTool import Message
-from zLOG import LOG
+from zLOG import LOG, WARNING, ERROR
 from ZODB.POSException import ConflictError
+import sha
+from cStringIO import StringIO
+
+try:
+  from transaction import get as get_transaction
+except ImportError:
+  pass
 
 # Error values for message validation
 EXCEPTION      = -1
@@ -122,7 +127,7 @@ class Queue:
     self.is_awake[processing_node] = 0
     self.is_alive[processing_node] = 0
 
-  def validate(self, activity_tool, message, **kw):
+  def validate(self, activity_tool, message, check_order_validation=1, **kw):
     """
       This is the place where activity semantics is implemented
       **kw contains all parameters which allow to implement synchronisation,
@@ -145,22 +150,79 @@ class Queue:
     try:
       if activity_tool.unrestrictedTraverse(message.object_path, None) is None:
         # Do not try to call methods on objects which do not exist
-        LOG('WARNING ActivityTool', 0,
+        LOG('CMFActivity', WARNING,
            'Object %s does not exist' % '/'.join(message.object_path))
         return INVALID_PATH
-      for k, v in kw.items():
+      if check_order_validation:
+        for k, v in kw.iteritems():
           if activity_tool.validateOrder(message, k, v):
             return INVALID_ORDER
     except ConflictError:
       raise
     except:
-      LOG('WARNING ActivityTool', 0,
+      LOG('CMFActivity', WARNING,
           'Validation of Object %s raised exception' % '/'.join(message.object_path),
           error=sys.exc_info())
       # Do not try to call methods on objects which cause errors
       return EXCEPTION
     return VALID
 
+  def getDependentMessageList(self, activity_tool, message, **kw):
+    message_list = []
+    for k, v in kw.iteritems():
+      result = activity_tool.getDependentMessageList(message, k, v)
+      if result:
+        message_list.extend(result)
+    return message_list
+
+  def getExecutableMessageList(self, activity_tool, message, message_dict,
+                               validation_text_dict):
+    """Get messages which have no dependent message, and store them in the dictionary.
+
+    If the passed message itself is executable, simply store only that message.
+    Otherwise, try to find at least one message executable from dependent messages.
+
+    This may result in no new message, if all dependent messages are already present
+    in the dictionary, if all dependent messages are in different activities, or if
+    the message has a circular dependency.
+
+    The validation text dictionary is used only to cache the results of validations,
+    in order to reduce the number of SQL queries.
+    """
+    if message.uid in message_dict:
+      # Nothing to do. But detect a circular dependency.
+      if message_dict[message.uid] is None:
+        LOG('CMFActivity', ERROR,
+            'message uid %r has a circular dependency' % (message.uid,))
+      return
+
+    cached_result = validation_text_dict.get(message.order_validation_text)
+    if cached_result is None:
+      message_list = message.getDependentMessageList(self, activity_tool)
+      get_transaction().commit() # Release locks.
+      if message_list:
+        # The result is not empty, so this message is not executable.
+        validation_text_dict[message.order_validation_text] = 0
+        now_date = DateTime()
+        for activity, m in message_list:
+          # Note that the messages may contain ones which are already assigned or not
+          # executable yet.
+          if activity is self and m.processing_node == -1 and m.date <= now_date:
+            # Call recursively. Set None as a marker to detect a circular dependency.
+            message_dict[message.uid] = None
+            try:
+              self.getExecutableMessageList(activity_tool, m, message_dict,
+                                             validation_text_dict)
+            finally:
+              del message_dict[message.uid]
+      else:
+        validation_text_dict[message.order_validation_text] = 1
+        message_dict[message.uid] = message
+    elif cached_result:
+      message_dict[message.uid] = message
+    else:
+      pass
+
   def isAwake(self, activity_tool, processing_node):
     return self.is_awake[processing_node]
 
@@ -179,12 +241,30 @@ class Queue:
     pass
 
   def loadMessage(self, s, **kw):
-    m = pickle.loads(s)
+    m = cPickle.load(StringIO(s))
     m.__dict__.update(kw)
     return m
 
   def dumpMessage(self, m):
-    return pickle.dumps(m)
+    return cPickle.dumps(m)
+
+  def getOrderValidationText(self, message):
+    # Return an identifier of validators related to ordering.
+    order_validation_item_list = []
+    key_list = message.activity_kw.keys()
+    key_list.sort()
+    for key in key_list:
+      method_id = "_validate_%s" % key
+      if hasattr(self, method_id):
+        order_validation_item_list.append((key, message.activity_kw[key]))
+    if len(order_validation_item_list) == 0:
+      # When no order validation argument is specified, skip the computation
+      # of the checksum for speed. Here, 'none' is used, because this never be
+      # identical to SHA1 hexdigest (which is always 40 characters), and 'none'
+      # is true in Python. This is important, because dtml-if assumes that an empty
+      # string is false, so we must use a non-empty string for this.
+      return 'none'
+    return sha.new(repr(order_validation_item_list)).hexdigest()
 
   def getMessageList(self, activity_tool, processing_node=None,**kw):
     return []

product/CMFActivity/Activity/RAMDict.py

@@ -27,8 +27,8 @@
 ##############################################################################
 
 from Products.CMFActivity.ActivityTool import registerActivity
+from Products.CMFActivity.Errors import ActivityFlushError
 from Queue import Queue, VALID
-from Products.CMFActivity.ActiveObject import DISTRIBUTABLE_STATE, INVOKE_ERROR_STATE, VALIDATE_ERROR_STATE
 
 from zLOG import LOG
 

product/CMFActivity/Activity/RAMQueue.py

@@ -28,7 +28,6 @@
 
 from Products.CMFActivity.ActivityTool import registerActivity
 from Queue import Queue, VALID
-from Products.CMFActivity.ActiveObject import DISTRIBUTABLE_STATE, INVOKE_ERROR_STATE, VALIDATE_ERROR_STATE
 
 try:
   from transaction import get as get_transaction

product/CMFActivity/ActivityTool.py

@@ -89,11 +89,11 @@ class Message:
 
   Message instances are stored in an activity queue, inside the Activity Tool.
   """
-  def __init__(self, object, active_process, activity_kw, method_id, args, kw):
-    if type(object) is StringType:
-      self.object_path = object.split('/')
+  def __init__(self, obj, active_process, activity_kw, method_id, args, kw):
+    if isinstance(obj, str):
+      self.object_path = obj.split('/')
     else:
-      self.object_path = object.getPhysicalPath()
+      self.object_path = obj.getPhysicalPath()
     if type(active_process) is StringType:
       self.active_process = active_process.split('/')
     elif active_process is None:
@@ -198,8 +198,13 @@ class Message:
       if hasattr(activity_tool, 'error_log'):
         activity_tool.error_log.raising(sys.exc_info())
 
-  def validate(self, activity, activity_tool):
-    return activity.validate(activity_tool, self, **self.activity_kw)
+  def validate(self, activity, activity_tool, check_order_validation=1):
+    return activity.validate(activity_tool, self,
+                             check_order_validation=check_order_validation,
+                             **self.activity_kw)
+
+  def getDependentMessageList(self, activity, activity_tool):
+    return activity.getDependentMessageList(activity_tool, self, **self.activity_kw)
 
   def notifyUser(self, activity_tool, message="Failed Processing Activity"):
     """Notify the user that the activity failed."""
@@ -827,7 +832,7 @@ class ActivityTool (Folder, UniqueObject):
                   'could not dump messages from %s' %
                   (activity,), error=sys.exc_info())
 
-      if hasattr(folder, 'SQLDict_createMessageTable'):
+      if getattr(folder, 'SQLDict_createMessageTable', None) is not None:
         try:
           folder.SQLDict_dropMessageTable()
         except ConflictError:
@@ -838,7 +843,7 @@ class ActivityTool (Folder, UniqueObject):
               error=sys.exc_info())
         folder.SQLDict_createMessageTable()
 
-      if hasattr(folder, 'SQLQueue_createMessageTable'):
+      if getattr(folder, 'SQLQueue_createMessageTable', None) is not None:
         try:
           folder.SQLQueue_dropMessageTable()
         except ConflictError:
@@ -920,16 +925,24 @@ class ActivityTool (Folder, UniqueObject):
       self.immediateReindexObject()
 
     # Active synchronisation methods
+    security.declarePrivate('validateOrder')
     def validateOrder(self, message, validator_id, validation_value):
+      message_list = self.getDependentMessageList(message, validator_id, validation_value)
+      return len(message_list) > 0
+
+    security.declarePrivate('getDependentMessageList')
+    def getDependentMessageList(self, message, validator_id, validation_value):
       global is_initialized
       if not is_initialized: self.initialize()
+      message_list = []
       for activity in activity_list:
         method_id = "_validate_%s" % validator_id
-        if hasattr(activity, method_id):
-          if getattr(activity,method_id)(aq_inner(self),
-                     message, validation_value):
-            return 1
-      return 0
+        method = getattr(activity, method_id, None)
+        if method is not None:
+          result = method(aq_inner(self), message, validation_value)
+          if result:
+            message_list.extend([(activity, m) for m in result])
+      return message_list
 
     # Required for tests (time shift)
     def timeShift(self, delay):

product/CMFActivity/skins/activity/SQLDict_clearProcessingFlag.zsql

@@ -2,12 +2,12 @@
 title:
 connection_id:cmf_activity_sql_connection
 max_rows:1000
-max_cache:100
+max_cache:0
 cache_time:0
 class_name:
 class_file:
 </dtml-comment>
-<params>processing_node:int=-1</params>
+<params>processing_node</params>
 UPDATE
   message
 SET

product/CMFActivity/skins/activity/SQLDict_createMessageTable.zsql

@@ -2,7 +2,7 @@
 title:
 connection_id:cmf_activity_sql_connection
 max_rows:1000
-max_cache:100
+max_cache:0
 cache_time:0
 class_name:
 class_file:

product/CMFActivity/skins/activity/SQLDict_processMessage.zsql

@@ -13,5 +13,6 @@ SET
   processing_date = <dtml-sqlvar "_.DateTime()" type="datetime">,
   processing = 1
 WHERE
-<dtml-in uid>uid = <dtml-sqlvar sequence-item type="int"><dtml-if sequence-end><dtml-else>
- OR </dtml-if></dtml-in>
+  uid IN (
+<dtml-in uid><dtml-sqlvar sequence-item type="int"><dtml-if sequence-end><dtml-else>,</dtml-if></dtml-in>
+  )

product/CMFActivity/skins/activity/SQLDict_readMessageList.zsql

@@ -11,19 +11,9 @@ class_file:
 method_id
 processing_node
 priority
-to_processing_date
-include_processing</params>
-<dtml-if to_processing_date>UPDATE message
-SET
-  processing = 0
-WHERE
-  processing = 1
-AND
-  processing_date < <dtml-sqlvar to_processing_date type="datetime">
-  
-<dtml-var "'\0'">
-
-</dtml-if>SELECT * FROM
+include_processing
+to_date</params>
+SELECT * FROM
     message
 WHERE
 1 = 1
@@ -34,5 +24,6 @@ WHERE
 <dtml-if priority> AND priority = <dtml-sqlvar priority type="int"> </dtml-if>
 <dtml-if path>AND path = <dtml-sqlvar path type="string"> </dtml-if>
 <dtml-if method_id> AND method_id = <dtml-sqlvar method_id type="string"> </dtml-if>
+<dtml-if to_date> AND date <= <dtml-sqlvar to_date type="datetime"> </dtml-if>
 ORDER BY
      priority, date, uid

product/CMFActivity/skins/activity/SQLDict_readUidList.zsql

 <dtml-comment>
 title:
 connection_id:cmf_activity_sql_connection
-max_rows:10000
-max_cache:100
+max_rows:1000
+max_cache:0
 cache_time:0
 class_name:
 class_file:

product/CMFActivity/skins/activity/SQLDict_validateMessageList.zsql

@@ -11,32 +11,31 @@ class_file:
 message_uid
 path
 tag
+count
 </params>
 SELECT
-    COUNT(DISTINCT uid) as uid_count
+<dtml-if count>
+    COUNT(*) AS uid_count
+<dtml-else>
+    *
+</dtml-if>
 FROM
     message
 WHERE
     processing_node >= -2
 <dtml-if method_id>
-    AND (
-<dtml-in method_id>
-        method_id = <dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else> OR </dtml-if>
-</dtml-in>
+    AND method_id IN (
+<dtml-in method_id><dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else>,</dtml-if></dtml-in>
     )
 </dtml-if>
 <dtml-if message_uid>AND uid = <dtml-sqlvar message_uid type="int"> </dtml-if>
 <dtml-if path>
-    AND (
-<dtml-in path>
-        path = <dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else> OR </dtml-if>
-</dtml-in>
+    AND path IN (
+<dtml-in path><dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else>,</dtml-if></dtml-in>
     )
 </dtml-if>
 <dtml-if tag>
-    AND (
-<dtml-in tag>
-        tag = <dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else> OR </dtml-if>
-</dtml-in>
+    AND tag IN (
+<dtml-in tag><dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else>,</dtml-if></dtml-in>
     )
 </dtml-if>

product/CMFActivity/skins/activity/SQLQueue_assignMessage.zsql

@@ -11,7 +11,7 @@ class_file:
 processing_node
 method_id
 broadcast
-uid:int=0</params>
+uid</params>
 UPDATE message_queue
 SET
   processing_node=<dtml-sqlvar processing_node type="int">,

product/CMFActivity/skins/activity/SQLQueue_clearProcessingFlag.zsql

@@ -7,7 +7,7 @@ cache_time:0
 class_name:
 class_file:
 </dtml-comment>
-<params>processing_node:int=-1</params>
+<params>processing_node</params>
 UPDATE
   message_queue
 SET

product/CMFActivity/skins/activity/SQLQueue_createMessageTable.zsql

@@ -2,7 +2,7 @@
 title:
 connection_id:cmf_activity_sql_connection
 max_rows:1000
-max_cache:100
+max_cache:0
 cache_time:0
 class_name:
 class_file:

product/CMFActivity/skins/activity/SQLQueue_readMessage.zsql

@@ -19,4 +19,4 @@ WHERE
 <dtml-if to_date> AND date <= <dtml-sqlvar to_date type="datetime"> </dtml-if>
 
 ORDER BY
-    priority, date
+    priority, date, uid

product/CMFActivity/skins/activity/SQLQueue_readMessageList.zsql

@@ -10,7 +10,8 @@ class_file:
 <params>path
 method_id
 processing_node
-priority</params>
+priority
+to_date</params>
 SELECT * FROM
     message_queue
 WHERE
@@ -19,3 +20,6 @@ WHERE
 <dtml-if priority>AND priority = <dtml-sqlvar priority type="int"> </dtml-if>
 <dtml-if path>AND path = <dtml-sqlvar path type="string"></dtml-if>
 <dtml-if method_id>AND method_id = <dtml-sqlvar method_id type="string"></dtml-if>
+<dtml-if to_date> AND date <= <dtml-sqlvar to_date type="datetime"> </dtml-if>
+ORDER BY
+    priority, date, uid

product/CMFActivity/skins/activity/SQLQueue_readUidList.zsql

 <dtml-comment>
 title:
 connection_id:cmf_activity_sql_connection
-max_rows:10000
-max_cache:100
+max_rows:1000
+max_cache:0
 cache_time:0
 class_name:
 class_file:
@@ -16,5 +16,5 @@ SELECT uid FROM
 WHERE
     processing <> 1
 <dtml-if processing_node> AND processing_node = <dtml-sqlvar processing_node type="int"></dtml-if>
-<dtml-if path>AND path = <dtml-sqlvar path type="string"></dtml-if>
-<dtml-if to_date>AND date <= <dtml-sqlvar to_date type="datetime"> </dtml-if>
+<dtml-if path> AND path = <dtml-sqlvar path type="string"></dtml-if>
+<dtml-if to_date> AND date <= <dtml-sqlvar to_date type="datetime"> </dtml-if>

product/CMFActivity/skins/activity/SQLQueue_validateMessageList.zsql

@@ -11,32 +11,31 @@ class_file:
 message_uid
 path
 tag
+count
 </params>
 SELECT
-    COUNT(DISTINCT uid) as uid_count
+<dtml-if count>
+    COUNT(*) AS uid_count
+<dtml-else>
+    *
+</dtml-if>
 FROM
     message_queue
 WHERE
     processing_node >= -2
 <dtml-if method_id>
-    AND (
-<dtml-in method_id>
-        method_id = <dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else> OR </dtml-if>
-</dtml-in>
+    AND method_id IN (
+<dtml-in method_id><dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else>,</dtml-if></dtml-in>
     )
 </dtml-if>
 <dtml-if message_uid>AND uid = <dtml-sqlvar message_uid type="int"> </dtml-if>
 <dtml-if path>
-    AND (
-<dtml-in path>
-        path = <dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else> OR </dtml-if>
-</dtml-in>
+    AND path IN ( 
+<dtml-in path><dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else>,</dtml-if></dtml-in>
     )
 </dtml-if>
 <dtml-if tag>
-    AND (
-<dtml-in tag>
-        tag = <dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else> OR </dtml-if>
-</dtml-in>
+    AND tag IN (
+<dtml-in tag><dtml-sqlvar sequence-item type="string"><dtml-if sequence-end><dtml-else>,</dtml-if></dtml-in>
     )
 </dtml-if>

product/CMFActivity/skins/activity/SQLQueue_writeMessage.zsql

@@ -13,7 +13,7 @@ method_id
 message
 priority
 broadcast
-processing_node=-1
+processing_node
 date
 tag</params>
 INSERT INTO message_queue
@@ -22,7 +22,9 @@ SET
   path = <dtml-sqlvar path type="string">,
   <dtml-if date>date = <dtml-sqlvar date type="datetime">, <dtml-else>date = <dtml-sqlvar "_.DateTime()" type="datetime">, </dtml-if> 
   method_id = <dtml-sqlvar method_id type="string">,
+<dtml-if processing_node>
   processing_node = <dtml-sqlvar processing_node type="int">,
+</dtml-if>
   broadcast = <dtml-sqlvar broadcast type="int">,
   processing = -1,
   priority = <dtml-sqlvar priority type="int">,