stop using a local cache on TaskDistributionTool

A dictionary was used as cached to store test result path and
duration of tests. This was quite convenient, however this
dictionary was growing, and since this data was commited thouzands
time per day, this was making the zodb growing faster and faster.

So we now rely on catalog + tags to retrieve test results. We
also use int_index on test result line to order them by duration
time. We will surely increase the load of sql database (we might
need to add specific index), but this will avoid zodb using too
much space.

product/ERP5/Tool/TaskDistributionTool.py

@@ -48,11 +48,6 @@ class TaskDistributionTool(BaseTool):
 
   def __init__(self, *args, **kw):
     BaseTool.__init__(self, *args, **kw)
-    # XXX Cache information about running test results, because the protocol
-    #     is synchronous and we can't rely on the catalog.
-    #     This is a hack until a better (and asynchronous) protocol is
-    #     implemented.
-    self.test_result_dict = {}
 
   security.declarePublic('getProtocolRevision')
   def getProtocolRevision(self):
@@ -89,9 +84,19 @@ class TaskDistributionTool(BaseTool):
     portal = self.getPortalObject()
     if test_title is None:
       test_title = name
-    test_result_path, line_dict = self.test_result_dict.get(
-         test_title, ('', {}))
-    duration_dict = {}
+    tag = "test_result_creation_%s" % test_title
+    # If there is already pending creation, we raise error, the testnode
+    # will automatically retry later
+    assert portal.portal_activities.countMessageWithTag(tag) == 0, \
+           "There is already a test result under creation"
+    test_result_path = None
+    result_list = portal.test_result_module.searchFolder(
+                         portal_type="Test Result",
+                         simulation_state="started",
+                         title=test_title,
+                         sort_on=[("creation_date","descending")])
+    if len(result_list):
+      test_result_path = result_list[0].getRelativeUrl()
     def createNode(test_result, node_title):
       if node_title is not None:
         node = self._getTestResultNode(test_result, node_title)
@@ -99,7 +104,8 @@ class TaskDistributionTool(BaseTool):
           node = test_result.newContent(portal_type='Test Result Node',
                                  title=node_title)
           node.start()
-    def createTestResultLineList(test_result, test_name_list, line_dict):
+    def createTestResultLineList(test_result, test_name_list):
+      duration_list = []
       previous_test_result_list = portal.test_result_module.searchFolder(
              title='=%s' % test_result.getTitle(),
              sort_on=[('creation_date','descending')],
@@ -109,11 +115,19 @@ class TaskDistributionTool(BaseTool):
         previous_test_result = previous_test_result_list[0].getObject()
         for line in previous_test_result.objectValues():
           if line.getSimulationState() == 'stopped':
-            duration_dict[line.getTitle()] = line.getProperty('duration')
+            duration_list.append((line.getTitle(),line.getProperty('duration')))
+      duration_list.sort(key=lambda x: -x[1])
+      sorted_test_list = [x[0] for x in duration_list]
       for test_name in test_name_list:
+        index = 0
+        if sorted_test_list:
+          try:
+            index = sorted_test_list.index(test_name)
+          except ValueError:
+            pass
         line = test_result.newContent(portal_type='Test Result Line',
-                                      title=test_name)
-        line_dict[line.getId()] = duration_dict.get(test_name)
+                                      title=test_name,
+                                      int_index=index)
     reference_list_string = None
     if type(revision) is str and '=' in revision:
       reference_list_string = revision
@@ -129,8 +143,7 @@ class TaskDistributionTool(BaseTool):
       test_result = portal.unrestrictedTraverse(test_result_path, None)
       if test_result is None or test_result.getSimulationState() in \
                ('cancelled', 'failed'):
-        del self.test_result_dict[test_title]
-        line_dict = {}
+        pass
       else:
         last_state = test_result.getSimulationState()
         last_revision = str(test_result.getIntIndex())
@@ -141,9 +154,10 @@ class TaskDistributionTool(BaseTool):
             last_revision = reference
           elif reference:
             last_revision = last_revision, reference
-          if len(line_dict) == 0 and len(test_name_list):
-            self._p_changed = 1
-            createTestResultLineList(test_result, test_name_list, line_dict)
+          if len(test_result.objectValues(portal_type="Test Result Line")) == 0 \
+              and len(test_name_list):
+            self._p_changed = 1 # avoid two parallel creation
+            createTestResultLineList(test_result, test_name_list)
           return test_result_path, last_revision
         if last_state == 'stopped':
           if reference_list_string is not None:
@@ -155,7 +169,8 @@ class TaskDistributionTool(BaseTool):
       portal_type='Test Result',
       title=test_title,
       reference=reference,
-      predecessor=test_result_path)
+      predecessor=test_result_path,
+      activate_kw={"tag": tag})
     if int_index is not None:
       test_result.setIntIndex(int_index)
     if project_title is not None:
@@ -168,10 +183,9 @@ class TaskDistributionTool(BaseTool):
                       ([x.path for x in project_list], project_title))
     test_result.updateLocalRolesOnSecurityGroups() # XXX
     test_result_path = test_result.getRelativeUrl()
-    self.test_result_dict[test_title] = test_result_path, line_dict
     self._p_changed = 1
     test_result.start()
-    createTestResultLineList(test_result, test_name_list, line_dict)
+    createTestResultLineList(test_result, test_name_list)
     createNode(test_result, node_title)
     return test_result_path, revision
 
@@ -188,12 +202,9 @@ class TaskDistributionTool(BaseTool):
     test_result = portal.restrictedTraverse(test_result_path)
     if test_result.getSimulationState() != 'started':
       return
-    path, line_dict = self.test_result_dict[test_result.getTitle()]
-    assert path == test_result_path
     started_list = []
-    for line_id, duration in sorted(line_dict.iteritems(),
-                                    key=lambda x: x[1], reverse=1):
-      line = test_result[line_id]
+    for line in test_result.objectValues(portal_type="Test Result Line",
+                                         sort_on=[("int_index","ascending")]):
       test = line.getTitle()
       if test not in exclude_list:
         state = line.getSimulationState()
@@ -218,15 +229,11 @@ class TaskDistributionTool(BaseTool):
     portal = self.getPortalObject()
     line = portal.restrictedTraverse(test_path)
     test_result = line.getParentValue()
-    path, line_dict = self.test_result_dict[test_result.getTitle()]
     if test_result.getSimulationState() == 'started':
-      assert path == test_result.getRelativeUrl()
-      line_id = line.getId()
-      if line_id in line_dict:
+      if line.getSimulationState() == "started":
         line.stop(**status_dict)
-        del line_dict[line_id]
-        self._p_changed = 1
-        if not line_dict:
+      if set([x.getSimulationState() for x in test_result.objectValues(
+                portal_type="Test Result Line")]) == set(["stopped"]):
         test_result.stop()
 
   def _extractXMLRPCDict(self, xmlrpc_dict):