Commit b2bffc14 authored by Sebastien Robin's avatar Sebastien Robin

stop using a local cache on TaskDistributionTool

A dictionary was used as cached to store test result path and
duration of tests. This was quite convenient, however this
dictionary was growing, and since this data was commited thouzands
time per day, this was making the zodb growing faster and faster.

So we now rely on catalog + tags to retrieve test results. We
also use int_index on test result line to order them by duration
time. We will surely increase the load of sql database (we might
need to add specific index), but this will avoid zodb using too
much space.
parent 67f13f5a
......@@ -48,11 +48,6 @@ class TaskDistributionTool(BaseTool):
def __init__(self, *args, **kw):
BaseTool.__init__(self, *args, **kw)
# XXX Cache information about running test results, because the protocol
# is synchronous and we can't rely on the catalog.
# This is a hack until a better (and asynchronous) protocol is
# implemented.
self.test_result_dict = {}
security.declarePublic('getProtocolRevision')
def getProtocolRevision(self):
......@@ -89,9 +84,19 @@ class TaskDistributionTool(BaseTool):
portal = self.getPortalObject()
if test_title is None:
test_title = name
test_result_path, line_dict = self.test_result_dict.get(
test_title, ('', {}))
duration_dict = {}
tag = "test_result_creation_%s" % test_title
# If there is already pending creation, we raise error, the testnode
# will automatically retry later
assert portal.portal_activities.countMessageWithTag(tag) == 0, \
"There is already a test result under creation"
test_result_path = None
result_list = portal.test_result_module.searchFolder(
portal_type="Test Result",
simulation_state="started",
title=test_title,
sort_on=[("creation_date","descending")])
if len(result_list):
test_result_path = result_list[0].getRelativeUrl()
def createNode(test_result, node_title):
if node_title is not None:
node = self._getTestResultNode(test_result, node_title)
......@@ -99,7 +104,8 @@ class TaskDistributionTool(BaseTool):
node = test_result.newContent(portal_type='Test Result Node',
title=node_title)
node.start()
def createTestResultLineList(test_result, test_name_list, line_dict):
def createTestResultLineList(test_result, test_name_list):
duration_list = []
previous_test_result_list = portal.test_result_module.searchFolder(
title='=%s' % test_result.getTitle(),
sort_on=[('creation_date','descending')],
......@@ -109,11 +115,19 @@ class TaskDistributionTool(BaseTool):
previous_test_result = previous_test_result_list[0].getObject()
for line in previous_test_result.objectValues():
if line.getSimulationState() == 'stopped':
duration_dict[line.getTitle()] = line.getProperty('duration')
duration_list.append((line.getTitle(),line.getProperty('duration')))
duration_list.sort(key=lambda x: -x[1])
sorted_test_list = [x[0] for x in duration_list]
for test_name in test_name_list:
index = 0
if sorted_test_list:
try:
index = sorted_test_list.index(test_name)
except ValueError:
pass
line = test_result.newContent(portal_type='Test Result Line',
title=test_name)
line_dict[line.getId()] = duration_dict.get(test_name)
title=test_name,
int_index=index)
reference_list_string = None
if type(revision) is str and '=' in revision:
reference_list_string = revision
......@@ -129,8 +143,7 @@ class TaskDistributionTool(BaseTool):
test_result = portal.unrestrictedTraverse(test_result_path, None)
if test_result is None or test_result.getSimulationState() in \
('cancelled', 'failed'):
del self.test_result_dict[test_title]
line_dict = {}
pass
else:
last_state = test_result.getSimulationState()
last_revision = str(test_result.getIntIndex())
......@@ -141,9 +154,10 @@ class TaskDistributionTool(BaseTool):
last_revision = reference
elif reference:
last_revision = last_revision, reference
if len(line_dict) == 0 and len(test_name_list):
self._p_changed = 1
createTestResultLineList(test_result, test_name_list, line_dict)
if len(test_result.objectValues(portal_type="Test Result Line")) == 0 \
and len(test_name_list):
self._p_changed = 1 # avoid two parallel creation
createTestResultLineList(test_result, test_name_list)
return test_result_path, last_revision
if last_state == 'stopped':
if reference_list_string is not None:
......@@ -155,7 +169,8 @@ class TaskDistributionTool(BaseTool):
portal_type='Test Result',
title=test_title,
reference=reference,
predecessor=test_result_path)
predecessor=test_result_path,
activate_kw={"tag": tag})
if int_index is not None:
test_result.setIntIndex(int_index)
if project_title is not None:
......@@ -168,10 +183,9 @@ class TaskDistributionTool(BaseTool):
([x.path for x in project_list], project_title))
test_result.updateLocalRolesOnSecurityGroups() # XXX
test_result_path = test_result.getRelativeUrl()
self.test_result_dict[test_title] = test_result_path, line_dict
self._p_changed = 1
test_result.start()
createTestResultLineList(test_result, test_name_list, line_dict)
createTestResultLineList(test_result, test_name_list)
createNode(test_result, node_title)
return test_result_path, revision
......@@ -188,12 +202,9 @@ class TaskDistributionTool(BaseTool):
test_result = portal.restrictedTraverse(test_result_path)
if test_result.getSimulationState() != 'started':
return
path, line_dict = self.test_result_dict[test_result.getTitle()]
assert path == test_result_path
started_list = []
for line_id, duration in sorted(line_dict.iteritems(),
key=lambda x: x[1], reverse=1):
line = test_result[line_id]
for line in test_result.objectValues(portal_type="Test Result Line",
sort_on=[("int_index","ascending")]):
test = line.getTitle()
if test not in exclude_list:
state = line.getSimulationState()
......@@ -218,15 +229,11 @@ class TaskDistributionTool(BaseTool):
portal = self.getPortalObject()
line = portal.restrictedTraverse(test_path)
test_result = line.getParentValue()
path, line_dict = self.test_result_dict[test_result.getTitle()]
if test_result.getSimulationState() == 'started':
assert path == test_result.getRelativeUrl()
line_id = line.getId()
if line_id in line_dict:
if line.getSimulationState() == "started":
line.stop(**status_dict)
del line_dict[line_id]
self._p_changed = 1
if not line_dict:
if set([x.getSimulationState() for x in test_result.objectValues(
portal_type="Test Result Line")]) == set(["stopped"]):
test_result.stop()
def _extractXMLRPCDict(self, xmlrpc_dict):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment