Commit 32859fc4 by Sebastien Robin

task distribution: avoid test nodes stuck with test suite having failing runTestSuite

1 parent d2dd961f
......@@ -493,6 +493,12 @@ class TestTaskDistribution(ERP5TypeTestCase):
['testFailing', 'testSlow', 'testFast'],
[self.tool.startUnitTest(test_result_path)[1] for _ in range(3)])
def checkTestResultLine(self, test_result, expected):
line_list = test_result.objectValues(portal_type="Test Result Line")
found_list = [(x.getTitle(), x.getSimulationState()) for x in line_list]
found_list.sort(key=lambda x: x[0])
self.assertEqual(expected, found_list)
def test_06b_restartStuckTest(self):
"""
Check if a test result line is not stuck in 'started', if so, redraft
......@@ -503,18 +509,13 @@ class TestTaskDistribution(ERP5TypeTestCase):
test_result = self.portal.unrestrictedTraverse(test_result_path)
line_url, _ = self.tool.startUnitTest(test_result_path)
now = DateTime()
def checkTestResultLine(expected):
line_list = test_result.objectValues(portal_type="Test Result Line")
found_list = [(x.getTitle(), x.getSimulationState()) for x in line_list]
found_list.sort(key=lambda x: x[0])
self.assertEqual(expected, found_list)
checkTestResultLine([('testBar', 'started'), ('testFoo', 'draft')])
self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'draft')])
self._callRestartStuckTestResultAlarm()
checkTestResultLine([('testBar', 'started'), ('testFoo', 'draft')])
self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'draft')])
line_url, _ = self.tool.startUnitTest(test_result_path)
checkTestResultLine([('testBar', 'started'), ('testFoo', 'started')])
self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'started')])
self._callRestartStuckTestResultAlarm()
checkTestResultLine([('testBar', 'started'), ('testFoo', 'started')])
self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'started')])
# now let change history to do like if a test result line was started
# a long time ago
line = self.portal.restrictedTraverse(line_url)
......@@ -522,9 +523,9 @@ class TestTaskDistribution(ERP5TypeTestCase):
if history_line['action'] == 'start':
history_line['time'] = now - 1
self._callRestartStuckTestResultAlarm()
checkTestResultLine([('testBar', 'started'), ('testFoo', 'draft')])
self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'draft')])
self.tool.stopUnitTest(line_url, {})
checkTestResultLine([('testBar', 'started'), ('testFoo', 'stopped')])
self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'stopped')])
def test_07_reportTaskFailure(self):
"""
......@@ -594,6 +595,50 @@ class TestTaskDistribution(ERP5TypeTestCase):
finally:
self.unpinDateTime()
def test_07c_reportTaskFailureWithRedraftedTestResultLine(self):
"""
Similar to above test.
But on the other hand, if a test result line is started many times (due to
automatic redraft), then this might just means we have issue of runTestSuite unable
to finish tests, or we might have just tests that can never be executed within timeout time.
In such case, it's better to mark test result as failed to give a chance to other test
suites to be executed
"""
now = DateTime()
try:
self.pinDateTime(now - 1.0/24*8)
test_result_path, _ = self._createTestResult(test_list=['testFoo'])
test_result = self.getPortalObject().unrestrictedTraverse(test_result_path)
self.checkTestResultLine(test_result, [('testFoo', 'draft')])
self.assertEqual("started", test_result.getSimulationState())
node, = test_result.objectValues(portal_type="Test Result Node",
sort_on=[("title", "ascending")])
self.assertEqual("started", node.getSimulationState())
self.tool.startUnitTest(test_result_path)
self.checkTestResultLine(test_result, [('testFoo', 'started')])
# We have a failure but with recent activities on tests
self.pinDateTime(now - 1.0/24*7.5)
self.tool.reportTaskFailure(test_result_path, {}, "Node0")
self.assertEqual("failed", node.getSimulationState())
self.assertEqual("started", test_result.getSimulationState())
self.checkTestResultLine(test_result, [('testFoo', 'started')])
# some hours later, test line is redrafted
self.pinDateTime(now - 1.0/24*3)
self._callRestartStuckTestResultAlarm()
self.checkTestResultLine(test_result, [('testFoo', 'draft')])
# Test is then relaunched
self.tool.startUnitTest(test_result_path)
self.checkTestResultLine(test_result, [('testFoo', 'started')])
# We have another failure but remains only test result line that was already
# redrafted, so we have to mark the test result as failed
self.pinDateTime(now - 1.0/24*2.5)
self.tool.reportTaskFailure(test_result_path, {}, "Node0")
self.assertEqual("failed", node.getSimulationState())
self.assertEqual("failed", test_result.getSimulationState())
finally:
self.unpinDateTime()
def test_08_checkWeCanNotCreateTwoTestResultInParallel(self):
"""
To avoid duplicates of test result when several testnodes works on the
......
......@@ -285,7 +285,13 @@ class TaskDistributionTool(BaseTool):
for test_result_line in test_result.objectValues(
portal_type="Test Result Line"):
if test_result_line.getModificationDate() > recent_time:
break
# do not take into account redrafted lines, this means we already
# add issues with them
if len([x for x in portal.portal_workflow.getInfoFor(
ob=test_result_line,
name='history',
wf_id='test_result_workflow') if x['action']=='redraft']) == 0:
break
else:
if test_result.getSimulationState() not in ('failed', 'cancelled'):
test_result.fail()
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!