Commit 6a0c7290 authored by Sebastien Robin's avatar Sebastien Robin

Task Distribution: give a chance for test suite to finish when testnodes are missing

parent b96534a5
master DateTime.equalTo_fix TMP-2to3 UpdateValidationStateFromConsistency UserPropertySheet_backward_compatibility addToDate_implicit_localtime add_boolean_type arnau arnau-RD-Components-CacheTool arnau-RD-Components-ERP5Form-ERP5Report arnau-RD-Components-ERP5Form-SelectionTool-MemcachedTool arnau-RD-Components-ERP5OOo arnau-RD-Components-PreferenceTool-Preference arnau-RD-Components-Products-import-compat arnau-RD-Components-astroid_cache_per_component_reset arnau-RD-Components-erp5_crm arnau-RD-Components-erp5_trade-TODO-Interactor arnau-RD-ERP5ify-portal_workflow-1-seb-merged-with-recent-master arnau-RD-ERP5ify-portal_workflow-2-do-not-rename-erp5_workflow-portal_types arnau-RD-ERP5ify-portal_workflow-BAK arnau-RD-ERP5ify-portal_workflow-WIP arnau-RD-ERP5ify-portal_workflow-WITHOUT-MIGRATION arnau-RD-ERP5ify-portal_workflow-WORKFLOWS-NOT-MIGRATED arnau-RD-py3-master-TM arnau-RD-py3-master-TM-BAK arnau-RD-py3-master-WIP arnau-RD-py3-master-WIP-BAK arnau-RD-py3-master-WIP-bt5-and-tests arnau-RD-zope4py3 arnau-TM-Components-Migrate-PortalTransforms arnau-TM-Components-ModuleSecurityInfo arnau-TM-FEC-output arnau-TM-WIP arnau-TM-isBuildable-with-multiple-BusinessLinks arnau-TM-newContent-temp_object arnau-TM-runUnitTest-clear-previous-execution-catalog arnau-TM-wkhtmltopdf arnau-WIP arnau-my2to3 arnau-real-time-inventory-accounting arnau-zope4py2 aurel-zope4py2 bt5_config cache-control-304-response catalog_filter cribjs-bootloader drop-ZServer e2e-erp5 enhance_scalability_testing erp5-component erp5-component-bak erp5-imt erp5-messenger erp5-vifib erp5-vifib-no-Products.DCWorkflowGraph erp5-vifib-py3 erp5_catalog erp5_catalog_final erp5_corporate_identity erp5_drone_simulator erp5_hal_json_style_fix_restricted_access_with_traverse erp5_payslip_migration feat/coding_style_test feat/dedup_roles_in_pickles feat/dms_implicit_predecessor_successor_fields feat/erp5_ide feat/erp5pt feat/fsum feat/improve_rounding_tool feat/inventory_api_group_by_time_interval_list feat/lxml-html-snapshot feat/mariadb-10.11 feat/mariadb-10.11bis feat/mariadb-10.11ter feat/mariadb-10.5 feat/mariadb-10.6 feat/mariadb-11.4bis feat/mariadb-11.4bis-old feat/mariadb-11.4ter feat/notification-message-ignore-missing feat/olapy feat/python_language_support feat/reindexlastobjects_log feat/round_half_up feat/selenium-unexpected-success feat/slapos_agent_distributor feat/subject_set_query feature/renderjs-ui-no-header fix/GHSA-g5vw-3h65-2q3v fix/TALES_hide_error fix/ZMI_editor_preference fix/accounting-fec-no-line fix/advance_ecommerce_coding_crimes fix/balance-0-asset-price fix/bt-module-uninstall fix/erp5_site_global_id fix/invoice-line-currency-price fix/mariadb-1927 fix/measure-optional-variation fix/monaco-altClick fix/officejs_support_request_rss_secu fix/state_var fix/support-request-app-empty fix/testnode_proctitle fix/workflow_info fix/workflow_method_security fix_web_illustration for_testrunner_1 for_testrunner_2 for_testrunner_3 graphic_gadget_js hotfix/rjs-formfields-padding initsite inventory js-ui limit_accelerated_http_cache_manager listbox_url lle-bout/metadata-modules macros_fix master_w monitoring-graph mr1362 my2to3 notebook_roque oauth-login-minor-improvement override_cache_control_header_by_caching_policy_manager poc/json-forms-study publish_recursiveReindexObject refactor/base_edit restore-from-trashbin revert-192c2000 revert-38554dbe rfc/activate_default rjs_listbox_sort_icon roque_quick scalability-master scalability-roque scalability-roque-2 scalability-run-command scalability_crash_mariadb shop-box-rebase streaming_fix streaming_fix-0 support_legacy_sftp_server support_relative_url_in_hyperlink_field support_request test-renderjs-float-field-step test_cmfactivity_isolation_level test_dynamic_methods tomo_testnode_slap_request translatable_path_master trustable-x-forwarded-for ttrm unify_predicate_edit without_legacy_workflow workaround_mroonga_14 wsgi wsgi-gevent wsgi_backport_setbody_lock wsgi_medusa_stream_fix zope2 zope2zope4py2 zope4py3 zope4py3-BEFORE-CLEANUP zope4py3-master-rebase 0.4.59.1 0.4.59 test-ui test-rjsacc test-rjs renderjs-test nexedi-erp5-vifib-20201105 erp5.util-0.4.76 erp5.util-0.4.75 erp5.util-0.4.74 erp5.util-0.4.73 erp5.util-0.4.72 erp5.util-0.4.71 erp5.util-0.4.69 erp5.util-0.4.68 erp5.util-0.4.67 erp5.util-0.4.66 erp5.util-0.4.65 erp5.util-0.4.64 erp5.util-0.4.63 erp5.util-0.4.62 erp5.util-0.4.61 erp5.util-0.4.60 erp5.util-0.4.59.1 erp5.util-0.4.59 erp5.util-0.4.58 erp5.util-0.4.57 erp5.util-0.4.56 erp5.util-0.4.55 erp5.util-0.4.54 erp5.util-0.4.53 erp5.util-0.4.52 erp5-vifib-20240326 erp5-vifib-20230331 erp5-vifib-20230201 erp5-vifib-20220526 erp5-vifib-20220302 erp5-vifib-20210707 erp5-vifib-20201229 erp5-vifib-20200129
No related merge requests found
......@@ -475,6 +475,12 @@ class TestTaskDistribution(ERP5TypeTestCase):
checkTestResultLine([('testBar', 'started'), ('testFoo', 'stopped')])
def test_07_reportTaskFailure(self):
"""
When all test nodes report failures, we should mark the test result as
failed. If we do not do so, test node would always pickup same repository
revision and might fail with same failure forever (for example, a slapos
build issue).
"""
test_result_path, revision = self._createTestResult(node_title="Node0")
next_test_result_path, revision = self._createTestResult(node_title="Node1")
self.assertEqual(test_result_path, next_test_result_path)
......@@ -493,6 +499,48 @@ class TestTaskDistribution(ERP5TypeTestCase):
self.assertEqual("failed", test_result.getSimulationState())
checkNodeState("failed", "failed")
def test_07b_reportTaskFailureWithRunningTest(self):
"""
Similar to above test. Though, sometimes there is failure reported only because
runTestSuite reached timeout. This happens when not enough testnode are working
on a very long test suite. So code investigate if tests looked working fine, and
it might try to not cancel test result if there is chance that tests could be
continued.
For example :
- testnode0 start test suite Foo with revision r0 which would take 6 hours (other
testnodes are busy)
- after 4 hours, runTestSuite reach timeout of 4 hours (value set in test nodes).
thus it report a failure. We do not cancel the test result since everything went
fine up to know
- after some time testnode0 come back to run test suite Foo, revision r0, and
just do the 2 remaining hours. Test Suite can go up to the end even if we have
timeout smaller than total time for test suite.
"""
now = DateTime()
try:
self.pinDateTime(now - 1.0/24*2)
test_result_path, revision = self._createTestResult(node_title="Node0",
test_list=['testFoo', 'testBar'])
test_result = self.getPortalObject().unrestrictedTraverse(test_result_path)
self.assertEqual("started", test_result.getSimulationState())
node, = test_result.objectValues(portal_type="Test Result Node",
sort_on=[("title", "ascending")])
self.assertEqual("started", node.getSimulationState())
line_url, test = self.tool.startUnitTest(test_result_path)
# We have a failure but with recent activities on tests
self.pinDateTime(now - 1.0/24*1.5)
self.tool.reportTaskFailure(test_result_path, {}, "Node0")
self.assertEqual("failed", node.getSimulationState())
self.assertEqual("started", test_result.getSimulationState())
# We have a failure but with no recent activities on tests
self.pinDateTime(now)
self.tool.reportTaskFailure(test_result_path, {}, "Node0")
self.assertEqual("failed", node.getSimulationState())
self.assertEqual("failed", test_result.getSimulationState())
finally:
self.unpinDateTime()
def test_08_checkWeCanNotCreateTwoTestResultInParallel(self):
"""
To avoid duplicates of test result when several testnodes works on the
......
......@@ -27,6 +27,7 @@
##############################################################################
import random
from DateTime import DateTime
from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions, PropertySheet, Constraint, interfaces
from Products.ERP5Type.Tool.BaseTool import BaseTool
......@@ -269,6 +270,15 @@ class TaskDistributionTool(BaseTool):
for node in test_result.objectValues(portal_type='Test Result Node'):
if node.getSimulationState() != 'failed':
break
else:
# now check if we had recent work on test line, if so, this means
# we might just add timeout due to too much tests to execute for too
# little nodes. In that case we would like to continue the work later
recent_time = DateTime() - 1.0/24
for test_result_line in test_result.objectValues(
portal_type="Test Result Line"):
if test_result_line.getModificationDate() > recent_time:
break
else:
if test_result.getSimulationState() not in ('failed', 'cancelled'):
test_result.fail()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment