Commit 1561b3a0 authored by Kirill Smelkov's avatar Kirill Smelkov

Clarify meaning of error vs failure

failure is assertion failure.
error is any other error.

Source: https://docs.python.org/3/library/unittest.html#organizing-test-code
-> "If the test fails, an exception will be raised ... identify the test
case as a failure. Any other exceptions will be treated as errors"

While searching ERP5 code for failure_count to clarify its meaning, I've
also discovered expected_failures and unexpected_successes (*) so they
are also added to the table.

(*) see e.g. https://lab.nexedi.com/nexedi/erp5/blob/1e31c091/product/ERP5Type/tests/ERP5TypeTestSuite.py#L6-8
parent 74a7853c
......@@ -267,10 +267,16 @@ class LocalTestResultLine:
_ = v('error_count')
if _ == '?':
st = '?'
elif _ == 0:
st = 'ok'
elif _ != 0:
st = 'error'
else:
st = 'fail'
_ = v('failure_count')
if _ == '?':
st = '?'
elif _ != 0:
st = 'fail'
else:
st = 'ok'
print('%s\t%s\t%.3fs\t# %st %se %sf %ss' % (st, self.name, kw['duration'], v('test_count'), v('error_count'), v('failure_count'), v('skip_count')))
# XXX + dump .json ?
......@@ -290,17 +296,19 @@ class PyTest:
return default
return int(m.group(1))
stat = {}
stat = {'test_count': 0}
def stat_set(stat_key, from_name):
v = get(from_name)
if v is None:
return
stat[stat_key] = v
stat_set('skip_count', 'skipped')
stat_set('error_count', 'failed')
stat_set('failure_count', 'xfailed') # XXX ok?
npass = get('passed', 0)
stat['test_count'] = npass + stat.get('failure_count', 0) + stat.get('skip_count', 0) + stat.get('error_count', 0)
stat['test_count'] += v
stat_set('skip_count', 'skipped')
stat_set('failure_count', 'failed')
stat_set('expected_failures', 'xfailed')
stat_set('unexpected_successes', 'xpassed')
# don't set error_count - everything goes to failure_count
stat['test_count'] += get('passed', 0)
return stat
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment