Unittest and Python3 support

These are the necessary changes to run `SlapOS.Eggs.UnitTest-*` and `SlapOS.SoftwareReleases.IntegrationTest-*` using nxdtest

See merge request nexedi/nxdtest!6

/reviewed-by @kirr

nxdtest/__init__.py

@@ -60,12 +60,14 @@ from subprocess import Popen, PIPE
 from time import time, strftime, gmtime, localtime
 import os, sys, threading, argparse, logging, traceback, re, pwd, socket
 import six
+from golang import b
 
 # loadNXDTestFile loads .nxdtest file located @path.
 def loadNXDTestFile(path): # -> TestEnv
     t = TestEnv()
     g = {'TestCase': t.TestCase,    # TODO + all other public TestEnv methods
-         'PyTest':   PyTest}
+         'PyTest':   PyTest,
+         'UnitTest': UnitTest,}
     with open(path, "r") as f:
         src = f.read()
     six.exec_(src, g)
@@ -99,6 +101,12 @@ class TestEnv:
         self.testv.append(t)
         self.byname[name] = t
 
+def emit(*message):
+  """Emit a message on stdout and flush output.
+  """
+  print(*message)
+  sys.stdout.flush()
+
 def main():
     # testnode executes us giving URL to master results collecting instance and other details
     # https://lab.nexedi.com/nexedi/erp5/blob/744f3fde/erp5/util/testnode/UnitTestRunner.py#L137
@@ -134,7 +142,7 @@ def main():
     # --list
     if args.list:
         for t in tenv.testv:
-            print(t.name)
+            emit(t.name)
         return
 
     # master_url provided -> run tests under master control
@@ -164,6 +172,13 @@ def main():
     # log information about local node
     system_info()
 
+    if sys.version_info < (3,):
+      bstdout = sys.stdout
+      bstderr = sys.stderr
+    else:
+      bstdout = sys.stdout.buffer
+      bstderr = sys.stderr.buffer
+
     # run the tests
     devnull = open(os.devnull)
     while 1:
@@ -176,8 +191,8 @@ def main():
         t = tenv.byname[test_result_line.name]
         tstart = time()
 
-        print('\n>>> %s' % t.name)
-        print('$ %s' % t.command_str())
+        emit('\n>>> %s' % t.name)
+        emit('$ %s' % t.command_str())
 
         # default status dict
         status = {
@@ -193,29 +208,28 @@ def main():
             # In addition to kw['env'], kw['envadj'] allows users to define
             # only adjustments instead of providing full env dict.
             # Test command is spawned with unchanged cwd. Instance wrapper cares to set cwd before running us.
-            # bufsize=1 means 'line buffered'
             kw = t.kw.copy()
             env = kw.pop('env', os.environ)
             env = env.copy()
             envadj = kw.pop('envadj', {})
             env.update(envadj)
-            p = Popen(t.argv, env=env, stdin=devnull, stdout=PIPE, stderr=PIPE, bufsize=1, **kw)
+            p = Popen(t.argv, env=env, stdin=devnull, stdout=PIPE, stderr=PIPE, bufsize=0, **kw)
         except:
-            stdout, stderr = '', traceback.format_exc()
-            sys.stderr.write(stderr)
+            stdout, stderr = b'', b(traceback.format_exc())
+            bstderr.write(stderr)
             status['error_count'] += 1
         else:
             # tee >stdout,stderr so we can also see in testnode logs
             # (explicit teeing instead of p.communicate() to be able to see incremental progress)
             buf_out = []
             buf_err = []
-            tout = threading.Thread(target=tee, args=(p.stdout, sys.stdout, buf_out))
-            terr = threading.Thread(target=tee, args=(p.stderr, sys.stderr, buf_err))
+            tout = threading.Thread(target=tee, args=(p.stdout, bstdout, buf_out))
+            terr = threading.Thread(target=tee, args=(p.stderr, bstderr, buf_err))
             tout.start()
             terr.start()
 
-            tout.join(); stdout = ''.join(buf_out)
-            terr.join(); stderr = ''.join(buf_err)
+            tout.join(); stdout = b''.join(buf_out)
+            terr.join(); stderr = b''.join(buf_err)
             p.wait()
 
             if p.returncode != 0:
@@ -224,10 +238,10 @@ def main():
             # postprocess output, if we can
             if t.summaryf is not None:
                 try:
-                    summary = t.summaryf(stdout)
+                    summary = t.summaryf(stdout, stderr)
                 except:
-                    bad = traceback.format_exc()
-                    sys.stderr.write(bad)
+                    bad = b(traceback.format_exc())
+                    bstderr.write(bad)
                     stderr += bad
                     status['error_count'] += 1
 
@@ -248,7 +262,7 @@ def main():
         }
         tres.update(status)
 
-        print(_test_result_summary(t.name, tres))
+        emit(_test_result_summary(t.name, tres))
         test_result_line.stop(**tres)
 
 # tee, similar to tee(1) utility, copies data from fin to fout appending them to buf.
@@ -296,11 +310,11 @@ def _test_result_summary(name, kw):
 
 # system_info prints information about local computer.
 def system_info():
-    print('date:\t%s' % (strftime("%a, %d %b %Y %H:%M:%S %Z", localtime())))
+    emit('date:\t%s' % (strftime("%a, %d %b %Y %H:%M:%S %Z", localtime())))
     whoami = pwd.getpwuid(os.getuid()).pw_name
-    print('xnode:\t%s@%s' % (whoami, socket.getfqdn()))
-    print('uname:\t%s' % ' '.join(os.uname()))
-    print('cpu:\t%s' % get1('/proc/cpuinfo', 'model name'))
+    emit('xnode:\t%s@%s' % (whoami, socket.getfqdn()))
+    emit('uname:\t%s' % ' '.join(os.uname()))
+    emit('cpu:\t%s' % get1('/proc/cpuinfo', 'model name'))
 
 # get1 returns first entry from file @path prefixed with ^<field>\s*:
 def get1(path, field, default=None):
@@ -350,7 +364,7 @@ class LocalTestResultLine:
 # support for well-known summary functions
 class PyTest:
     @staticmethod
-    def summary(out): # -> status_dict
+    def summary(out, err): # -> status_dict
         # end of output is like
         # ================ 1 failed, 1 passed, 12 skipped in 0.39 seconds ================
         # ...
@@ -363,7 +377,7 @@ class PyTest:
             return {}
 
         def get(name, default=None):
-            m = re.search(r'\b([0-9]+) '+name+r'\b', pytail)
+            m = re.search(br'\b([0-9]+) ' + name.encode() + br'\b', pytail)
             if m is None:
                 return default
             return int(m.group(1))
@@ -384,5 +398,46 @@ class PyTest:
         return stat
 
 
+class UnitTest:
+
+    @staticmethod
+    def summary(out, err): # -> status_dict
+        run_re = re.compile(
+            br'.*Ran (?P<all_tests>\d+) tests? in (?P<seconds>\d+\.\d+)s',
+            re.DOTALL)
+        status_re = re.compile(br"""
+              .*(OK|FAILED)\s+\(
+                (failures=(?P<failures>\d+),?\s*)?
+                (errors=(?P<errors>\d+),?\s*)?
+                (skipped=(?P<skips>\d+),?\s*)?
+                (expected\s+failures=(?P<expected_failures>\d+),?\s*)?
+                (unexpected\s+successes=(?P<unexpected_successes>\d+),?\s*)?
+              \)
+              """, re.DOTALL | re.VERBOSE)
+
+        status_dict = {
+        }
+        run = run_re.search(err)
+        if run:
+            groupdict = run.groupdict()
+            status_dict.update(
+                duration=float(groupdict['seconds']),
+                test_count=int(groupdict['all_tests']),
+                error_count=0,
+                failure_count=0,
+                skip_count=0,
+            )
+        status = status_re.search(err)
+        if status:
+            groupdict = status.groupdict()
+            status_dict.update(
+                error_count=int(groupdict.get('errors') or 0),
+                failure_count=int(groupdict.get('failures') or 0)
+                              + int(groupdict.get('unexpected_successes') or 0),
+                skip_count=int(groupdict.get('skips') or 0)
+                           + int(groupdict.get('expected_failures') or 0))
+        return status_dict
+
+
 if __name__ == '__main__':
     main()

nxdtest/nxdtest_pytest_test.py

@@ -21,12 +21,13 @@
 
 from nxdtest import _test_result_summary, PyTest
 import pytest
+from golang import b
 
-# [] of (name, textout, summaryok)
+# [] of (name, out, err, summaryok)
 testv = []
-def case1(name, textout, summaryok): testv.append((name, textout, summaryok))
+def case1(name, out, err, summaryok): testv.append((name, out, err, summaryok))
 
-case1('ok+xfail', """\
+case1('ok+xfail', b("""\
 ============================= test session starts ==============================
 platform linux2 -- Python 2.7.18, pytest-4.6.11, py-1.9.0, pluggy-0.13.1
 rootdir: /srv/slapgrid/slappart9/srv/testnode/dfq/soft/46d349541123ed5fc6ceea58fd013a51/parts/zodbtools-dev
@@ -39,10 +40,11 @@ zodbtools/test/test_tidrange.py .............................            [ 81%]
 zodbtools/test/test_zodb.py ........                                     [100%]
 
 =============== 41 passed, 2 xfailed, 1 warnings in 4.62 seconds ===============
-""",
+"""),
+b(''),
 '?\ttestname\t1.000s\t# 43t ?e ?f ?s')
 
-case1('ok+fail', """\
+case1('ok+fail', b("""\
 ============================= test session starts ==============================
 platform linux2 -- Python 2.7.18, pytest-4.6.11, py-1.9.0, pluggy-0.13.1
 rootdir: /srv/slapgrid/slappart16/srv/testnode/dfj/soft/8b9988ce0aa31334c6bd56b40e4bba65/parts/pygolang-dev
@@ -111,10 +113,11 @@ E         Use -v to get the full diff
 
 golang/time_test.py:106: AssertionError
 =============== 1 failed, 98 passed, 13 skipped in 26.85 seconds ===============
-""",
+"""),
+b(''),
 '?\ttestname\t1.000s\t# 112t ?e 1f 13s')
 
-case1('ok+tailtext', """\
+case1('ok+tailtext', b("""\
 date:   Sun, 08 Nov 2020 12:26:24 MSK
 xnode:  kirr@deco.navytux.spb.ru
 uname:  Linux deco 5.9.0-1-amd64 #1 SMP Debian 5.9.1-1 (2020-10-17) x86_64
@@ -129,13 +132,14 @@ wcfs: 2020/11/08 12:26:38 /tmp/testdb_fs.Z9IvT0/1.fs: watcher: stat /tmp/testdb_
 # unmount/stop wcfs pid39653 @ /tmp/wcfs/40cc7154ed758d6a867205e79e320c1d3b56458d
 wcfs: 2020/11/08 12:26:38 /tmp/testdb_fs.B3rbby/1.fs: watcher: stat /tmp/testdb_fs.B3rbby/1.fs: use of closed file
 # unmount/stop wcfs pid39595 @ /tmp/wcfs/d0b5d036a2cce47fe73003cf2d9f0b22c7043817
-""",
+"""),
+b(''),
 '?\ttestname\t1.000s\t# 55t ?e ?f ?s')
 
 
-@pytest.mark.parametrize("name,textout,summaryok", testv)
-def test_pytest_summary(name,textout, summaryok):
+@pytest.mark.parametrize("name,out,err,summaryok", testv)
+def test_pytest_summary(name, out, err, summaryok):
     kw = {'duration': 1.0}
-    kw.update(PyTest.summary(textout))
+    kw.update(PyTest.summary(out, err))
     summary = _test_result_summary('testname', kw)
     assert summary == summaryok

nxdtest/nxdtest_test.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2020  Nexedi SA and Contributors.
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+# verify general functionality
+
+import sys
+import re
+
+import pytest
+
+from nxdtest import main
+
+
+@pytest.fixture
+def run_nxdtest(tmpdir):
+    """Fixture which returns a function which invokes nxdtest in a temporary
+    directory, with the provided .nxdtest file content and with arguments
+    passed as `argv`.
+    """
+
+    def _run_nxdtest(nxdtest_file_content, argv=("nxdtest",)):
+        with tmpdir.as_cwd():
+            with open(".nxdtest", "w") as f:
+                f.write(nxdtest_file_content)
+            sys_argv = sys.argv
+            sys.argv = argv
+            try:
+                main()
+            finally:
+                sys.argv = sys_argv
+
+    return _run_nxdtest
+
+
+def test_main(run_nxdtest, capsys):
+    run_nxdtest(
+        """\
+TestCase('TESTNAME', ['echo', 'TEST OUPUT'])
+"""
+    )
+
+    captured = capsys.readouterr()
+    output_lines = captured.out.splitlines()
+    assert ">>> TESTNAME" in output_lines
+    assert "$ echo TEST OUPUT" in output_lines
+    assert "TEST OUPUT" in output_lines
+    assert re.match("ok\tTESTNAME\t.*s\t# 1t 0e 0f 0s", output_lines[-1])
+
+
+def test_error_invoking_command(run_nxdtest, capsys):
+    run_nxdtest(
+        """\
+TestCase('TESTNAME', ['not exist command'])
+"""
+    )
+
+    captured = capsys.readouterr()
+    assert "No such file or directory" in captured.err
+
+
+def test_error_invoking_summary(run_nxdtest, capsys):
+    run_nxdtest(
+        """\
+TestCase('TESTNAME', ['echo'], summaryf="error")
+"""
+    )
+
+    captured = capsys.readouterr()
+    assert "TypeError" in captured.err
+
+
+def test_run_argument(run_nxdtest, capsys):
+    run_nxdtest(
+        """\
+TestCase('TEST1', ['echo', 'TEST1'])
+TestCase('TEST2', ['echo', 'TEST2'])
+""",
+        argv=["nxdtest", "--run", "TEST1"],
+    )
+
+    captured = capsys.readouterr()
+    assert "TEST1" in captured.out
+    assert "TEST2" not in captured.out

nxdtest/nxdtest_unittest_test.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2020  Nexedi SA and Contributors.
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+# verify unittest-related functionality
+
+from nxdtest import _test_result_summary, UnitTest
+import pytest
+from golang import b
+
+# [] of (name, out, err, summaryok)
+testv = []
+def case1(name, out, err, summaryok): testv.append((name, out, err, summaryok))
+
+case1('ok', b(''), b("""\
+test_1 (test.Test) ... ok
+test_2 (test.Test) ... ok
+test_3 (test.Test) ... ok
+
+----------------------------------------------------------------------
+Ran 3 tests in 1.761s
+
+OK
+"""),
+'ok\ttestname\t1.761s\t# 3t 0e 0f 0s')
+
+
+case1('ok+xfail', b(''), b("""\
+test_1 (test.Test) ... ok
+test_2 (test.Test) ... ok
+test_3 (test.Test) ... expected failure
+
+----------------------------------------------------------------------
+Ran 3 tests in 1.098s
+
+OK (expected failures=1)
+"""),
+'ok\ttestname\t1.098s\t# 3t 0e 0f 1s')
+
+
+case1('fail', b(''), b("""\
+test_1 (test.Test) ... ok
+test_2 (test.Test) ... ok
+test_3 (test.Test) ... FAIL
+
+======================================================================
+FAIL: test_3 (test.Test)
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/srv/slapgrid/slappart4/srv/project/nxdtest/tmp/test.py", line 14, in test_3
+    self.assertEqual(1, 2)
+AssertionError: 1 != 2
+
+----------------------------------------------------------------------
+Ran 3 tests in 2.198s
+
+FAILED (failures=1)
+"""),
+'fail\ttestname\t2.198s\t# 3t 0e 1f 0s')
+
+
+case1('error', b(''), b("""\
+test_1 (test.Test) ... ok
+test_2 (test.Test) ... ok
+test_3 (test.Test) ... ERROR
+
+======================================================================
+ERROR: test_3 (test.Test)
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/srv/slapgrid/slappart4/srv/project/nxdtest/tmp/test.py", line 14, in test_3
+    boom
+NameError: name 'boom' is not defined
+
+----------------------------------------------------------------------
+Ran 3 tests in 1.684s
+
+FAILED (errors=1)
+"""),
+'error\ttestname\t1.684s\t# 3t 1e 0f 0s')
+
+
+case1('error-no-test', b(''), b("""\
+usage: python -m unittest discover [-h] [-v] [-q] [--locals] [-f] [-c] [-b]
+                                   [-k TESTNAMEPATTERNS] [-s START]
+                                   [-p PATTERN] [-t TOP]
+python -m unittest discover: error: unrecognized arguments: --argument-error
+"""),
+'?\ttestname\t1.000s\t# ?t ?e ?f ?s')
+
+
+case1('error-no-output', b(''), b(''), '?\ttestname\t1.000s\t# ?t ?e ?f ?s')
+
+
+case1('failed+unexpected_success', b(''), b("""\
+test_1 (test.Test) ... ok
+test_2 (test.Test) ... ok
+test_3 (test.Test) ... unexpected success
+
+----------------------------------------------------------------------
+Ran 3 tests in 1.039s
+
+FAILED (unexpected successes=1)
+"""),
+'fail\ttestname\t1.039s\t# 3t 0e 1f 0s')
+
+
+case1('mixed-output', b(''), b("""\
+----------------------------------------------------------------------
+Ran 1 tests in 1.111s
+
+FAILED (failures=1)
+
+
+----------------------------------------------------------------------
+Ran 3 tests in 2.222s
+
+FAILED (failures=3)
+"""),
+'fail\ttestname\t2.222s\t# 3t 0e 3f 0s')
+
+
+@pytest.mark.parametrize("name,out,err,summaryok", testv)
+def test_unittest_summary(name, out, err, summaryok):
+    kw = {'duration': 1.0}
+    kw.update(UnitTest.summary(out, err))
+    summary = _test_result_summary('testname', kw)
+    assert summary == summaryok

setup.py

@@ -13,7 +13,7 @@ setup(
     keywords    = 'Nexedi testing infrastructure tool tox',
 
     packages    = find_packages(),
-    install_requires = ['erp5.util', 'six'],
+    install_requires = ['erp5.util', 'six', 'pygolang'],
     extras_require = {
                    'test': ['pytest'],
     },