Raise ctx.err() if test run was cancelled

This is normal rule to return an error from a task if the task has to
abort due to cancellation. We already do this in tee, but not in the
function that is waiting for spawned process to complete(*).

-> Fix that. Wrap corresponding wg.wait() into try/except and check if
it fails due to cancellation, upon which we should not raise, but
instead should continue to finish current test_result_line and only
after stop test run normally (again without raise, but with regular one
line log entry).

(*) added in 0ad45a9c (Detect if a test leaks processes and terminate them)

/reviewed-by @jerome
/reviewed-on nexedi/nxdtest!14

nxdtest/__init__.py

@@ -62,7 +62,7 @@ import os, sys, argparse, logging, traceback, re, pwd, socket
 from errno import ESRCH, EPERM
 import six
 from golang import b, defer, func, select, default
-from golang import context, sync
+from golang import errors, context, sync
 import psutil
 
 # loadNXDTestFile loads .nxdtest file located @path.
@@ -214,6 +214,10 @@ def main():
     # run the tests
     devnull = open(os.devnull)
     while 1:
+        if ctx.err() is not None:
+            emit("# test run canceled")
+            break
+
         # ask master for next test to run; stop if no more.
         test_result_line = test_result.start()
         if test_result_line is None:
@@ -264,6 +268,7 @@ def main():
             wg.go(tee, p.stderr, bstderr, buf_err)
             # wait for p to exit
             def _(ctx):
+                err = None
                 while 1:
                     done = p.poll()
                     if done is not None:
@@ -275,7 +280,9 @@ def main():
                             ctx.done().recv,    # 1
                     )
                     if _ == 1:
+                        emit("# stopping due to cancel")
                         p.terminate()
+                        err = ctx.err()
                         break
 
                     sleep(0.1)
@@ -293,9 +300,19 @@ def main():
                     gone, alive = psutil.wait_procs(procv, timeout=5)
                     for proc in alive:
                         p.kill()
+
+                if err is not None:
+                    raise err
             wg.go(_)
 
+            try:
                 wg.wait()
+            except Exception as e:
+                if errors.Is(e, context.canceled):
+                    pass # ok, finish current test_result_line
+                else:
+                    raise
+
             stdout = b''.join(buf_out)
             stderr = b''.join(buf_err)