simplify testrunner: remove 'name' from util

always calculate it from environ and file name

greentest/test___monkey_patching.py

@@ -29,8 +29,8 @@ def TESTRUNNER(tests=None):
         atexit.register(os.system, 'rm -f */@test*_tmp')
 
     for filename in tests:
-        yield directory + '/' + filename, [sys.executable, '-u', '-m', 'monkey_test', filename], options.copy()
-        yield directory + '/' + filename + '/Event', [sys.executable, '-u', '-m', 'monkey_test', '--Event', filename], options.copy()
+        yield [sys.executable, '-u', '-m', 'monkey_test', filename], options.copy()
+        yield [sys.executable, '-u', '-m', 'monkey_test', '--Event', filename], options.copy()
 
 
 def main():

greentest/testrunner.py

@@ -5,6 +5,7 @@ from gevent import monkey; monkey.patch_all()
 import sys
 import os
 import glob
+import traceback
 from time import time
 
 from gevent.pool import Pool
@@ -18,57 +19,35 @@ pool = None
 
 def spawn(*args, **kwargs):
     g = pool.spawn(*args, **kwargs)
-    g.link_exception(lambda *args: sys.exit('Internal error'))
+    g.link_exception(lambda *args: sys.exit('Internal error in testrunner.py'))
     return g
 
 
-def process_test(name, cmd, options):
-    options = options or {}
-    setenv = options.get('setenv', {}).copy()
-    setenv.pop('PYTHONPATH', '')
-    environ = options.get('env')
-    if environ is None:
-        environ = os.environ.copy()
-    for key, value in environ.items():
-        if key.startswith('GEVENT_') or key.startswith('GEVENTARES_'):
-            if key not in setenv:
-                setenv[key] = value
-    env_str = ' '.join('%s=%s' % x for x in setenv.items())
-    if env_str and env_str not in name:
-        name = env_str + ' ' + name
-    return name, cmd, options
-
-
-def process_tests(tests):
-    return [process_test(name, cmd, options) for (name, cmd, options) in tests]
-
-
 def run_many(tests, expected=None, failfast=False):
     global NWORKERS, pool
     start = time()
     total = 0
     failed = {}
 
-    tests = process_tests(tests)
     NWORKERS = min(len(tests), NWORKERS)
     pool = Pool(NWORKERS)
     util.BUFFER_OUTPUT = NWORKERS > 1
 
-    def run_one(name, cmd, **kwargs):
-        result = util.run(cmd, name=name, **kwargs)
+    def run_one(cmd, **kwargs):
+        result = util.run(cmd, **kwargs)
         if result:
             if failfast:
                 sys.exit(1)
             # the tests containing AssertionError might have failed because
             # we spawned more workers than CPUs
             # we therefore will retry them sequentially
-            failed[name] = [cmd, kwargs, 'AssertionError' in (result.output or '')]
+            failed[result.name] = [cmd, kwargs, 'AssertionError' in (result.output or '')]
 
     try:
         try:
-            for name, cmd, options in tests:
+            for cmd, options in tests:
                 total += 1
-                spawn(run_one, name, cmd, **options)
+                spawn(run_one, cmd, **(options or {}))
             gevent.wait()
         except KeyboardInterrupt:
             try:
@@ -80,6 +59,7 @@ def run_many(tests, expected=None, failfast=False):
                 util.log('(partial results)\n')
                 raise
     except:
+        traceback.print_exc()
         pool.kill()  # this needed to kill the processes
         raise
 
@@ -89,7 +69,7 @@ def run_many(tests, expected=None, failfast=False):
     if NWORKERS > 1 and toretry:
         util.log('\nWill retry %s failed tests without concurrency:\n- %s\n', len(toretry), '\n- '.join(toretry))
         for name, (cmd, kwargs, _ignore) in failed.items():
-            if not util.run(cmd, name=name, buffer_output=False, **kwargs):
+            if not util.run(cmd, buffer_output=False, **kwargs):
                 failed.pop(name)
                 failed_then_succeeded.append(name)
 
@@ -120,12 +100,12 @@ def discover(tests=None, ignore=None):
     for filename in tests:
         if 'TESTRUNNER' in open(filename).read():
             module = __import__(filename.rsplit('.', 1)[0])
-            for name, cmd, options in module.TESTRUNNER():
+            for cmd, options in module.TESTRUNNER():
                 if remove_options(cmd)[-1] in ignore:
                     continue
-                to_process.append((filename + ' ' + name, cmd, options))
+                to_process.append((cmd, options))
         else:
-            to_process.append((filename, [sys.executable, '-u', filename], default_options.copy()))
+            to_process.append(([sys.executable, '-u', filename], default_options.copy()))
 
     return to_process
 
@@ -150,14 +130,14 @@ def full(args=None):
     for setenv, ignore in [('GEVENT_RESOLVER=thread', None),
                            ('GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8', 'tests_that_dont_use_resolver.txt')]:
         setenv = dict(x.split('=') for x in setenv.split())
-        for name, cmd, options in discover(args, ignore=ignore):
+        for cmd, options in discover(args, ignore=ignore):
             my_setenv = options.get('setenv', {})
             my_setenv.update(setenv)
             options['setenv'] = my_setenv
-            tests.append((name, cmd, options))
+            tests.append((cmd, options))
 
     if sys.version_info[:2] == (2, 7):
-        tests.append(('xtest_pep8.py', [sys.executable, '-u', 'xtest_pep8.py'], None))
+        tests.append(([sys.executable, '-u', 'xtest_pep8.py'], None))
 
     return tests
 
@@ -178,8 +158,8 @@ def main():
     else:
         tests = discover(args, options.ignore)
     if options.discover:
-        for name, cmd, options in process_tests(tests):
-            print '%s: %s' % (name, ' '.join(cmd))
+        for cmd, options in tests:
+            print util.getname(cmd, env=options.get('env'), setenv=options.get('setenv'))
         print '%s tests found.' % len(tests)
     else:
         run_many(tests, expected=options.expected, failfast=options.failfast)

greentest/util.py

@@ -97,20 +97,33 @@ def kill(popen):
         traceback.print_exc()
 
 
-def getname(command):
-    return ' '.join(command)
+def getname(command, env=None, setenv=None):
+    result = []
+
+    for key, value in sorted((env or {}).items()):
+        if key.startswith('GEVENT_') or key.startswith('GEVENTARES_'):
+            result.append('%s=%s' % (key, value))
+
+    for key, value in sorted((setenv or {}).items()):
+        if key.startswith('GEVENT_') or key.startswith('GEVENTARES_'):
+            result.append('%s=%s' % (key, value))
+
+    if isinstance(command, basestring):
+        result.append(command)
+    else:
+        result.extend(command)
+
+    return ' '.join(result)
 
 
 def start(command, **kwargs):
-    # XXX should not really need 'name' there: can still figure it out
-    # from command + environment vars starting with GEVENT_ and GEVENTARES_
-    name = kwargs.pop('name', None) or getname(command)
     timeout = kwargs.pop('timeout', None)
     preexec_fn = None
     if not os.environ.get('DO_NOT_SETPGRP'):
         preexec_fn = getattr(os, 'setpgrp', None)
     env = kwargs.pop('env', None)
     setenv = kwargs.pop('setenv', None) or {}
+    name = getname(command, env=env, setenv=setenv)
     if preexec_fn is not None:
         setenv['DO_NOT_SETPGRP'] = '1'
     if setenv:
@@ -119,8 +132,10 @@ def start(command, **kwargs):
         else:
             env = os.environ.copy()
         env.update(setenv)
+
     log('+ %s', name)
     popen = Popen(command, preexec_fn=preexec_fn, env=env, **kwargs)
+    popen.name = name
     popen.setpgrp_enabled = preexec_fn is not None
     if timeout is not None:
         popen._killer = spawn_later(timeout, kill, popen)
@@ -132,9 +147,10 @@ def start(command, **kwargs):
 
 class RunResult(object):
 
-    def __init__(self, code, output=None):
+    def __init__(self, code, output=None, name=None):
         self.code = code
         self.output = output
+        self.name = name
 
     def __nonzero__(self):
         return bool(self.code)
@@ -144,13 +160,13 @@ class RunResult(object):
 
 
 def run(command, **kwargs):
-    name = kwargs.get('name') or getname(command)
     buffer_output = kwargs.pop('buffer_output', BUFFER_OUTPUT)
     if buffer_output:
         assert 'stdout' not in kwargs and 'stderr' not in kwargs, kwargs
         kwargs['stderr'] = subprocess.STDOUT
         kwargs['stdout'] = subprocess.PIPE
     popen = start(command, **kwargs)
+    name = popen.name
     try:
         time_start = time()
         out, err = popen.communicate()
@@ -177,7 +193,7 @@ def run(command, **kwargs):
         log('- %s [took %.1fs]', name, took)
     if took >= MIN_RUNTIME:
         runtimelog.append((-took, name))
-    return RunResult(result, out)
+    return RunResult(result, out, name)
 
 
 def expected_match(name, expected):