many improvements to tests

- testrunner.py: implement --full and --discover
- setup.py now recognizes GEVENTSETUP_EV_VERIFY env var which is passed as EV_VERIFY to make
- relevant env vars are now logged long test names
- add support for expected failures
- allow passing tests on the command line in test___monkey_patching.py
- show longest running tests before final status
- make exit status report number of failing tests
- greentest/util.py: add 'setenv' and 'name' arguments to run()
- add tests_that_dont_use_resolver.txt
- add expected_failures.txt
- fix test___monkey_patching.py to do clean up via atexit

.travis.yml

@@ -5,19 +5,16 @@ python:
   - "2.7"
 env:
   -
-  - GEVENT_RESOLVER=ares
+  - GEVENTSETUP_EV_VERIFY=3
   - GEVENT_FILE=thread
-  - GEVENT_BACKEND=select
-  - GEVENT_BACKEND=poll
+  - GEVENT_BACKEND=select GEVENTSETUP_EV_VERIFY=3
+  - GEVENT_BACKEND=poll GEVENTSETUP_EV_VERIFY=3
   - GEVENT_BACKEND=signalfd
   - GEVENT_BACKEND=nosigmask
-  - GEVENT_RESOLVER=ares GEVENT_BACKEND=signalfd
-  - GEVENT_RESOLVER=ares GEVENT_FILE=thread
-  - NWORKERS=1 TESTS="*dns*.py" GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8
+  - GEVENT_BACKEND=noinotify
 matrix:
-  include:
-    - python: 2.7
-      env: TESTS=xtest_pep8.py INSTALL_PEP8=1
+ allow_failures:
+   GEVENT_BACKEND=signalfd
 install:
  - deactivate
  - export VER=$TRAVIS_PYTHON_VERSION
@@ -34,14 +31,14 @@ install:
  - sudo dpkg -i python$VER-psycopg2_2.4.5_i386.deb
  - sudo dpkg -i python$VER-pysendfile_2.0.0_i386.deb
  - tar -xf web.py-0.37.tar.gz && cd web.py-0.37 && sudo $PYTHON setup.py -q install && cd -
- - if [[ $INSTALL_PEP8 == '1' ]]; then sudo pip install --use-mirrors -q pep8; fi
+ - if [[ $VER == '2.7' ]]; then sudo pip install --use-mirrors -q pep8; fi
  - cython --version
  - $PYTHON -c 'import greenlet; print greenlet, greenlet.__version__; import psycopg2; print psycopg2, psycopg2.__version__; import web; print web, web.__version__'
  - "echo `date`: Started installing gevent"
  - sudo $PYTHON setup.py install
 script:
  - "echo `date`: Started testing"
- - cd greentest && $PYTHON testrunner.py $TESTS
+ - cd greentest && $PYTHON testrunner.py --full --expected expected_failures.txt
  - "echo `date`: Finished testing"
 notifications:
   email:
@@ -49,3 +46,4 @@ notifications:
       - denis.bilenko@gmail.com
     on_success: change
     on_failure: change
+

greentest/expected_failures.txt

+GEVENT_RESOLVER=ares test__socket_dns.py
+GEVENT_RESOLVER=ares test__socket_dns6.py

greentest/patched_tests_setup.py

 import sys
+import os
 import re
 
 # By default, test cases are expected to switch and emit warnings if there was none
@@ -133,6 +134,14 @@ disabled_tests = \
     , 'test_thread.ThreadRunningTests.test__count'
 ]
 
+# if 'signalfd' in os.environ.get('GEVENT_BACKEND', ''):
+#     # tests that don't interact well with signalfd
+#     disabled_tests.extend([
+#         'test_signal.SiginterruptTest.test_siginterrupt_off',
+#         'test_socketserver.SocketServerTest.test_ForkingTCPServer',
+#         'test_socketserver.SocketServerTest.test_ForkingUDPServer',
+#         'test_socketserver.SocketServerTest.test_ForkingUnixStreamServer'])
+
 
 def disable_tests_in_source(source, name):
     my_disabled_tests = [x for x in disabled_tests if x.startswith(name + '.')]

greentest/test___monkey_patching.py

@@ -2,9 +2,10 @@ import sys
 import os
 import glob
 import util
+import atexit
 
 
-TIMEOUT = 120
+TIMEOUT = 45
 directory = '%s.%s' % sys.version_info[:2]
 version = '%s.%s.%s' % sys.version_info[:3]
 
@@ -14,23 +15,27 @@ def TESTRUNNER(tests=None):
     if preferred_version != version:
         util.log('WARNING: The tests in %s/ are from version %s and your Python is %s', directory, preferred_version, version)
 
-    env = os.environ.copy()
-    env['PYTHONPATH'] = os.getcwd() + ':' + os.environ.get('PYTHONPATH', '')
-
     if not tests:
         tests = sorted(glob.glob('%s/test_*.py' % directory))
 
+    PYTHONPATH = (os.getcwd() + ':' + os.environ.get('PYTHONPATH', '')).rstrip(':')
+
     tests = [os.path.basename(x) for x in tests]
-    options = {'cwd': directory, 'env': env}
+    options = {'cwd': directory,
+               'timeout': TIMEOUT,
+               'setenv': {'PYTHONPATH': PYTHONPATH}}
+
+    if tests:
+        atexit.register(os.system, 'rm -f */@test*_tmp')
 
     for filename in tests:
-        yield directory + '/' + filename, [sys.executable, '-u', '-m', 'monkey_test', filename], options
-        yield directory + '/' + filename + '/Event', [sys.executable, '-u', '-m', 'monkey_test', '--Event', filename], options
+        yield directory + '/' + filename, [sys.executable, '-u', '-m', 'monkey_test', filename], options.copy()
+        yield directory + '/' + filename + '/Event', [sys.executable, '-u', '-m', 'monkey_test', '--Event', filename], options.copy()
 
 
 def main():
     import testrunner
-    return testrunner.run_many(TESTRUNNER())
+    return testrunner.run_many(TESTRUNNER(sys.argv[1:]))
 
 
 if __name__ == '__main__':

greentest/testrunner.py

@@ -5,7 +5,7 @@ from gevent import monkey; monkey.patch_all()
 import sys
 import os
 import glob
-import time
+from time import time
 
 from gevent.pool import Pool
 import util
@@ -16,50 +16,57 @@ NWORKERS = int(os.environ.get('NWORKERS') or 8)
 pool = None
 
 
-def info():
-    lastmsg = None
-    while True:
-        gevent.sleep(10, ref=False)
-        if pool:
-            msg = '# Currently running: %s: %s' % (len(pool), ', '.join(x.name for x in pool))
-            if msg != lastmsg:
-                lastmsg = msg
-                util.log(msg)
-
-
 def spawn(*args, **kwargs):
     g = pool.spawn(*args, **kwargs)
     g.link_exception(lambda *args: sys.exit('Internal error'))
     return g
 
 
-def run_many(tests):
+def process_test(name, cmd, options):
+    options = options or {}
+    setenv = options.get('setenv', {}).copy()
+    setenv.pop('PYTHONPATH', '')
+    environ = options.get('env')
+    if environ is None:
+        environ = os.environ.copy()
+    for key, value in environ.items():
+        if key.startswith('GEVENT_') or key.startswith('GEVENTARES_'):
+            if key not in setenv:
+                setenv[key] = value
+    env_str = ' '.join('%s=%s' % x for x in setenv.items())
+    if env_str and env_str not in name:
+        name = env_str + ' ' + name
+    return name, cmd, options
+
+
+def process_tests(tests):
+    return [process_test(name, cmd, options) for (name, cmd, options) in tests]
+
+
+def run_many(tests, expected=None):
     global NWORKERS, pool
-    start = time.time()
+    start = time()
     total = 0
     failed = {}
 
-    tests = list(tests)
+    tests = process_tests(tests)
     NWORKERS = min(len(tests), NWORKERS)
     pool = Pool(NWORKERS)
     util.BUFFER_OUTPUT = NWORKERS > 1
 
     def run_one(name, cmd, **kwargs):
-        result = util.run(cmd, **kwargs)
+        result = util.run(cmd, name=name, **kwargs)
         if result:
             # the tests containing AssertionError might have failed because
             # we spawned more workers than CPUs
             # we therefore will retry them sequentially
             failed[name] = [cmd, kwargs, 'AssertionError' in (result.output or '')]
 
-    if NWORKERS > 1:
-        gevent.spawn(info)
-
     try:
         try:
             for name, cmd, options in tests:
                 total += 1
-                spawn(run_one, name, cmd, **options).name = ' '.join(cmd)
+                spawn(run_one, name, cmd, **options)
             gevent.run()
         except KeyboardInterrupt:
             try:
@@ -67,7 +74,7 @@ def run_many(tests):
                     util.log('Waiting for currently running to finish...')
                     pool.join()
             except KeyboardInterrupt:
-                util.report(total, failed, exit=False, took=time.time() - start)
+                util.report(total, failed, exit=False, took=time() - start, expected=expected)
                 util.log('(partial results)\n')
                 raise
     except:
@@ -78,25 +85,31 @@ def run_many(tests):
     failed_then_succeeded = []
 
     if NWORKERS > 1 and toretry:
-        util.log('\nWill re-try %s failed tests without concurrency:\n- %s\n', len(toretry), '\n- '.join(toretry))
+        util.log('\nWill retry %s failed tests without concurrency:\n- %s\n', len(toretry), '\n- '.join(toretry))
         for name, (cmd, kwargs, _ignore) in failed.items():
-            if not util.run(cmd, buffer_output=False, **kwargs):
+            if not util.run(cmd, name=name, buffer_output=False, **kwargs):
                 failed.pop(name)
                 failed_then_succeeded.append(name)
 
-    util.report(total, failed, took=time.time() - start)
-
     if failed_then_succeeded:
         util.log('\n%s tests failed during concurrent run but succeeded when ran sequentially:', len(failed_then_succeeded))
         util.log('- ' + '\n- '.join(failed_then_succeeded))
+
+    util.log('gevent version %s from %s', gevent.__version__, gevent.__file__)
+    util.report(total, failed, took=time() - start, expected=expected)
     assert not pool, pool
 
-    os.system('rm -f */@test*_tmp')
 
+def discover(tests=None, ignore=None):
+    if isinstance(ignore, basestring):
+        ignore = load_list_from_file(ignore)
+
+    ignore = set(ignore or [])
 
-def discover(tests):
     if not tests:
         tests = set(glob.glob('test_*.py')) - set(['test_support.py'])
+        if ignore:
+            tests -= ignore
         tests = sorted(tests)
 
     to_process = []
@@ -106,15 +119,67 @@ def discover(tests):
         if 'TESTRUNNER' in open(filename).read():
             module = __import__(filename.rsplit('.', 1)[0])
             for name, cmd, options in module.TESTRUNNER():
+                if remove_options(cmd)[-1] in ignore:
+                    continue
                 to_process.append((filename + ' ' + name, cmd, options))
         else:
-            to_process.append((filename, [sys.executable, '-u', filename], default_options))
+            to_process.append((filename, [sys.executable, '-u', filename], default_options.copy()))
 
     return to_process
 
 
+def remove_options(lst):
+    return [x for x in lst if x and not x.startswith('-')]
+
+
+def load_list_from_file(filename):
+    result = []
+    if filename:
+        for x in open(filename):
+            x = x.split('#', 1)[0].strip()
+            if x:
+                result.append(x)
+    return result
+
+
+def full(args=None):
+    tests = []
+
+    for setenv, ignore in [('GEVENT_RESOLVER=thread', None),
+                           ('GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8', 'tests_that_dont_use_resolver.txt')]:
+        setenv = dict(x.split('=') for x in setenv.split())
+        for name, cmd, options in discover(args, ignore=ignore):
+            my_setenv = options.get('setenv', {})
+            my_setenv.update(setenv)
+            options['setenv'] = my_setenv
+            tests.append((name, cmd, options))
+
+    if sys.version_info[:2] == (2, 7):
+        tests.append(('xtest_pep8.py', [sys.executable, '-u', 'xtest_pep8.py'], None))
+
+    return tests
+
+
 def main():
-    run_many(discover(sys.argv[1:]))
+    import optparse
+    parser = optparse.OptionParser()
+    parser.add_option('--ignore')
+    parser.add_option('--discover', action='store_true')
+    parser.add_option('--full', action='store_true')
+    parser.add_option('--expected')
+    options, args = parser.parse_args()
+    options.expected = load_list_from_file(options.expected)
+    if options.full:
+        assert options.ignore is None, '--ignore and --full are not compatible'
+        tests = full(args)
+    else:
+        tests = discover(args, options.ignore)
+    if options.discover:
+        for name, cmd, options in process_tests(tests):
+            print '%s: %s' % (name, ' '.join(cmd))
+        print '%s tests found.' % len(tests)
+    else:
+        run_many(tests, expected=options.expected)
 
 
 if __name__ == '__main__':

greentest/tests_that_dont_use_resolver.txt

+test__all__.py
+#uses socket test__api.py
+test__api_timeout.py
+test__ares_host_result.py
+test_ares_timeout.py  # explicitly uses ares resolver
+# uses socket test__backdoor.py
+test_close_backend_fd.py
+test__core_async.py
+test__core_callback.py
+test__core_loop_run.py
+test__core.py
+test__core_stat.py
+test__core_timer.py
+test__core_watcher.py
+test__destroy.py
+# uses socket test__doctests.py
+test__environ.py
+test__event.py
+# uses socket test__example_echoserver.py
+# uses socket test__example_portforwarder.py
+# uses socket test___example_servers.py
+# uses bunch of things test__examples.py
+# uses socket test__example_udp_client.py
+# uses socket test__example_udp_server.py
+test__exc_info.py
+#test__execmodules.py
+test__fileobject.py
+# uses socket test__greenio.py
+test__GreenletExit.py
+test__greenlet.py
+test__greenletset.py
+# uses socket test__greenness.py
+test_hub_join.py
+test_hub_join_timeout.py
+# uses socket test__hub.py
+test_issue112.py
+test__joinall.py
+test__local.py
+test__loop_callback.py
+test__memleak.py
+# uses lots of things test___monkey_patching.py
+test__monkey.py
+test__order.py
+test__os.py
+test__pool.py
+# uses socket test__pywsgi.py
+test__queue.py
+test_queue.py
+# uses socket test__refcount.py
+test__select.py
+test__semaphore.py
+# uses socket test__server.py
+# test__server_pywsgi.py
+test__signal.py
+# uses socket test__socket_close.py
+# test__socket_dns6.py
+# test__socket_dns.py
+# test__socket_errors.py
+# test__socket.py
+# test__socket_ssl.py
+# test__socket_timeout.py
+test__subprocess_interrupted.py
+test__subprocess.py
+test__systemerror.py
+test_threading_2.py
+test__threading_patched_local.py
+test__threading_vs_settrace.py
+test__threadpool.py
+test__timeout.py
+
+# monkey patched standard tests:
+test_queue.py
+test_select.py
+test_signal.py
+test_subprocess.py
+test_threading_local.py
+test_threading.py
+test_thread.py

greentest/util.py

@@ -3,6 +3,7 @@ import sys
 import os
 import traceback
 import unittest
+from datetime import timedelta
 from time import time
 from gevent import subprocess, sleep, spawn_later
 
@@ -24,7 +25,10 @@ class Popen(subprocess.Popen):
 
 def log(message, *args):
     try:
-        string = message % args
+        if args:
+            string = message % args
+        else:
+            string = message
     except Exception:
         traceback.print_exc()
         try:
@@ -98,16 +102,24 @@ def getname(command):
 
 
 def start(command, **kwargs):
+    # XXX should not really need 'name' there: can still figure it out
+    # from command + environment vars starting with GEVENT_ and GEVENTARES_
+    name = kwargs.pop('name', None) or getname(command)
     timeout = kwargs.pop('timeout', None)
     preexec_fn = None
     if not os.environ.get('DO_NOT_SETPGRP'):
         preexec_fn = getattr(os, 'setpgrp', None)
     env = kwargs.pop('env', None)
+    setenv = kwargs.pop('setenv', None) or {}
     if preexec_fn is not None:
-        if env is None:
+        setenv['DO_NOT_SETPGRP'] = '1'
+    if setenv:
+        if env:
+            env = env.copy()
+        else:
             env = os.environ.copy()
-        env['DO_NOT_SETPGRP'] = '1'
-    log('+ %s', getname(command))
+        env.update(setenv)
+    log('+ %s', name)
     popen = Popen(command, preexec_fn=preexec_fn, env=env, **kwargs)
     popen.setpgrp_enabled = preexec_fn is not None
     if timeout is not None:
@@ -132,7 +144,7 @@ class RunResult(object):
 
 
 def run(command, **kwargs):
-    name = getname(command)
+    name = kwargs.get('name') or getname(command)
     buffer_output = kwargs.pop('buffer_output', BUFFER_OUTPUT)
     if buffer_output:
         assert 'stdout' not in kwargs and 'stderr' not in kwargs, kwargs
@@ -156,6 +168,8 @@ def run(command, **kwargs):
         out = out.strip()
         if out:
             out = '  ' + out.replace('\n', '\n  ')
+            out = out.rstrip()
+            out += '\n'
             log('| %s\n%s', name, out)
     if result:
         log('! %s [code %s] [took %.1fs]', name, result, took)
@@ -166,25 +180,52 @@ def run(command, **kwargs):
     return RunResult(result, out)
 
 
-def report(total, failed, exit=True, took=None):
+def expected_match(name, expected):
+    name_parts = set(name.split())
+    assert name_parts
+    for item in expected:
+        item = set(item.split())
+        if item.issubset(name_parts):
+            return True
+
+
+def format_seconds(seconds):
+    if seconds < 20:
+        return '%.1fs' % seconds
+    seconds = str(timedelta(seconds=round(seconds)))
+    if seconds.startswith('0:'):
+        seconds = seconds[2:]
+    return seconds
+
+
+def report(total, failed, exit=True, took=None, expected=None):
+    if runtimelog:
+        log('\nLongest-running tests:')
+        runtimelog.sort()
+        length = len('%.1f' % -runtimelog[0][0])
+        frmt = '%' + str(length) + '.1f seconds: %s'
+        for delta, name in runtimelog[:5]:
+            log(frmt, -delta, name)
     if took:
-        took = ' in %.1fs' % took
+        took = ' in %s' % format_seconds(took)
     else:
         took = ''
+
+    error_count = 0
     if failed:
-        log('\n%s/%s tests failed%s\n- %s', len(failed), total, took, '\n- '.join(failed))
+        log('\n%s/%s tests failed%s', len(failed), total, took)
+        expected = set(expected or [])
+        for name in failed:
+            if expected_match(name, expected):
+                log('- %s (expected)', name)
+            else:
+                log('- %s', name)
+                error_count += 1
     else:
         log('\n%s tests passed%s', total, took)
-    if runtimelog:
-        log('\nLongest-running tests:')
-        runtimelog.sort()
-        length = len('%.1f' % -runtimelog[0][0])
-        frmt = '%' + str(length) + '.1f seconds: %s'
-        for took, name in runtimelog[:5]:
-            log(frmt, -took, name)
     if exit:
-        if failed:
-            sys.exit(1)
+        if error_count:
+            sys.exit(min(100, error_count))
         if total <= 0:
             sys.exit('No tests found.')
 

setup.py

@@ -18,6 +18,9 @@ from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatfo
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, IOError)
 
 
+# XXX make all env variables that setup.py parses start with GEVENTSETUP_
+
+
 __version__ = re.search("__version__\s*=\s*'(.*)'", open('gevent/__init__.py').read(), re.M).group(1)
 assert __version__
 
@@ -173,6 +176,8 @@ if LIBEV_EMBED:
     CORE.configure = configure_libev
     if sys.platform == "darwin":
         os.environ["CFLAGS"] = ("%s %s" % (os.environ.get("CFLAGS", ""), "-U__llvm__")).lstrip()
+    if os.environ.get('GEVENTSETUP_EV_VERIFY') is not None:
+        CORE.define_macros.append(('EV_VERIFY', os.environ['GEVENTSETUP_EV_VERIFY']))
 else:
     CORE.libraries.append('ev')