Support for coverage.py in testrunner. Experimental support in the lint check.

.gitignore

@@ -9,6 +9,10 @@ gevent/libev
 Makefile.ext
 MANIFEST
 
+greentest/.coverage\.*
+greentest/htmlcov
+greentest/.coverage
+
 doc/changelog.rst
 doc/_build
 doc/__pycache__

.travis.yml

@@ -21,7 +21,7 @@ install:
     # means that installing cython on cpython 3.5 cannot use the wheel cache, which
     # means that the LINT=true case is quite slow (2.5 minutes). If this takes very long to fix,
     # we might want to do some refactoring to move installation into the makefile or a script.
-    - travis_retry pip install -U tox cython greenlet pep8 pyflakes
+    - travis_retry pip install -U tox cython greenlet pep8 pyflakes "coverage>=4.0b3" coveralls
 script:
     - if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $LINT == true ]]; then python setup.py develop && make travis_test_linters; elif [[ $LINT == false && $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then python setup.py develop && make fulltoxtest; elif [[ $LINT == false ]]; then python setup.py develop && make fulltoxtest; fi
 notifications:

Makefile

@@ -92,6 +92,12 @@ travis_cpython:
 travis_test_linters:
 	make lint
 	make leaktest
+# XXX Can we do this here? It would be nice to get these reports, but
+# first, it may cause issues with the leak tests. And second, 'coverage combine'
+# chokes if a datafile hasn't been fully written, which can happen when
+# our timeout killer kills a script (and they do take longer under coverage)
+	coverage combine
+	coveralls
 
 
 .PHONY: clean all doc pep8 whitespace pyflakes lint travistest travis

greentest/.coveragerc

+[run]
+concurrency = gevent
+parallel = True
+source = gevent
+omit = test_*

greentest/coveragesite/sitecustomize.py

+# When testrunner.py is invoked with --coverage, it puts this first
+# on the path as per http://coverage.readthedocs.org/en/coverage-4.0b3/subprocess.html.
+# Note that this disables other sitecustomize.py files.
+import coverage
+coverage.process_startup()

greentest/testrunner.py

@@ -23,6 +23,14 @@ RUN_ALONE = [
     'test__examples.py'
 ]
 
+# tests that can't be run when coverage is enabled
+IGNORE_COVERAGE = [
+    # Hangs forever
+    'test__threading_vs_settrace.py',
+    # XXX ?
+    'test__issue302monkey.py'
+]
+
 
 def run_many(tests, expected=(), failfast=False):
     global NWORKERS
@@ -106,11 +114,13 @@ def run_many(tests, expected=(), failfast=False):
     report(total, failed, passed, took=time.time() - start, expected=expected)
 
 
-def discover(tests=None, ignore=None):
+def discover(tests=None, ignore=None, coverage=False):
     if isinstance(ignore, six.string_types):
         ignore = load_list_from_file(ignore)
 
     ignore = set(ignore or [])
+    if coverage:
+        ignore.update(IGNORE_COVERAGE)
 
     if not tests:
         tests = set(glob.glob('test_*.py')) - set(['test_support.py'])
@@ -128,14 +138,15 @@ def discover(tests=None, ignore=None):
             # try to decode those as ASCII, which fails with UnicodeDecodeError.
             # Thus, be sure to open and compare in binary mode.
             contents = f.read()
-        if b'TESTRUNNER' in contents:
+        if b'TESTRUNNER' in contents: # test__monkey_patching.py
             module = __import__(filename.rsplit('.', 1)[0])
             for cmd, options in module.TESTRUNNER():
                 if remove_options(cmd)[-1] in ignore:
                     continue
                 to_process.append((cmd, options))
         else:
-            to_process.append(([sys.executable, '-u', filename], default_options.copy()))
+            cmd = [sys.executable, '-u', filename]
+            to_process.append((cmd, default_options.copy()))
 
     return to_process
 
@@ -241,13 +252,22 @@ def main():
     parser.add_option('--full', action='store_true')
     parser.add_option('--config')
     parser.add_option('--failfast', action='store_true')
+    parser.add_option("--coverage", action="store_true")
     options, args = parser.parse_args()
     FAILING_TESTS = []
+    if options.coverage:
+        # NOTE: This must be run from the greentest directory
+        os.environ['COVERAGE_PROCESS_START'] = os.path.abspath(".coveragerc")
+        os.environ['PYTHONPATH'] = os.path.abspath("coveragesite") + os.pathsep + os.environ.get("PYTHONPATH", "")
+        # We change directory often, use an absolute path to keep all the
+        # coverage files (which will have distinct suffixes because of parallel=true in .coveragerc
+        # in this directory; makes them easier to combine and use with coverage report)
+        os.environ['COVERAGE_FILE'] = os.path.abspath(".") + os.sep + ".coverage"
     if options.config:
         config = {}
         six.exec_(open(options.config).read(), config)
         FAILING_TESTS = config['FAILING_TESTS']
-    tests = discover(args, options.ignore)
+    tests = discover(args, options.ignore, options.coverage)
     if options.discover:
         for cmd, options in tests:
             print(util.getname(cmd, env=options.get('env'), setenv=options.get('setenv')))

known_failures.py

@@ -8,6 +8,7 @@ import struct
 
 
 LEAKTEST = os.getenv('GEVENTTEST_LEAKCHECK')
+COVERAGE = os.getenv("COVERAGE_PROCESS_START")
 PYPY = hasattr(sys, 'pypy_version_info')
 PY3 = sys.version_info[0] >= 3
 PY26 = sys.version_info[0] == 2 and sys.version_info[1] == 6
@@ -156,6 +157,15 @@ if PY3:
             'FLAKY test__socket.py',
         ]
 
+if COVERAGE:
+    # The gevent concurrency plugin tends to slow things
+    # down and get us past our default timeout value. These
+    # tests in particular are sensitive to it
+    FAILING_TESTS += [
+        'FLAKY test__issue302monkey.py',
+        'FLAKY test__example_portforwarder.py',
+    ]
+
 FAILING_TESTS = [x.strip() for x in FAILING_TESTS if x.strip()]
 
 

tox.ini

@@ -6,6 +6,7 @@ envlist =
 deps =
     greenlet
     cython
+    coverage >= 4.0b3
 whitelist_externals =
     *
 commands =