Merge pull request #655 from gevent/coverage

Support coverage.py in the testrunner and coveralls.io

.gitignore

@@ -9,6 +9,10 @@ gevent/libev
 Makefile.ext
 MANIFEST
 
+greentest/.coverage\.*
+greentest/htmlcov
+greentest/.coverage
+
 doc/changelog.rst
 doc/_build
 doc/__pycache__

.travis.yml

@@ -21,7 +21,7 @@ install:
     # means that installing cython on cpython 3.5 cannot use the wheel cache, which
     # means that the LINT=true case is quite slow (2.5 minutes). If this takes very long to fix,
     # we might want to do some refactoring to move installation into the makefile or a script.
-    - travis_retry pip install -U tox cython greenlet pep8 pyflakes
+    - travis_retry pip install -U tox cython greenlet pep8 pyflakes "coverage>=4.0b3" "coveralls>=1.0b1"
 script:
     - if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $LINT == true ]]; then python setup.py develop && make travis_test_linters; elif [[ $LINT == false && $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then python setup.py develop && make fulltoxtest; elif [[ $LINT == false ]]; then python setup.py develop && make fulltoxtest; fi
 notifications:

Makefile

@@ -56,12 +56,18 @@ travistest:
 
 	${PYTHON} -c 'import greenlet; print(greenlet, greenlet.__version__)'
 
-	${PYTHON} setup.py install
+# develop, not install, so that coverage doesn't think it's part of the stdlib
+	${PYTHON} setup.py develop
 	make bench
 
+# combine after each step so we don't lose anything (append is the default). This also
+# moves the coverage file from greentest to the root.
 	cd greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config ../known_failures.py
+	coverage combine greentest/
 	cd greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config ../known_failures.py --ignore tests_that_dont_use_resolver.txt
+	coverage combine greentest/
 	cd greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config ../known_failures.py `grep -l subprocess test_*.py`
+	coverage combine greentest/
 
 toxtest:
 	cd greentest && GEVENT_RESOLVER=thread python testrunner.py --config ../known_failures.py
@@ -91,7 +97,9 @@ travis_cpython:
 
 travis_test_linters:
 	make lint
-	make leaktest
+	GEVENTTEST_COVERAGE=1 make leaktest
+	coverage combine
+	coveralls
 
 
 .PHONY: clean all doc pep8 whitespace pyflakes lint travistest travis

greentest/.coveragerc

+[run]
+concurrency = gevent
+parallel = True
+source = gevent
+omit = test_*

greentest/coveragesite/sitecustomize.py

+# When testrunner.py is invoked with --coverage, it puts this first
+# on the path as per http://coverage.readthedocs.org/en/coverage-4.0b3/subprocess.html.
+# Note that this disables other sitecustomize.py files.
+import coverage
+coverage.process_startup()

greentest/testrunner.py

@@ -23,6 +23,18 @@ RUN_ALONE = [
     'test__examples.py'
 ]
 
+# tests that can't be run when coverage is enabled
+IGNORE_COVERAGE = [
+    # Hangs forever
+    'test__threading_vs_settrace.py',
+    # XXX ?
+    'test__issue302monkey.py'
+]
+
+if os.environ.get('TRAVIS'):
+    # Flaky on travis, unknown why
+    IGNORE_COVERAGE.append("test_subprocess.py")
+
 
 def run_many(tests, expected=(), failfast=False):
     global NWORKERS
@@ -106,15 +118,24 @@ def run_many(tests, expected=(), failfast=False):
     report(total, failed, passed, took=time.time() - start, expected=expected)
 
 
-def discover(tests=None, ignore=None):
+def discover(tests=None, ignore=(), coverage=False):
     if isinstance(ignore, six.string_types):
         ignore = load_list_from_file(ignore)
 
-    ignore = set(ignore or [])
+    ignore = set(ignore or ())
+    if coverage:
+        ignore.update(IGNORE_COVERAGE)
 
     if not tests:
         tests = set(glob.glob('test_*.py')) - set(['test_support.py'])
+    else:
+        tests = set(tests)
+
     if ignore:
+        # Always ignore the designated list, even if tests were specified
+        # on the command line. This fixes a nasty interaction with test__threading_vs_settrace.py
+        # being run under coverage when 'grep -l subprocess test*py' is used to list the tests
+        # to run.
         tests -= ignore
     tests = sorted(tests)
 
@@ -128,14 +149,15 @@ def discover(tests=None, ignore=None):
             # try to decode those as ASCII, which fails with UnicodeDecodeError.
             # Thus, be sure to open and compare in binary mode.
             contents = f.read()
-        if b'TESTRUNNER' in contents:
+        if b'TESTRUNNER' in contents: # test__monkey_patching.py
             module = __import__(filename.rsplit('.', 1)[0])
             for cmd, options in module.TESTRUNNER():
                 if remove_options(cmd)[-1] in ignore:
                     continue
                 to_process.append((cmd, options))
         else:
-            to_process.append(([sys.executable, '-u', filename], default_options.copy()))
+            cmd = [sys.executable, '-u', filename]
+            to_process.append((cmd, default_options.copy()))
 
     return to_process
 
@@ -241,13 +263,25 @@ def main():
     parser.add_option('--full', action='store_true')
     parser.add_option('--config')
     parser.add_option('--failfast', action='store_true')
+    parser.add_option("--coverage", action="store_true")
     options, args = parser.parse_args()
     FAILING_TESTS = []
+    coverage = False
+    if options.coverage or os.environ.get("GEVENTTEST_COVERAGE"):
+        coverage = True
+        # NOTE: This must be run from the greentest directory
+        os.environ['COVERAGE_PROCESS_START'] = os.path.abspath(".coveragerc")
+        os.environ['PYTHONPATH'] = os.path.abspath("coveragesite") + os.pathsep + os.environ.get("PYTHONPATH", "")
+        # We change directory often, use an absolute path to keep all the
+        # coverage files (which will have distinct suffixes because of parallel=true in .coveragerc
+        # in this directory; makes them easier to combine and use with coverage report)
+        os.environ['COVERAGE_FILE'] = os.path.abspath(".") + os.sep + ".coverage"
+        print("Enabling coverage to", os.environ['COVERAGE_FILE'])
     if options.config:
         config = {}
         six.exec_(open(options.config).read(), config)
         FAILING_TESTS = config['FAILING_TESTS']
-    tests = discover(args, options.ignore)
+    tests = discover(args, options.ignore, coverage)
     if options.discover:
         for cmd, options in tests:
             print(util.getname(cmd, env=options.get('env'), setenv=options.get('setenv')))

known_failures.py

@@ -8,6 +8,7 @@ import struct
 
 
 LEAKTEST = os.getenv('GEVENTTEST_LEAKCHECK')
+COVERAGE = os.getenv("COVERAGE_PROCESS_START")
 PYPY = hasattr(sys, 'pypy_version_info')
 PY3 = sys.version_info[0] >= 3
 PY26 = sys.version_info[0] == 2 and sys.version_info[1] == 6
@@ -156,6 +157,16 @@ if PY3:
             'FLAKY test__socket.py',
         ]
 
+if COVERAGE:
+    # The gevent concurrency plugin tends to slow things
+    # down and get us past our default timeout value. These
+    # tests in particular are sensitive to it
+    FAILING_TESTS += [
+        'FLAKY test__issue302monkey.py',
+        'FLAKY test__example_portforwarder.py',
+        'FLAKY test__threading_vs_settrace.py',
+    ]
+
 FAILING_TESTS = [x.strip() for x in FAILING_TESTS if x.strip()]
 
 

tox.ini

@@ -6,6 +6,7 @@ envlist =
 deps =
     greenlet
     cython
+    coverage >= 4.0b3
 whitelist_externals =
     *
 commands =