Commit 4144a2f6 authored by Jason Madden's avatar Jason Madden

Merge pull request #655 from gevent/coverage

Support coverage.py in the testrunner and coveralls.io
parents cc2d3076 45426641
......@@ -9,6 +9,10 @@ gevent/libev
Makefile.ext
MANIFEST
greentest/.coverage\.*
greentest/htmlcov
greentest/.coverage
doc/changelog.rst
doc/_build
doc/__pycache__
......
......@@ -21,7 +21,7 @@ install:
# means that installing cython on cpython 3.5 cannot use the wheel cache, which
# means that the LINT=true case is quite slow (2.5 minutes). If this takes very long to fix,
# we might want to do some refactoring to move installation into the makefile or a script.
- travis_retry pip install -U tox cython greenlet pep8 pyflakes
- travis_retry pip install -U tox cython greenlet pep8 pyflakes "coverage>=4.0b3" "coveralls>=1.0b1"
script:
- if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $LINT == true ]]; then python setup.py develop && make travis_test_linters; elif [[ $LINT == false && $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then python setup.py develop && make fulltoxtest; elif [[ $LINT == false ]]; then python setup.py develop && make fulltoxtest; fi
notifications:
......
......@@ -56,12 +56,18 @@ travistest:
${PYTHON} -c 'import greenlet; print(greenlet, greenlet.__version__)'
${PYTHON} setup.py install
# develop, not install, so that coverage doesn't think it's part of the stdlib
${PYTHON} setup.py develop
make bench
# combine after each step so we don't lose anything (append is the default). This also
# moves the coverage file from greentest to the root.
cd greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config ../known_failures.py
coverage combine greentest/
cd greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config ../known_failures.py --ignore tests_that_dont_use_resolver.txt
coverage combine greentest/
cd greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config ../known_failures.py `grep -l subprocess test_*.py`
coverage combine greentest/
toxtest:
cd greentest && GEVENT_RESOLVER=thread python testrunner.py --config ../known_failures.py
......@@ -91,7 +97,9 @@ travis_cpython:
travis_test_linters:
make lint
make leaktest
GEVENTTEST_COVERAGE=1 make leaktest
coverage combine
coveralls
.PHONY: clean all doc pep8 whitespace pyflakes lint travistest travis
[run]
concurrency = gevent
parallel = True
source = gevent
omit = test_*
# When testrunner.py is invoked with --coverage, it puts this first
# on the path as per http://coverage.readthedocs.org/en/coverage-4.0b3/subprocess.html.
# Note that this disables other sitecustomize.py files.
import coverage
coverage.process_startup()
......@@ -23,6 +23,18 @@ RUN_ALONE = [
'test__examples.py'
]
# tests that can't be run when coverage is enabled
IGNORE_COVERAGE = [
# Hangs forever
'test__threading_vs_settrace.py',
# XXX ?
'test__issue302monkey.py'
]
if os.environ.get('TRAVIS'):
# Flaky on travis, unknown why
IGNORE_COVERAGE.append("test_subprocess.py")
def run_many(tests, expected=(), failfast=False):
global NWORKERS
......@@ -106,17 +118,26 @@ def run_many(tests, expected=(), failfast=False):
report(total, failed, passed, took=time.time() - start, expected=expected)
def discover(tests=None, ignore=None):
def discover(tests=None, ignore=(), coverage=False):
if isinstance(ignore, six.string_types):
ignore = load_list_from_file(ignore)
ignore = set(ignore or [])
ignore = set(ignore or ())
if coverage:
ignore.update(IGNORE_COVERAGE)
if not tests:
tests = set(glob.glob('test_*.py')) - set(['test_support.py'])
if ignore:
tests -= ignore
tests = sorted(tests)
else:
tests = set(tests)
if ignore:
# Always ignore the designated list, even if tests were specified
# on the command line. This fixes a nasty interaction with test__threading_vs_settrace.py
# being run under coverage when 'grep -l subprocess test*py' is used to list the tests
# to run.
tests -= ignore
tests = sorted(tests)
to_process = []
default_options = {'timeout': TIMEOUT}
......@@ -128,14 +149,15 @@ def discover(tests=None, ignore=None):
# try to decode those as ASCII, which fails with UnicodeDecodeError.
# Thus, be sure to open and compare in binary mode.
contents = f.read()
if b'TESTRUNNER' in contents:
if b'TESTRUNNER' in contents: # test__monkey_patching.py
module = __import__(filename.rsplit('.', 1)[0])
for cmd, options in module.TESTRUNNER():
if remove_options(cmd)[-1] in ignore:
continue
to_process.append((cmd, options))
else:
to_process.append(([sys.executable, '-u', filename], default_options.copy()))
cmd = [sys.executable, '-u', filename]
to_process.append((cmd, default_options.copy()))
return to_process
......@@ -241,13 +263,25 @@ def main():
parser.add_option('--full', action='store_true')
parser.add_option('--config')
parser.add_option('--failfast', action='store_true')
parser.add_option("--coverage", action="store_true")
options, args = parser.parse_args()
FAILING_TESTS = []
coverage = False
if options.coverage or os.environ.get("GEVENTTEST_COVERAGE"):
coverage = True
# NOTE: This must be run from the greentest directory
os.environ['COVERAGE_PROCESS_START'] = os.path.abspath(".coveragerc")
os.environ['PYTHONPATH'] = os.path.abspath("coveragesite") + os.pathsep + os.environ.get("PYTHONPATH", "")
# We change directory often, use an absolute path to keep all the
# coverage files (which will have distinct suffixes because of parallel=true in .coveragerc
# in this directory; makes them easier to combine and use with coverage report)
os.environ['COVERAGE_FILE'] = os.path.abspath(".") + os.sep + ".coverage"
print("Enabling coverage to", os.environ['COVERAGE_FILE'])
if options.config:
config = {}
six.exec_(open(options.config).read(), config)
FAILING_TESTS = config['FAILING_TESTS']
tests = discover(args, options.ignore)
tests = discover(args, options.ignore, coverage)
if options.discover:
for cmd, options in tests:
print(util.getname(cmd, env=options.get('env'), setenv=options.get('setenv')))
......
......@@ -8,6 +8,7 @@ import struct
LEAKTEST = os.getenv('GEVENTTEST_LEAKCHECK')
COVERAGE = os.getenv("COVERAGE_PROCESS_START")
PYPY = hasattr(sys, 'pypy_version_info')
PY3 = sys.version_info[0] >= 3
PY26 = sys.version_info[0] == 2 and sys.version_info[1] == 6
......@@ -156,6 +157,16 @@ if PY3:
'FLAKY test__socket.py',
]
if COVERAGE:
# The gevent concurrency plugin tends to slow things
# down and get us past our default timeout value. These
# tests in particular are sensitive to it
FAILING_TESTS += [
'FLAKY test__issue302monkey.py',
'FLAKY test__example_portforwarder.py',
'FLAKY test__threading_vs_settrace.py',
]
FAILING_TESTS = [x.strip() for x in FAILING_TESTS if x.strip()]
......
......@@ -6,6 +6,7 @@ envlist =
deps =
greenlet
cython
coverage >= 4.0b3
whitelist_externals =
*
commands =
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment