Skip to content

G
gevent
Commit 0f3061bc authored by Jason Madden's avatar Jason Madden
Browse files
Options

Refactor known_failures.py to use a compact DSL instead of deeply nested conditions. 

Part of #1367
parent 48f6b449
Hide whitespace changes
Inline Side-by-side
Showing with 508 additions and 412 deletions
docs/changes/1367.misc 0 → 100644
View file @ 0f3061bc
Refactor ``known_failures.py`` to use a compact DSL instead of deeply
nested condition statements.
src/gevent/testing/monkey_test.py
View file @ 0f3061bc
......@@ -13,15 +13,17 @@ monkey.patch_all()
from .sysinfo import RUNNING_ON_APPVEYOR
from .sysinfo import PY37
from .sysinfo import PYPY3
from .patched_tests_setup import disable_tests_in_source
from . import support
from . import resources
from . import SkipTest
from . import util
if RUNNING_ON_APPVEYOR and PY37:
if (RUNNING_ON_APPVEYOR and PY37) or PYPY3:
# 3.7 added a stricter mode for thread cleanup.
# It appears to be unstable on Windows (at least appveyor)
# and PyPy3
# and test_socket.py constantly fails with an extra thread
# on some random test. We disable it entirely.
# XXX: Figure out how to make a *definition* in ./support.py actually
......
src/gevent/testing/testrunner.py
View file @ 0f3061bc
......@@ -32,10 +32,11 @@ except (ImportError, OSError, IOError):
pass
TIMEOUT = 100 # seconds
DEFAULT_NWORKERS = int(os.environ.get('NWORKERS') or max(cpu_count() - 1, 4))
if DEFAULT_NWORKERS > 10:
AVAIL_NWORKERS = cpu_count() - 1
DEFAULT_NWORKERS = int(os.environ.get('NWORKERS') or max(AVAIL_NWORKERS, 4))
if DEFAULT_NWORKERS > 15:
DEFAULT_NWORKERS = 10
SUGGESTED_NWORKERS = DEFAULT_NWORKERS
if RUN_LEAKCHECKS:
# Capturing the stats takes time, and we run each
......@@ -149,7 +150,7 @@ class Runner(object):
def __call__(self):
util.log("Running tests in parallel with concurrency %s %s." % (
self._worker_count,
util._colorize('number', '(concurrency available: %d)' % SUGGESTED_NWORKERS)
util._colorize('number', '(concurrency available: %d)' % AVAIL_NWORKERS)
),)
# Setting global state, in theory we can be used multiple times.
# This is fine as long as we are single threaded and call these
......
src/gevent/tests/known_failures.py
View file @ 0f3061bc
......@@ -2,343 +2,468 @@
# The tests listed there must fail (or testrunner.py will report error) unless they are prefixed with FLAKY
# in which cases the result of them is simply ignored
from __future__ import print_function
import os
import sys
import struct
from gevent.testing.sysinfo import RUNNING_ON_APPVEYOR as APPVEYOR
from gevent.testing.sysinfo import RUNNING_ON_TRAVIS as TRAVIS
from gevent.testing.sysinfo import RUN_LEAKCHECKS as LEAKTEST
from gevent.testing.sysinfo import RUN_COVERAGE as COVERAGE
from gevent.testing.sysinfo import RESOLVER_NOT_SYSTEM
from gevent.testing.sysinfo import PYPY
from gevent.testing.sysinfo import PY3
from gevent.testing.sysinfo import PY35
from gevent.testing.sysinfo import OSX
from gevent.testing.sysinfo import LIBUV
IGNORED_TESTS = []
from gevent.testing import sysinfo
class Condition(object):
__slots__ = ()
def __and__(self, other):
return AndCondition(self, other)
def __or__(self, other):
return OrCondition(self, other)
def __nonzero__(self):
return self.__bool__()
def __bool__(self):
raise NotImplementedError
class AbstractBinaryCondition(Condition): # pylint:disable=abstract-method
__slots__ = (
'lhs',
'rhs',
)
OP = None
def __init__(self, lhs, rhs):
self.lhs = lhs
self.rhs = rhs
def __repr__(self):
return "(%r %s %r)" % (
self.lhs,
self.OP,
self.rhs
)
class OrCondition(AbstractBinaryCondition):
__slots__ = ()
OP = '|'
def __bool__(self):
return bool(self.lhs) or bool(self.rhs)
class AndCondition(AbstractBinaryCondition):
__slots__ = ()
OP = '&'
def __bool__(self):
return bool(self.lhs) and bool(self.rhs)
class ConstantCondition(Condition):
__slots__ = (
'value',
'__name__',
)
def __init__(self, value, name=None):
self.value = bool(value)
self.__name__ = name or str(value)
def __bool__(self):
return self.value
def __repr__(self):
return self.__name__
ALWAYS = ConstantCondition(True)
NEVER = ConstantCondition(False)
class _AttrCondition(ConstantCondition):
__slots__ = (
)
def __init__(self, name):
ConstantCondition.__init__(self, getattr(sysinfo, name), name)
PYPY = _AttrCondition('PYPY')
PY3 = _AttrCondition('PY3')
PY2 = _AttrCondition('PY2')
OSX = _AttrCondition('OSX')
LIBUV = _AttrCondition('LIBUV')
WIN = _AttrCondition('WIN')
APPVEYOR = _AttrCondition('RUNNING_ON_APPVEYOR')
TRAVIS = _AttrCondition('RUNNING_ON_TRAVIS')
CI = _AttrCondition('RUNNING_ON_CI')
LEAKTEST = _AttrCondition('RUN_LEAKCHECKS')
COVERAGE = _AttrCondition('RUN_COVERAGE')
RESOLVER_NOT_SYSTEM = _AttrCondition('RESOLVER_NOT_SYSTEM')
BIT_64 = ConstantCondition(struct.calcsize('P') * 8 == 64, 'BIT_64')
PY380_EXACTLY = ConstantCondition(sys.version_info[:3] == (3, 8, 0), 'PY380_EXACTLY')
class _Definition(object):
__slots__ = (
'__name__',
# When does the class of this condition apply?
'when',
# When should this test be run alone, if it's run?
'run_alone',
# Should this test be ignored during coverage measurement?
'ignore_coverage',
# {name: (Condition, value)}
'options',
)
def __init__(self, when, run_alone, ignore_coverage, options):
assert isinstance(when, Condition)
assert isinstance(run_alone, Condition)
assert isinstance(ignore_coverage, Condition)
self.when = when
self.__name__ = None
self.run_alone = run_alone
self.ignore_coverage = ignore_coverage
if options:
for v in options.values():
assert isinstance(v, tuple) and len(v) == 2
assert isinstance(v[0], Condition)
self.options = options
def __set_name__(self, owner, name):
self.__name__ = name
def __repr__(self):
return '<%s for %s when=%r=%s run_alone=%r=%s>' % (
type(self).__name__,
self.__name__,
self.when, bool(self.when),
self.run_alone, bool(self.run_alone)
)
class _Action(_Definition):
__slots__ = (
'reason',
)
def __init__(self, reason='', when=ALWAYS, run_alone=NEVER, ignore_coverage=NEVER,
options=None):
_Definition.__init__(self, when, run_alone, ignore_coverage, options)
self.reason = reason
class RunAlone(_Action):
__slots__ = ()
def __init__(self, reason='', when=ALWAYS, ignore_coverage=NEVER):
_Action.__init__(self, reason, run_alone=when, ignore_coverage=ignore_coverage)
class Failing(_Action):
__slots__ = ()
class Flaky(Failing):
__slots__ = ()
class Ignored(_Action):
__slots__ = ()
class Multi(object):
def __init__(self):
self._conds = []
def flaky(self, reason='', when=True):
self._conds.append(Flaky(reason, when))
return self
def ignored(self, reason='', when=True):
self._conds.append(Ignored(reason, when))
return self
def __set_name__(self, owner, name):
for c in self._conds:
c.__set_name__(owner, name)
class DefinitionsMeta(type):
# a metaclass on Python 3 that makes sure we only set attributes once. pylint doesn't
# warn about that.
@classmethod
def __prepare__(cls, name, bases): # pylint:disable=unused-argument
return SetOnceMapping()
class SetOnceMapping(dict):
def __setitem__(self, name, value):
if name in self:
raise AttributeError(name)
dict.__setitem__(self, name, value)
som = SetOnceMapping()
som[1] = 1
try:
som[1] = 2
except AttributeError:
del som
else:
raise AssertionError("SetOnceMapping is broken")
DefinitionsBase = DefinitionsMeta('DefinitionsBase', (object,), {})
class Definitions(DefinitionsBase):
test__issue6 = Flaky(
"""test__issue6 (see comments in test file) is really flaky on both Travis and Appveyor;
on Travis we could just run the test again (but that gets old fast), but on appveyor
we don't have that option without a new commit---and sometimes we really need a build
to succeed in order to get a release wheel"""
)
test__core_fork = Ignored(
"""fork watchers don't get called on windows
because fork is not a concept windows has.
See this file for a detailed explanation.""",
when=WIN
)
test__greenletset = Flaky(
when=WIN,
ignore_coverage=PYPY
)
test__example_udp_client = test__example_udp_server = Flaky(
"""
These both run on port 9000 and can step on each other...seems
like the appveyor containers aren't fully port safe? Or it
takes longer for the processes to shut down? Or we run them in
a different order in the process pool than we do other places?
On PyPy on Travis, this fails to get the correct results,
sometimes. I can't reproduce locally
""",
when=APPVEYOR | (PYPY & TRAVIS)
)
# This one sometimes randomly closes connections, but no indication
# of a server crash, only a client side close.
test__server_pywsgi = Flaky(when=APPVEYOR)
test_threading = Multi().ignored(
"""
This one seems to just stop right after patching is done. It
passes on a local win 10 vm, and the main test_threading_2.py
does as well. Based on the printouts we added, it appears to
not even finish importing:
https://ci.appveyor.com/project/denik/gevent/build/1.0.1277/job/tpvhesij5gldjxqw#L1190
Ignored because it takes two minutes to time out.
""",
when=APPVEYOR & LIBUV & PYPY
).flaky(
"""
test_set_and_clear in Py3 relies on 5 threads all starting and
coming to an Event wait point while a sixth thread sleeps for a half
second. The sixth thread then does something and checks that
the 5 threads were all at the wait point. But the timing is sometimes
too tight for appveyor. This happens even if Event isn't
monkey-patched
""",
when=APPVEYOR & PY3
)
test__socketpair = Ignored(
"""
Py35 added socket.socketpair, all other releases
are missing it. No reason to even test it.
""",
when=WIN & PY2
)
test_ftplib = Flaky(
r"""
could be a problem of appveyor - not sure
======================================================================
ERROR: test_af (__main__.TestIPv6Environment)
----------------------------------------------------------------------
File "C:\Python27-x64\lib\ftplib.py", line 135, in connect
self.sock = socket.create_connection((self.host, self.port), self.timeout)
File "c:\projects\gevent\gevent\socket.py", line 73, in create_connection
raise err
error: [Errno 10049] [Error 10049] The requested address is not valid in its context.
XXX: On Jan 3 2016 this suddenly started passing on Py27/64; no idea why, the python version
was 2.7.11 before and after.
""",
when=APPVEYOR & BIT_64
)
test__backdoor = Flaky(when=LEAKTEST | PYPY)
test__socket_errors = Flaky(when=LEAKTEST)
test_signal = Flaky(
"On Travis, this very frequently fails due to timing",
when=TRAVIS & LEAKTEST,
# Partial workaround for the _testcapi issue on PyPy,
# but also because signal delivery can sometimes be slow, and this
# spawn processes of its own
run_alone=APPVEYOR,
)
test__socket_dns = Flaky(
"""
A few errors and differences:
AssertionError: ('255.255.255.255', 'http') != gaierror(-2,) # DNS Python
AssertionError: ('255.255.255.255', 'http') != gaierror(4, 'ARES_ENOTFOUND: Domain name not found')
AssertionError: OverflowError('port must be 0-65535.',) != ('readthedocs.org', '65535')
AssertionError: Lists differ:
(10, 1, 6, '', ('2607:f8b0:4004:810::200e', 80, 0L, 0L))
(10, 1, 6, '', ('2607:f8b0:4004:805::200e', 80, 0, 0))
Somehow it seems most of these are fixed with PyPy3.6-7 under dnspython,
(once we commented out TestHostname)?
""",
when=RESOLVER_NOT_SYSTEM | PY3
)
test__monkey_sigchld_2 = Ignored(
"""
This hangs for no apparent reason when run by the testrunner,
even wher maked standalone when run standalone from the
command line, it's fine. Issue in pypy2 6.0?
""",
when=PYPY & LIBUV
)
test_ssl = Ignored(
"""
PyPy 7.0 and 7.1 on Travis with Ubunto Xenial 16.04 can't
allocate SSL Context objects, either in Python 2.7 or 3.6.
There must be some library incompatibility. No point even
running them. XXX: Remember to turn this back on.
""",
when=PYPY & TRAVIS
)
test__pywsgi = Ignored(
"""
XXX: Re-enable this when we can investigate more. This has
started crashing with a SystemError. I cannot reproduce with
the same version on macOS and I cannot reproduce with the same
version in a Linux vm. Commenting out individual tests just
moves the crash around.
https://bitbucket.org/pypy/pypy/issues/2769/systemerror-unexpected-internal-exception
On Appveyor 3.8.0, for some reason this takes *way* too long, about 100s, which
often goes just over the default timeout of 100s. This makes no sense.
But it also takes nearly that long in 3.7. 3.6 and earlier are much faster.
""",
when=(PYPY & TRAVIS & LIBUV) | PY380_EXACTLY,
# https://bitbucket.org/pypy/pypy/issues/2769/systemerror-unexpected-internal-exception
run_alone=(CI & LEAKTEST & PY3) | (PYPY & LIBUV),
)
test_subprocess = Flaky(
"Unknown, can't reproduce locally; times out one test",
when=PYPY & PY3 & TRAVIS,
ignore_coverage=ALWAYS,
)
test__threadpool = Ignored(
"""
XXX: Re-enable these when we have more time to investigate.
This test, which normally takes ~60s, sometimes
hangs forever after running several tests. I cannot reproduce,
it seems highly load dependent. Observed with both libev and libuv.
""",
when=TRAVIS & (PYPY | OSX),
run_alone=ALWAYS,
# This often takes much longer on PyPy on CI.
options={'timeout': (CI & PYPY, 180)},
)
test__threading_2 = Ignored(
"""
This test, which normally takes 4-5s, sometimes
hangs forever after running two tests. I cannot reproduce,
it seems highly load dependent. Observed with both libev and libuv.
""",
when=TRAVIS & (PYPY | OSX),
# This often takes much longer on PyPy on CI.
options={'timeout': (CI & PYPY, 180)},
)
test__issue230 = Ignored(
"""
This rarely hangs for unknown reasons. I cannot reproduce
locally.
""",
when=TRAVIS & OSX
)
test_selectors = Flaky(
"""
Timing issues on appveyor.
""",
when=PY3 & APPVEYOR,
ignore_coverage=ALWAYS,
)
test__example_portforwarder = Flaky(
"""
This one sometimes times out, often after output "The process
with PID XXX could not be terminated. Reason: There is no
running instance of the task.",
""",
when=APPVEYOR | COVERAGE
)
test__issue302monkey = test__threading_vs_settrace = Flaky(
"""
The gevent concurrency plugin tends to slow things
down and get us past our default timeout value. These
tests in particular are sensitive to it. So in fact we just turn them
off.
""",
when=COVERAGE,
ignore_coverage=ALWAYS
)
test__hub_join_timeout = Ignored(
r"""
This sometimes times out. It appears to happen when the
times take too long and a test raises a FlakyTestTimeout error,
aka a unittest.SkipTest error. This probably indicates that we're
not cleaning something up correctly:
.....ss
GEVENTTEST_USE_RESOURCES=-network C:\Python38-x64\python.exe -u \
-mgevent.tests.test__hub_join_timeout [code TIMEOUT] [took 100.4s]
""",
when=APPVEYOR
)
test__example_wsgiserver = test__example_webproxy = RunAlone(
"""
These share the same port, which means they can conflict
between concurrent test runs too
XXX: Fix this by dynamically picking a port.
""",
)
test__pool = test__queue = RunAlone(
"""
On a heavily loaded box, these can all take upwards of 200s.
""",
when=CI & LEAKTEST | PY3
)
test_socket = RunAlone(
"Sometimes has unexpected timeouts",
when=CI & PYPY & PY3,
ignore_coverage=ALWAYS, # times out
)
test__refcount = RunAlone(
when=NEVER,
ignore_coverage=PYPY
)
FAILING_TESTS = [
# test__issue6 (see comments in test file) is really flaky on both Travis and Appveyor;
# on Travis we could just run the test again (but that gets old fast), but on appveyor
# we don't have that option without a new commit---and sometimes we really need a build
# to succeed in order to get a release wheel
'FLAKY test__issue6.py',
]
if sys.platform == 'win32':
IGNORED_TESTS = [
# fork watchers don't get called on windows
# because fork is not a concept windows has.
# See this file for a detailed explanation.
'test__core_fork.py',
]
# other Windows-related issues (need investigating)
FAILING_TESTS += [
'FLAKY test__greenletset.py',
]
if APPVEYOR:
FAILING_TESTS += [
# These both run on port 9000 and can step on each other...seems like the
# appveyor containers aren't fully port safe? Or it takes longer
# for the processes to shut down? Or we run them in a different order
# in the process pool than we do other places?
'FLAKY test__example_udp_client.py',
'FLAKY test__example_udp_server.py',
# This one sometimes times out, often after output "The process with PID XXX could not be
# terminated. Reason: There is no running instance of the task."
'FLAKY test__example_portforwarder.py',
# This one sometimes randomly closes connections, but no indication
# of a server crash, only a client side close.
'FLAKY test__server_pywsgi.py',
]
if PYPY and LIBUV:
IGNORED_TESTS += [
# This one seems to just stop right after
# patching is done. It passes on a local win 10 vm, and the main
# test_threading_2.py does as well.
# Based on the printouts we added, it appears to not even
# finish importing:
# https://ci.appveyor.com/project/denik/gevent/build/1.0.1277/job/tpvhesij5gldjxqw#L1190
# Ignored because it takes two minutes to time out.
'test_threading.py',
]
if PY3:
FAILING_TESTS += [
# test_set_and_clear in Py3 relies on 5 threads all starting and
# coming to an Event wait point while a sixth thread sleeps for a half
# second. The sixth thread then does something and checks that
# the 5 threads were all at the wait point. But the timing is sometimes
# too tight for appveyor. This happens even if Event isn't
# monkey-patched
'FLAKY test_threading.py',
# Starting in November 2018, on Python 3.7.0, we observe this test crashing.
# I can't reproduce locally.
# | C:\Python37-x64\python.exe -u -mgevent.tests.test__greenness
# 127.0.0.1 - - [09/Nov/2018 16:34:12] code 501, message Unsupported method ('GET')
# 127.0.0.1 - - [09/Nov/2018 16:34:12] "GET / HTTP/1.1" 501 -
# .
# ----------------------------------------------------------------------
# Ran 1 test in 0.031s
# OK
# Windows fatal exception: access violation
# Current thread 0x000003c8 (most recent call first):
# File "c:\projects\gevent\src\gevent\threadpool.py", line 261 in _worker
# Thread 0x00000600 (most recent call first):
# File "c:\projects\gevent\src\gevent\libuv\watcher.py", line 577 in send
# File "c:\projects\gevent\src\gevent\threadpool.py", line 408 in set
# File "c:\projects\gevent\src\gevent\threadpool.py", line 290 in _worker
# Thread 0x000007d4 (most recent call first):
# File "C:\Python37-x64\lib\weakref.py", line 356 in remove
# ! C:\Python37-x64\python.exe -u -mgevent.tests.test__greenness [code 3221225477] [took 1.3s]
# We have also seen this for Python 3.6.6 Nov 13 2018:
# | C:\Python36-x64\python.exe -u -mgevent.tests.test__backdoor
# ss.s.s
# ----------------------------------------------------------------------
# Ran 6 tests in 0.953s
# OK (skipped=4)
# Windows fatal exception: access violation
# Thread 0x00000aec (most recent call first):
# File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait
# File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get
# File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker
# Thread 0x00000548 (most recent call first):
# Thread 0x000003d0 (most recent call first):
# File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait
# File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get
# File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker
# Thread 0x00000ad0 (most recent call first):
# Thread 0x00000588 (most recent call first):
# File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait
# File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get
# File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker
# Thread 0x00000a54 (most recent call first):
# Thread 0x00000768 (most recent call first):
# File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait
# File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get
# File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker
# Current thread 0x00000894 (most recent call first):
# File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 261 in _worker
# Thread 0x00000634 (most recent call first):
# File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait
# File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get
# File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker
# Thread 0x00000538 (most recent call first):
# Thread 0x0000049c (most recent call first):
# File "C:\Python36-x64\lib\weakref.py", line 356 in remove
# ! C:\Python36-x64\python.exe -u -mgevent.tests.test__backdoor [code 3221225477] [Ran 6 tests in 2.1s]
# Note the common factors:
# - The test is finished (successfully) and we're apparently exiting the VM,
# doing GC
# - A weakref is being cleaned up
# weakref.py line 356 remove() is in WeakKeyDictionary. We only use WeakKeyDictionary
# in gevent._ident.IdentRegistry, which is only used in two places:
# gevent.hub.hub_ident_registry, which has weak references to Hub objects,
# and gevent.greenlet.Greenlet.minimal_ident, which uses its parent Hub's
# IdentRegistry to get its own identifier. So basically they have weak references
# to Hub and arbitrary Greenlets.
# Our attempted solution: stop using a module-level IdentRegistry to get
# Hub idents, and reduce how often we auto-generate one for greenlets.
# Commenting out the tests, lets see if it works.
#'FLAKY test__greenness.py',
#'FLAKY test__backdoor.py',
]
if not PY35:
# Py35 added socket.socketpair, all other releases
# are missing it. No reason to even test it.
IGNORED_TESTS += [
'test__socketpair.py',
]
if struct.calcsize('P') * 8 == 64:
# could be a problem of appveyor - not sure
# ======================================================================
# ERROR: test_af (__main__.TestIPv6Environment)
# ----------------------------------------------------------------------
# File "C:\Python27-x64\lib\ftplib.py", line 135, in connect
# self.sock = socket.create_connection((self.host, self.port), self.timeout)
# File "c:\projects\gevent\gevent\socket.py", line 73, in create_connection
# raise err
# error: [Errno 10049] [Error 10049] The requested address is not valid in its context.
# XXX: On Jan 3 2016 this suddenly started passing on Py27/64; no idea why, the python version
# was 2.7.11 before and after.
FAILING_TESTS.append('FLAKY test_ftplib.py')
if PY3:
pass
if LEAKTEST:
FAILING_TESTS += [
'FLAKY test__backdoor.py',
'FLAKY test__socket_errors.py',
]
if os.environ.get("TRAVIS") == "true":
FAILING_TESTS += [
# On Travis, this very frequently fails due to timing
'FLAKY test_signal.py',
]
if PYPY:
FAILING_TESTS += [
## Different in PyPy:
## Not implemented:
## ---
## BUGS:
## UNKNOWN:
# AssertionError: '>>> ' != ''
# test__backdoor.py:52
'FLAKY test__backdoor.py',
]
if RESOLVER_NOT_SYSTEM:
FAILING_TESTS += [
# A few errors and differences:
# AssertionError: ('255.255.255.255', 'http') != gaierror(-2,) # DNS Python
# AssertionError: ('255.255.255.255', 'http') != gaierror(4, 'ARES_ENOTFOUND: Domain name not found')
# AssertionError: OverflowError('port must be 0-65535.',) != ('readthedocs.org', '65535')
# AssertionError: Lists differ:
# (10, 1, 6, '', ('2607:f8b0:4004:810::200e', 80, 0L, 0L))
# (10, 1, 6, '', ('2607:f8b0:4004:805::200e', 80, 0, 0))
#
# Somehow it seems most of these are fixed with PyPy3.6-7 under dnspython,
# (once we commented out TestHostname)?
'FLAKY test__socket_dns.py',
]
if LIBUV:
IGNORED_TESTS += [
# This hangs for no apparent reason when run by the testrunner,
# even wher maked standalone
# when run standalone from the command line, it's fine.
# Issue in pypy2 6.0?
'test__monkey_sigchld_2.py',
]
if TRAVIS:
FAILING_TESTS += [
# This fails to get the correct results, sometimes. I can't reproduce locally
'FLAKY test__example_udp_server.py',
'FLAKY test__example_udp_client.py',
]
IGNORED_TESTS += [
# PyPy 7.0 and 7.1 on Travis with Ubunto Xenial 16.04
# can't allocate SSL Context objects, either in Python 2.7
# or 3.6. There must be some library incompatibility.
# No point even running them.
# XXX: Remember to turn this back on.
'test_ssl.py',
]
if LIBUV:
IGNORED_TESTS += [
# XXX: Re-enable this when we can investigate more.
# This has started crashing with a SystemError.
# I cannot reproduce with the same version on macOS
# and I cannot reproduce with the same version in a Linux vm.
# Commenting out individual tests just moves the crash around.
# https://bitbucket.org/pypy/pypy/issues/2769/systemerror-unexpected-internal-exception
'test__pywsgi.py',
]
if PY3 and TRAVIS:
FAILING_TESTS += [
## ---
## Unknown; can't reproduce locally on OS X
'FLAKY test_subprocess.py', # timeouts on one test.
]
if TRAVIS and (PYPY or OSX):
IGNORED_TESTS += [
# XXX Re-enable these when we have more time to investigate.
# This test, which normally takes ~60s, sometimes
# hangs forever after running several tests. I cannot reproduce,
# it seems highly load dependent. Observed with both libev and libuv.
'test__threadpool.py',
# This test, which normally takes 4-5s, sometimes
# hangs forever after running two tests. I cannot reproduce,
# it seems highly load dependent. Observed with both libev and libuv.
'test__threading_2.py',
]
if TRAVIS and OSX:
IGNORED_TESTS += [
# This rarely hangs for unknown reasons. I cannot reproduce
# locally.
'test__issue230.py',
]
if LIBUV:
if OSX:
FAILING_TESTS += [
]
if PY3:
# No idea / TODO
FAILING_TESTS += [
'FLAKY test__socket_dns.py',
]
if PY3 and APPVEYOR:
FAILING_TESTS += [
# Timing issues on appveyor
'FLAKY test_selectors.py'
]
if COVERAGE:
# The gevent concurrency plugin tends to slow things
# down and get us past our default timeout value. These
# tests in particular are sensitive to it
FAILING_TESTS += [
'FLAKY test__issue302monkey.py',
'FLAKY test__example_portforwarder.py',
'FLAKY test__threading_vs_settrace.py',
]
FAILING_TESTS = [x.strip() for x in set(FAILING_TESTS) if x.strip()]
# tests that can't be run when coverage is enabled
# TODO: Now that we have this declarative, we could eliminate this list,
# just add them to the main IGNORED_TESTS list.
IGNORE_COVERAGE = [
]
# A mapping from test file basename to a dictionary of
# options that will be applied on top of the DEFAULT_RUN_OPTIONS.
......@@ -346,89 +471,55 @@ TEST_FILE_OPTIONS = {
}
FAILING_TESTS = []
IGNORED_TESTS = []
# tests that don't do well when run on busy box
# or that are mutually exclusive
RUN_ALONE = [
'test__threadpool.py',
# These share the same port, which means they can conflict
# between concurrent test runs too
# XXX: Fix this by dynamically picking a port.
'test__example_wsgiserver.py',
'test__example_webproxy.py',
]
if APPVEYOR or TRAVIS:
RUN_ALONE += [
# Partial workaround for the _testcapi issue on PyPy,
# but also because signal delivery can sometimes be slow, and this
# spawn processes of its own
'test_signal.py',
]
if LEAKTEST and PY3:
# On a heavily loaded box, these can all take upwards of 200s
RUN_ALONE += [
'test__pool.py',
'test__pywsgi.py',
'test__queue.py',
]
if PYPY:
# This often takes much longer on PyPy on CI.
TEST_FILE_OPTIONS['test__threadpool.py'] = {'timeout': 180}
TEST_FILE_OPTIONS['test__threading_2.py'] = {'timeout': 180}
if PY3:
RUN_ALONE += [
# Sometimes shows unexpected timeouts
'test_socket.py',
]
if LIBUV:
RUN_ALONE += [
# https://bitbucket.org/pypy/pypy/issues/2769/systemerror-unexpected-internal-exception
'test__pywsgi.py',
]
if APPVEYOR:
IGNORED_TESTS += [
# This sometimes times out. It appears to happen when the
# times take too long and a test raises a FlakyTestTimeout error,
# aka a unittest.SkipTest error. This probably indicates that we're
# not cleaning something up correctly:
#
# .....ss
# GEVENTTEST_USE_RESOURCES=-network C:\Python38-x64\python.exe -u \
# -mgevent.tests.test__hub_join_timeout [code TIMEOUT] [took 100.4s]
'test__hub_join_timeout.py',
]
if sys.version_info[:3] == (3, 8, 0):
# For some reason this takes *way* too long, about 100s, which
# often goes just over the default timeout of 100s. This makes no sense.
# But it also takes nearly that long in 3.7. 3.6 and earlier are much faster.
IGNORED_TESTS += [
'test__pywsgi.py',
]
# tests that can't be run when coverage is enabled
IGNORE_COVERAGE = [
# Hangs forever
'test__threading_vs_settrace.py',
# times out
'test_socket.py',
# Doesn't get the exceptions it expects
'test_selectors.py',
# XXX ?
'test__issue302monkey.py',
"test_subprocess.py",
]
if PYPY:
IGNORE_COVERAGE += [
# Tends to timeout
'test__refcount.py',
'test__greenletset.py'
]
def populate(): # pylint:disable=too-many-branches
# TODO: Maybe move to the metaclass.
# TODO: This could be better.
for k, v in Definitions.__dict__.items():
if isinstance(v, Multi):
actions = v._conds
else:
actions = (v,)
test_name = k + '.py'
del k, v
for action in actions:
if not isinstance(action, _Action):
continue
if action.run_alone:
RUN_ALONE.append(test_name)
if action.ignore_coverage:
IGNORE_COVERAGE.append(test_name)
if action.options:
for opt_name, (condition, value) in action.options.items():
# TODO: Verify that this doesn't match more than once.
if condition:
TEST_FILE_OPTIONS.setdefault(test_name, {})[opt_name] = value
if not action.when:
if isinstance(action, Ignored):
IGNORED_TESTS.append(test_name)
elif isinstance(action, Flaky):
FAILING_TESTS.append('FLAKY ' + test_name)
elif isinstance(action, Failing):
FAILING_TESTS.append(test_name)
FAILING_TESTS.sort()
IGNORED_TESTS.sort()
RUN_ALONE.sort()
populate()
if __name__ == '__main__':
print('known_failures:\n', FAILING_TESTS)
print('ignored tests:\n', IGNORED_TESTS)
print('run alone:\n', RUN_ALONE)
print('options:\n', TEST_FILE_OPTIONS)
print("ignore during coverage:\n", IGNORE_COVERAGE)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7