Commit 92625467 authored by da-woods's avatar da-woods Committed by GitHub

Enable pypy as a required Travis test (GH-3392)

Reasoning being that this make it easier to catch pypy3
regressions as they happen.

* Fixed some very simple pypy3 failures (largely to do with
  different exception strings)
* Splits pypy3_bugs.txt into three files
  - one for bugs that cause hard crashes (which we don't want to
    run in Travis at all);
  - one for bugs that are probably unfixable because they're just
    due to implementation details (e.g. when destructors are
    called).
  - all other bugs remain in pypy3_bugs.txt
  (The categorization has been done fairly quickly, so some bugs
  may be in the wrong place)
* Made sure (hopefully) all bugs are now categorized, so a basic
  runtests.py with pypy3 should hopefully pass
* Changed pypy3 to be required in Travis
* Added an extra (optional) test that runs through pypy3_bugs.txt.
  The majority of this is expected to fail. This requires an
  extra option to runtest.py "--listfile", which just runs through
  the tests listed in the file.

I haven't made pypy2 a required test in this commit - since Python2 support is deprecated soon, there seemed limited value in putting much effort into pypy2.

Added faulthandler to runtests in the hope of being able to pin-down segmentation faults better on Travis

FileListExcluder matches regexes, not just name. Uses the same mechanism as is used for processing string passed on commandline
parent 4c7bd3c8
...@@ -71,10 +71,16 @@ matrix: ...@@ -71,10 +71,16 @@ matrix:
env: BACKEND=c env: BACKEND=c
- python: pypy3 - python: pypy3
env: BACKEND=c env: BACKEND=c
allow_failures: # a secondary pypy tests which is allowed to fail and which specifically
- python: 3.9-dev # tests known bugs
- python: pypy - python: pypy
env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt --listfile=tests/pypy2_bugs.txt bugs"
- python: pypy3 - python: pypy3
env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt bugs"
allow_failures:
- python: 3.9-dev
- env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt bugs"
- env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt --listfile=tests/pypy2_bugs.txt bugs"
branches: branches:
only: only:
......
...@@ -631,7 +631,7 @@ static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject ...@@ -631,7 +631,7 @@ static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject
if (res == NULL) PyErr_Clear(); if (res == NULL) PyErr_Clear();
return res; return res;
} }
#elif PY_MAJOR_VERSION >= 3 #elif PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000)
#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError #define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError
#define __Pyx_PyDict_GetItemStr PyDict_GetItem #define __Pyx_PyDict_GetItemStr PyDict_GetItem
#else #else
......
...@@ -190,7 +190,7 @@ static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObjec ...@@ -190,7 +190,7 @@ static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObjec
static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value) { static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value) {
PyObject* value; PyObject* value;
#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY #if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000)
value = PyDict_GetItemWithError(d, key); value = PyDict_GetItemWithError(d, key);
if (unlikely(!value)) { if (unlikely(!value)) {
if (unlikely(PyErr_Occurred())) if (unlikely(PyErr_Occurred()))
...@@ -238,7 +238,7 @@ static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *ke ...@@ -238,7 +238,7 @@ static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *ke
#else #else
if (is_safe_type == 1 || (is_safe_type == -1 && if (is_safe_type == 1 || (is_safe_type == -1 &&
/* the following builtins presumably have repeatably safe and fast hash functions */ /* the following builtins presumably have repeatably safe and fast hash functions */
#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY #if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000)
(PyUnicode_CheckExact(key) || PyString_CheckExact(key) || PyLong_CheckExact(key)))) { (PyUnicode_CheckExact(key) || PyString_CheckExact(key) || PyLong_CheckExact(key)))) {
value = PyDict_GetItemWithError(d, key); value = PyDict_GetItemWithError(d, key);
if (unlikely(!value)) { if (unlikely(!value)) {
......
...@@ -1826,6 +1826,10 @@ class EmbedTest(unittest.TestCase): ...@@ -1826,6 +1826,10 @@ class EmbedTest(unittest.TestCase):
except OSError: except OSError:
pass pass
def load_listfile(filename):
# just re-use the FileListExclude implementation
fle = FileListExcluder(filename)
return list(fle.excludes)
class MissingDependencyExcluder(object): class MissingDependencyExcluder(object):
def __init__(self, deps): def __init__(self, deps):
...@@ -1874,8 +1878,7 @@ class FileListExcluder(object): ...@@ -1874,8 +1878,7 @@ class FileListExcluder(object):
self.excludes[line.split()[0]] = True self.excludes[line.split()[0]] = True
def __call__(self, testname, tags=None): def __call__(self, testname, tags=None):
exclude = (testname in self.excludes exclude = any(string_selector(ex)(testname) for ex in self.excludes)
or testname.split('.')[-1] in self.excludes)
if exclude and self.verbose: if exclude and self.verbose:
print("Excluding %s because it's listed in %s" print("Excluding %s because it's listed in %s"
% (testname, self._list_file)) % (testname, self._list_file))
...@@ -2068,6 +2071,9 @@ def main(): ...@@ -2068,6 +2071,9 @@ def main():
parser.add_option("-x", "--exclude", dest="exclude", parser.add_option("-x", "--exclude", dest="exclude",
action="append", metavar="PATTERN", action="append", metavar="PATTERN",
help="exclude tests matching the PATTERN") help="exclude tests matching the PATTERN")
parser.add_option("--listfile", dest="listfile",
action="append",
help="specify a file containing a list of tests to run")
parser.add_option("-j", "--shard_count", dest="shard_count", metavar="N", parser.add_option("-j", "--shard_count", dest="shard_count", metavar="N",
type=int, default=1, type=int, default=1,
help="shard this run into several parallel runs") help="shard this run into several parallel runs")
...@@ -2161,6 +2167,10 @@ def main(): ...@@ -2161,6 +2167,10 @@ def main():
if options.xml_output_dir: if options.xml_output_dir:
shutil.rmtree(options.xml_output_dir, ignore_errors=True) shutil.rmtree(options.xml_output_dir, ignore_errors=True)
if options.listfile:
for listfile in options.listfile:
cmd_args.extend(load_listfile(listfile))
if options.capture: if options.capture:
keep_alive_interval = 10 keep_alive_interval = 10
else: else:
...@@ -2299,6 +2309,16 @@ def runtests_callback(args): ...@@ -2299,6 +2309,16 @@ def runtests_callback(args):
def runtests(options, cmd_args, coverage=None): def runtests(options, cmd_args, coverage=None):
# faulthandler should be able to provide a limited traceback
# in the event of a segmentation fault. Hopefully better than Travis
# just keeping running until timeout. Only available on Python 3.3+
try:
import faulthandler
except ImportError:
pass # OK - not essential
else:
faulthandler.enable()
WITH_CYTHON = options.with_cython WITH_CYTHON = options.with_cython
ROOTDIR = os.path.abspath(options.root_dir) ROOTDIR = os.path.abspath(options.root_dir)
...@@ -2416,6 +2436,9 @@ def runtests(options, cmd_args, coverage=None): ...@@ -2416,6 +2436,9 @@ def runtests(options, cmd_args, coverage=None):
bug_files = [ bug_files = [
('bugs.txt', True), ('bugs.txt', True),
('pypy_bugs.txt', IS_PYPY), ('pypy_bugs.txt', IS_PYPY),
('pypy2_bugs.txt', IS_PYPY and IS_PY2),
('pypy_crash_bugs.txt', IS_PYPY),
('pypy_implementation_detail_bugs.txt', IS_PYPY),
('limited_api_bugs.txt', options.limited_api), ('limited_api_bugs.txt', options.limited_api),
('windows_bugs.txt', sys.platform == 'win32'), ('windows_bugs.txt', sys.platform == 'win32'),
('cygwin_bugs.txt', sys.platform == 'cygwin') ('cygwin_bugs.txt', sys.platform == 'cygwin')
......
# Specific bugs that only apply to pypy2
build.cythonize_script
build.cythonize_script_package
run.initial_file_path
run.reduce_pickle
run.final_in_pxd
run.cdef_multiple_inheritance
run.cdef_multiple_inheritance_nodict
run.extstarargs
run.cpython_capi
run.isnot
# pypy 2 seems to be preferring .py files to .so files
# https://foss.heptapod.net/pypy/pypy/issues/3185
run.language_level
run.pure_pxd
# Silly error with doctest matching slightly different string outputs rather than
# an actual bug but one I can't easily resolve
run.with_gil
# looks like a "when does the GC run?" issue - slightly surprised it's OK on pypy3
memoryview.numpy_memoryview
...@@ -4,15 +4,58 @@ ...@@ -4,15 +4,58 @@
broken_exception broken_exception
bufaccess bufaccess
memoryview memoryview.memoryview$
memslice
sequential_parallel sequential_parallel
yield_from_pep380 yield_from_pep380
memoryview_inplace_division memoryview_inplace_division
run.unicodemethods
run.unicode_imports
run.tp_new
run.test_fstring
run.test_exceptions
run.test_dictviews
run.str_subclass_kwargs
run.special_method_docstrings
run.slice_ptr
compile.min_async
run.cython_includes
run.pyarray
run.test_unicode
run.__getattribute__
run.__getattribute_subclasses__
run.__debug__
run.array_cimport
run.builtin_abs
run.builtincomplex
run.cdef_multiple_inheritance_errors
run.cdivision_CEP_516
run.cyfunction
run.final_cdef_class
run.index
run.pyclass_special_methods
run.reimport_from_package
run.reimport_from_subinterpreter
pkg.cimportfrom
embedded
TestCyCache
run.ext_auto_richcmp
run.coverage_cmd
run.coverage_cmd_src_layout
run.coverage_installed_pkg
run.coverage_api
run.coverage_nogil
# very little coroutine-related seems to work
run.test_asyncgen
run.test_coroutines_pep492
run.async_iter_pep492
run.embedsignatures
run.py35_asyncio_async_def
run.asyncio_generators
# gc issue? # gc issue?
memoryview_in_subclasses
external_ref_reassignment external_ref_reassignment
run.exttype_dealloc run.exttype_dealloc
...@@ -20,20 +63,13 @@ run.exttype_dealloc ...@@ -20,20 +63,13 @@ run.exttype_dealloc
run.special_methods_T561 run.special_methods_T561
run.special_methods_T561_py2 run.special_methods_T561_py2
# tests for things that don't exist in cpyext # looks to be fixed in PyPy 7.3.0
compile.pylong # TODO - remove when Travis updates
run.datetime_pxd run.py_unicode_strings
run.datetime_cimport run.unicodeliterals
run.datetime_members run.unicode_identifiers
run.extern_builtins_T258 run.unicode_identifiers_import
run.line_trace errors.unicode_identifiers_e4
run.line_profile_test run.tracebacks
run.pstats_profile_test run.fstring
run.longintrepr run.unicode_identifiers_normalization
# refcounting-specific tests
double_dealloc_T796
run.exceptionrefcount
run.capiimpl
run.refcount_in_meth
# Bugs that causes hard crashes that we certainly don't
# want to run because it will break the testsuite
# segfault
run.fastcall
memslice
# """Fatal RPython error: NotImplementedError
# Aborted (core dumped)"""
run.py35_pep492_interop
# gc issue?
memoryview_in_subclasses
# PyPy "bugs" that are probably implementation differences from
# CPython rather than actual bugs. Therefore they aren't targets
# to be fixed (but there *may* be other details in the testfile
# that should be tested on PyPy?)
run.starargs
# refcounting-specific tests
double_dealloc_T796
run.exceptionrefcount
run.capiimpl
run.refcount_in_meth
# Ideally just disable the reference-counting tests on PyPy?
run.fused_types
run.generator_frame_cycle
run.generators_in_refcycles
run.generators_py
run.parallel
# "sys.getsizeof(object, default) will always return default on PyPy, and
# raise a TypeError if default is not provided."
buildenv
# tests for things that don't exist in cpyext
compile.pylong
run.datetime_pxd
run.datetime_cimport
run.datetime_members
run.extern_builtins_T258
run.line_trace
run.line_profile_test
run.pstats_profile_test
run.longintrepr
# tests probably rely on immediate GC (although maybe the tests could be tweaked so
# only these bits don't run in PyPy?)
buffers.buffer
buffers.userbuffer
memoryview.cythonarray
memoryview.memoryview_pep489_typing
run.cpp_classes
run.cpp_classes_def
# missing pypy feature?
matrix_multiplier
...@@ -168,15 +168,15 @@ def test_int_kwargs(f): ...@@ -168,15 +168,15 @@ def test_int_kwargs(f):
""" """
>>> test_int_kwargs(e) # doctest: +ELLIPSIS >>> test_int_kwargs(e) # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
TypeError: ...keywords must be strings TypeError: ...keywords must be strings...
>>> test_int_kwargs(f) # doctest: +ELLIPSIS >>> test_int_kwargs(f) # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
TypeError: ...keywords must be strings TypeError: ...keywords must be strings...
>>> test_int_kwargs(g) # doctest: +ELLIPSIS >>> test_int_kwargs(g) # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
TypeError: ...keywords must be strings TypeError: ...keywords must be strings...
>>> test_int_kwargs(h) # doctest: +ELLIPSIS >>> test_int_kwargs(h) # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
TypeError: ...keywords must be strings TypeError: ...keywords must be strings...
""" """
f(a=1,b=2,c=3, **{10:20,30:40}) f(a=1,b=2,c=3, **{10:20,30:40})
...@@ -143,10 +143,12 @@ def test_typedef_vector(o): ...@@ -143,10 +143,12 @@ def test_typedef_vector(o):
Traceback (most recent call last): Traceback (most recent call last):
... ...
OverflowError: ... OverflowError: ...
"TypeError: an integer is required" on CPython
>>> test_typedef_vector([1, 2, None]) #doctest: +ELLIPSIS >>> test_typedef_vector([1, 2, None]) #doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
... ...
TypeError: an integer is required TypeError: ...int...
""" """
cdef vector[my_int] v = o cdef vector[my_int] v = o
return v return v
......
...@@ -68,6 +68,10 @@ def typed_imports(): ...@@ -68,6 +68,10 @@ def typed_imports():
try: try:
from sys import version_info as maxunicode from sys import version_info as maxunicode
except TypeError, e: except TypeError, e:
if getattr(sys, "pypy_version_info", None):
# translate message
if e.args[0].startswith("int() argument must be"):
e = "an integer is required"
print(e) print(e)
try: try:
......
...@@ -4,7 +4,7 @@ def test(**kw): ...@@ -4,7 +4,7 @@ def test(**kw):
>>> d = {1 : 2} >>> d = {1 : 2}
>>> test(**d) # doctest: +ELLIPSIS >>> test(**d) # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
TypeError: ...keywords must be strings TypeError: ...keywords must be strings...
>>> d >>> d
{1: 2} {1: 2}
>>> d = {} >>> d = {}
......
...@@ -206,7 +206,7 @@ def crazy_pop(L): ...@@ -206,7 +206,7 @@ def crazy_pop(L):
""" """
>>> crazy_pop(list(range(10))) # doctest: +ELLIPSIS >>> crazy_pop(list(range(10))) # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
TypeError: pop... at most ... argument... TypeError: pop... argument...
>>> crazy_pop(A()) >>> crazy_pop(A())
(1, 2, 3) (1, 2, 3)
""" """
......
...@@ -9,7 +9,7 @@ def modobj(obj2, obj3): ...@@ -9,7 +9,7 @@ def modobj(obj2, obj3):
'5' '5'
>>> modobj(1, 0) # doctest: +ELLIPSIS >>> modobj(1, 0) # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
ZeroDivisionError: integer division or modulo by zero ZeroDivisionError: integer... modulo by zero
""" """
obj1 = obj2 % obj3 obj1 = obj2 % obj3
return obj1 return obj1
...@@ -17,9 +17,9 @@ def modobj(obj2, obj3): ...@@ -17,9 +17,9 @@ def modobj(obj2, obj3):
def mod_10_obj(int2): def mod_10_obj(int2):
""" """
>>> mod_10_obj(0) >>> mod_10_obj(0) # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
ZeroDivisionError: integer division or modulo by zero ZeroDivisionError: ... modulo by zero
>>> 10 % 1 >>> 10 % 1
0 0
>>> mod_10_obj(1) >>> mod_10_obj(1)
......
...@@ -326,7 +326,7 @@ def errors_non_string_kwarg(): ...@@ -326,7 +326,7 @@ def errors_non_string_kwarg():
""" """
>>> errors_non_string_kwarg() # doctest: +ELLIPSIS >>> errors_non_string_kwarg() # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
TypeError: ...keywords must be strings TypeError: ...keywords must be strings...
""" """
f(**{1:2}) f(**{1:2})
...@@ -463,10 +463,10 @@ def call_builtin_empty_dict(): ...@@ -463,10 +463,10 @@ def call_builtin_empty_dict():
def call_builtin_nonempty_dict(): def call_builtin_nonempty_dict():
""" """
>>> call_builtin_nonempty_dict() >>> call_builtin_nonempty_dict() # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
... ...
TypeError: id() takes no keyword arguments TypeError: id() ... keyword argument...
""" """
return id(1, **{'foo': 1}) return id(1, **{'foo': 1})
......
...@@ -123,9 +123,9 @@ def optimised_pow2(n): ...@@ -123,9 +123,9 @@ def optimised_pow2(n):
0.5 0.5
>>> optimised_pow2(0.5) == 2 ** 0.5 >>> optimised_pow2(0.5) == 2 ** 0.5
True True
>>> optimised_pow2('test') >>> optimised_pow2('test') # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
TypeError: unsupported operand type(s) for ** or pow(): 'int' and 'str' TypeError: ...operand... **...
""" """
if isinstance(n, (int, long)) and 0 <= n < 1000: if isinstance(n, (int, long)) and 0 <= n < 1000:
assert isinstance(2.0 ** n, float), 'float %s' % n assert isinstance(2.0 ** n, float), 'float %s' % n
...@@ -153,9 +153,9 @@ def optimised_pow2_inplace(n): ...@@ -153,9 +153,9 @@ def optimised_pow2_inplace(n):
0.5 0.5
>>> optimised_pow2_inplace(0.5) == 2 ** 0.5 >>> optimised_pow2_inplace(0.5) == 2 ** 0.5
True True
>>> optimised_pow2_inplace('test') >>> optimised_pow2_inplace('test') # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
TypeError: unsupported operand type(s) for ** or pow(): 'int' and 'str' TypeError: ...operand... **...
""" """
x = 2 x = 2
x **= n x **= n
......
...@@ -25,10 +25,12 @@ def test_declare(n): ...@@ -25,10 +25,12 @@ def test_declare(n):
(100, 100) (100, 100)
>>> test_declare(100.5) >>> test_declare(100.5)
(100, 100) (100, 100)
>>> test_declare(None)
# CPython: "TypeError: an integer is required"
>>> test_declare(None) # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
... ...
TypeError: an integer is required TypeError: ...int...
""" """
x = cython.declare(cython.int) x = cython.declare(cython.int)
y = cython.declare(cython.int, n) y = cython.declare(cython.int, n)
......
...@@ -25,7 +25,7 @@ def assign_py_hash_t(x): ...@@ -25,7 +25,7 @@ def assign_py_hash_t(x):
>>> assign_py_hash_t(IntLike(1.5)) # doctest: +ELLIPSIS >>> assign_py_hash_t(IntLike(1.5)) # doctest: +ELLIPSIS
Traceback (most recent call last): Traceback (most recent call last):
... ...
TypeError: __index__ ... (type float) TypeError: __index__ ... (type ...float...)
""" """
cdef Py_hash_t h = x cdef Py_hash_t h = x
return h return h
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment