Commit b84e1bd9 authored by Jason Madden's avatar Jason Madden Committed by GitHub

Merge pull request #1119 from gevent/issue1118

Honor PURE_PYTHON at runtime
parents cfea657f 7a915813
......@@ -109,6 +109,11 @@
before. See :pr:`1117`. If there are any compatibility
problems, please open issues.
- On CPython, allow the pure-Python implementations of
``gevent.greenlet``, ``gevent.local`` and ``gevent.sempahore`` to be
used when the environment variable ``PURE_PYTHON`` is set. This is
not recommended except for debugging and testing. See :issue:`1118`.
1.3a1 (2018-01-27)
==================
......
......@@ -41,7 +41,9 @@ prospector:
which pylint
# debugging
# pylint --rcfile=.pylintrc --init-hook="import sys, code; sys.excepthook = lambda exc, exc_type, tb: print(tb.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_frame.f_locals['self'])" gevent src/greentest/* || true
${PYTHON} scripts/gprospector.py -X
# XXX: prospector is failing right now. I can't reproduce locally:
# https://travis-ci.org/gevent/gevent/jobs/345474139
# ${PYTHON} scripts/gprospector.py -X
lint: prospector
......@@ -101,6 +103,9 @@ leaktest: test_prelim
@${PYTHON} scripts/travis.py fold_start leaktest "Running leak tests"
cd src/greentest && GEVENT_RESOLVER=thread GEVENTTEST_LEAKCHECK=1 ${PYTHON} testrunner.py --config known_failures.py --quiet --ignore tests_that_dont_do_leakchecks.txt
@${PYTHON} scripts/travis.py fold_end leaktest
@${PYTHON} scripts/travis.py fold_start default "Testing default backend pure python"
PURE_PYTHON=1 GEVENTTEST_COVERAGE=1 make basictest
@${PYTHON} scripts/travis.py fold_end default
bench:
${PYTHON} src/greentest/bench_sendall.py
......@@ -179,7 +184,7 @@ develop:
@${PYTHON} scripts/travis.py fold_end install
test-py27: $(PY27)
PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop lint leaktest allbackendtest
PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop lint leaktest cffibackendtest coverage_combine
test-py34: $(PY34)
PYTHON=python3.4.7 PATH=$(BUILD_RUNTIMES)/versions/python3.4.7/bin:$(PATH) make develop basictest
......
......@@ -131,7 +131,8 @@ install:
# pip will build them from source using the MSVC compiler matching the
# target Python version and architecture
# Note that psutil won't build under PyPy on Windows.
- "%CMD_IN_ENV% pip install -U setuptools wheel cython greenlet cffi dnspython idna"
- "%CMD_IN_ENV% pip install -e git+https://github.com/cython/cython.git@63cd3bbb5eac22b92808eeb90b512359e3def20a#egg=cython"
- "%CMD_IN_ENV% pip install -U setuptools wheel greenlet cffi dnspython idna"
- ps:
if ("${env:PYTHON_ID}" -ne "pypy") {
......
setuptools
wheel
# Python 3.7b1 requires at least this version.
# 0.28 is preferred due to 3.7 specific changes.
cython>=0.27.3
# Python 3.7 requires at least Cython 0.27.3.
# 0.28 is faster, and (important!) lets us specify the target module
# name to be created so that we can have both foo.py and _foo.so
# at the same time.
# This is an arbitrary commit that seems to work well.
-e git+https://github.com/cython/cython.git@471025858954d5b8429a9361a77dc41c6650ac52#egg=cython
# Python 3.7b1 requires this.
greenlet>=0.4.13
pylint>=1.8.0
......
......@@ -50,11 +50,6 @@ from _setuplibev import CORE
from _setupares import ARES
SEMAPHORE = Extension(name="gevent._semaphore",
sources=["src/gevent/_semaphore.py"],
depends=['src/gevent/_semaphore.pxd'])
SEMAPHORE = cythonize1(SEMAPHORE)
# The sysconfig dir is not enough if we're in a virtualenv
# See https://github.com/pypa/pip/issues/4610
include_dirs = [sysconfig.get_path("include")]
......@@ -64,32 +59,41 @@ venv_include_dir = os.path.abspath(venv_include_dir)
if os.path.exists(venv_include_dir):
include_dirs.append(venv_include_dir)
SEMAPHORE = Extension(name="gevent.__semaphore",
sources=["src/gevent/_semaphore.py"],
depends=['src/gevent/__semaphore.pxd'],
include_dirs=include_dirs)
LOCAL = Extension(name="gevent.local",
LOCAL = Extension(name="gevent._local",
sources=["src/gevent/local.py"],
depends=['src/gevent/local.pxd'],
depends=['src/gevent/_local.pxd'],
include_dirs=include_dirs)
LOCAL = cythonize1(LOCAL)
GREENLET = Extension(name="gevent.greenlet",
GREENLET = Extension(name="gevent._greenlet",
sources=[
"src/gevent/greenlet.py",
],
depends=[
'src/gevent/greenlet.pxd',
'src/gevent/_ident.pxd',
'src/gevent/_greenlet.pxd',
'src/gevent/__ident.pxd',
'src/gevent/_ident.py'
],
include_dirs=include_dirs)
GREENLET = cythonize1(GREENLET)
IDENT = Extension(name="gevent._ident",
IDENT = Extension(name="gevent.__ident",
sources=["src/gevent/_ident.py"],
depends=['src/gevent/_ident.pxd'],
depends=['src/gevent/__ident.pxd'],
include_dirs=include_dirs)
IDENT = cythonize1(IDENT)
_to_cythonize = [
SEMAPHORE,
LOCAL,
GREENLET,
IDENT,
]
EXT_MODULES = [
CORE,
......@@ -120,28 +124,37 @@ if not SKIP_LIBUV:
# but manylinux1 has only 2.5, so we set SKIP_LIBUV in the script make-manylinux
cffi_modules.append(LIBUV_CFFI_MODULE)
install_requires = [
# We need to watch our greenlet version fairly carefully,
# since we compile cython code that extends the greenlet object.
# Binary compatibility would break if the greenlet struct changes.
'greenlet >= 0.4.13; platform_python_implementation=="CPython"',
]
setup_requires = [
]
if PYPY:
install_requires = []
setup_requires = []
EXT_MODULES.remove(CORE)
# These use greenlet/greenlet.h, which doesn't exist on PyPy
EXT_MODULES.remove(LOCAL)
EXT_MODULES.remove(GREENLET)
EXT_MODULES.remove(IDENT)
EXT_MODULES.remove(SEMAPHORE)
# By building the semaphore with Cython under PyPy, we get
# atomic operations (specifically, exiting/releasing), at the
# cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
# at around 1s and the compiled version at around 4s). Some clever subclassing
# and having only the bare minimum be in cython might help reduce that penalty.
# NOTE: You must use version 0.23.4 or later to avoid a memory leak.
# https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
# However, that's all for naught on up to and including PyPy 4.0.1 which
# have some serious crashing bugs with GC interacting with cython,
# so this is disabled
else:
install_requires = ['greenlet >= 0.4.10'] # TODO: Replace this with platform markers?
setup_requires = []
# As of PyPy 5.10, this builds, but won't import (missing _Py_ReprEnter)
EXT_MODULES.remove(CORE)
# This uses PyWeakReference and doesn't compile on PyPy
EXT_MODULES.remove(IDENT)
_to_cythonize.remove(LOCAL)
_to_cythonize.remove(GREENLET)
_to_cythonize.remove(SEMAPHORE)
_to_cythonize.remove(IDENT)
for mod in _to_cythonize:
EXT_MODULES.remove(mod)
EXT_MODULES.append(cythonize1(mod))
del _to_cythonize
try:
cffi = __import__('cffi')
......
......@@ -7,6 +7,8 @@ cdef extern from "Python.h":
cdef heappop
cdef heappush
cdef object WeakKeyDictionary
cdef type ref
@cython.internal
@cython.final
......
# cython: auto_pickle=False
cdef Timeout
cdef get_hub
cdef bint _greenlet_imported
cdef extern from "greenlet/greenlet.h":
ctypedef class greenlet.greenlet [object PyGreenlet]:
pass
# These are actually macros and so much be included
# (defined) in each .pxd, as are the two functions
# that call them.
greenlet PyGreenlet_GetCurrent()
void PyGreenlet_Import()
cdef inline greenlet getcurrent():
return PyGreenlet_GetCurrent()
cdef inline void greenlet_init():
global _greenlet_imported
if not _greenlet_imported:
PyGreenlet_Import()
_greenlet_imported = True
cdef void _init()
cdef class Semaphore:
cdef public int counter
cdef readonly object _links
cdef readonly list _links
cdef readonly object _notifier
cdef public int _dirty
cdef object __weakref__
......
......@@ -6,6 +6,7 @@ internal gevent python 2/python 3 bridges. Not for external use.
from __future__ import print_function, absolute_import, division
import sys
import os
PY2 = sys.version_info[0] == 2
......@@ -13,6 +14,8 @@ PY3 = sys.version_info[0] >= 3
PYPY = hasattr(sys, 'pypy_version_info')
WIN = sys.platform.startswith("win")
PURE_PYTHON = PYPY or os.getenv('PURE_PYTHON')
## Types
if PY3:
......
# cython: auto_pickle=False
cimport cython
from gevent._ident cimport IdentRegistry
from gevent.__ident cimport IdentRegistry
cdef bint _greenlet_imported
cdef bint _PYPY
......
......@@ -74,3 +74,7 @@ class IdentRegistry(object):
def __len__(self):
return len(self._registry)
from gevent._util import import_c_accel
import_c_accel(globals(), 'gevent.__ident')
# cython: auto_pickle=False
# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
from __future__ import print_function, absolute_import, division
import sys
from gevent.hub import get_hub, getcurrent
from gevent.hub import get_hub
from gevent.timeout import Timeout
__all__ = ['Semaphore', 'BoundedSemaphore']
__all__ = [
'Semaphore',
'BoundedSemaphore',
]
# In Cython, we define these as 'cdef inline' functions. The
# compilation unit cannot have a direct assignment to them (import
# is assignment) without generating a 'lvalue is not valid target'
# error.
locals()['getcurrent'] = __import__('greenlet').getcurrent
locals()['greenlet_init'] = lambda: None
class Semaphore(object):
......@@ -101,7 +113,7 @@ class Semaphore(object):
try:
link(self) # Must use Cython >= 0.23.4 on PyPy else this leaks memory
except: # pylint:disable=bare-except
getcurrent().handle_error((link, self), *sys.exc_info())
getcurrent().handle_error((link, self), *sys.exc_info()) # pylint:disable=undefined-variable
if self._dirty:
# We mutated self._links so we need to start over
break
......@@ -158,7 +170,7 @@ class Semaphore(object):
elapses, return the exception. Otherwise, return None.
Raises timeout if a different timer expires.
"""
switch = getcurrent().switch
switch = getcurrent().switch # pylint:disable=undefined-variable
self.rawlink(switch)
try:
timer = Timeout._start_new_or_dummy(timeout)
......@@ -267,4 +279,25 @@ class BoundedSemaphore(Semaphore):
def release(self):
if self.counter >= self._initial_value:
raise self._OVER_RELEASE_ERROR("Semaphore released too many times")
return Semaphore.release(self)
Semaphore.release(self)
def _init():
greenlet_init() # pylint:disable=undefined-variable
_init()
# By building the semaphore with Cython under PyPy, we get
# atomic operations (specifically, exiting/releasing), at the
# cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
# at around 1s and the compiled version at around 4s). Some clever subclassing
# and having only the bare minimum be in cython might help reduce that penalty.
# NOTE: You must use version 0.23.4 or later to avoid a memory leak.
# https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
# However, that's all for naught on up to and including PyPy 4.0.1 which
# have some serious crashing bugs with GC interacting with cython.
# It hasn't been tested since then, and PURE_PYTHON is assumed to be true
# for PyPy in all cases anyway, so this does nothing.
from gevent._util import import_c_accel
import_c_accel(globals(), 'gevent.__semaphore')
......@@ -71,6 +71,36 @@ def copy_globals(source,
return copied
def import_c_accel(globs, cname):
"""
Import the C-accelerator for the __name__
and copy its globals.
"""
name = globs.get('__name__')
if not name or name == cname:
# Do nothing if we're being exec'd as a file (no name)
# or we're running from the C extension
return
import importlib
from gevent._compat import PURE_PYTHON
if PURE_PYTHON:
return
mod = importlib.import_module(cname)
# By adopting the entire __dict__, we get a more accurate
# __file__ and module repr, plus we don't leak any imported
# things we no longer need.
globs.clear()
globs.update(mod.__dict__)
if 'import_c_accel' in globs:
del globs['import_c_accel']
class Lazy(object):
"""
A non-data descriptor used just like @property. The
......
# Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details.
# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import sys
from weakref import ref as wref
......@@ -476,9 +476,9 @@ class Greenlet(greenlet):
.. versionadded:: 1.1
"""
e = self._exc_info
if e is not None and e[0] is not None:
return (e[0], e[1], load_traceback(e[2]))
exc_info = self._exc_info
if exc_info is not None and exc_info[0] is not None:
return (exc_info[0], exc_info[1], load_traceback(exc_info[2]))
def throw(self, *args):
"""Immediately switch into the greenlet and raise an exception in it.
......@@ -926,3 +926,6 @@ def _init():
greenlet_init() # pylint:disable=undefined-variable
_init()
from gevent._util import import_c_accel
import_c_accel(globals(), 'gevent._greenlet')
......@@ -532,3 +532,6 @@ finally:
del sys
_init()
from gevent._util import import_c_accel
import_c_accel(globals(), 'gevent._local')
......@@ -73,6 +73,8 @@ from greentest.skipping import skipOnLibuvOnWin
from greentest.skipping import skipOnLibuvOnCI
from greentest.skipping import skipOnLibuvOnCIOnPyPy
from greentest.skipping import skipOnLibuvOnPyPyOnWin
from greentest.skipping import skipOnPurePython
from greentest.skipping import skipWithCExtensions
from greentest.exception import ExpectedException
......
......@@ -40,6 +40,9 @@ skipOnPyPyOnCI = _do_not_skip
skipOnPyPy3OnCI = _do_not_skip
skipOnPyPy3 = _do_not_skip
skipOnPurePython = unittest.skip if sysinfo.PURE_PYTHON else _do_not_skip
skipWithCExtensions = unittest.skip if not sysinfo.PURE_PYTHON else _do_not_skip
skipOnLibuv = _do_not_skip
skipOnLibuvOnWin = _do_not_skip
skipOnLibuvOnCI = _do_not_skip
......
......@@ -22,13 +22,16 @@ import os
import sys
import gevent.core
from gevent import _compat as gsysinfo
PYPY = hasattr(sys, 'pypy_version_info')
PYPY = gsysinfo.PYPY
VERBOSE = sys.argv.count('-v') > 1
WIN = sys.platform.startswith("win")
WIN = gsysinfo.WIN
LINUX = sys.platform.startswith('linux')
OSX = sys.platform == 'darwin'
PURE_PYTHON = gsysinfo.PURE_PYTHON
# XXX: Formalize this better
LIBUV = 'libuv' in gevent.core.loop.__module__ # pylint:disable=no-member
CFFI_BACKEND = PYPY or LIBUV or 'cffi' in os.getenv('GEVENT_LOOP', '')
......
......@@ -66,6 +66,14 @@ class TestIdent(greentest.TestCase):
self.assertLessEqual(self.reg.get_ident(target), keep_count)
@greentest.skipOnPurePython("Needs C extension")
class TestCExt(greentest.TestCase):
def test_c_extension(self):
self.assertEqual(IdentRegistry.__module__,
'gevent.__ident')
if __name__ == '__main__':
......
......@@ -741,6 +741,25 @@ class TestRef(greentest.TestCase):
g.kill()
@greentest.skipOnPurePython("Needs C extension")
class TestCExt(greentest.TestCase):
def test_c_extension(self):
self.assertEqual(greenlet.Greenlet.__module__,
'gevent._greenlet')
self.assertEqual(greenlet.SpawnedLink.__module__,
'gevent._greenlet')
@greentest.skipWithCExtensions("Needs pure python")
class TestPure(greentest.TestCase):
def test_pure(self):
self.assertEqual(greenlet.Greenlet.__module__,
'gevent.greenlet')
self.assertEqual(greenlet.SpawnedLink.__module__,
'gevent.greenlet')
X = object()
del AbstractGenericGetTestCase
......
......@@ -308,6 +308,20 @@ class GeventLocalTestCase(greentest.TestCase):
self.assertEqual(count, len(deleted_sentinels))
@greentest.skipOnPurePython("Needs C extension")
class TestCExt(greentest.TestCase):
def test_c_extension(self):
self.assertEqual(local.__module__,
'gevent._local')
@greentest.skipWithCExtensions("Needs pure-python")
class TestPure(greentest.TestCase):
def test_extension(self):
self.assertEqual(local.__module__,
'gevent.local')
if __name__ == '__main__':
greentest.main()
......@@ -70,5 +70,13 @@ class TestLock(greentest.TestCase):
self.assertIsInstance(g_exc, type(std_exc))
@greentest.skipOnPurePython("Needs C extension")
class TestCExt(greentest.TestCase):
def test_c_extension(self):
self.assertEqual(Semaphore.__module__,
'gevent.__semaphore')
if __name__ == '__main__':
greentest.main()
......@@ -3,6 +3,7 @@ import sys
import subprocess
import unittest
from gevent.thread import allocate_lock
import greentest
script = """
from gevent import monkey
......@@ -47,6 +48,7 @@ sys.stdout.write("..finishing..")
class TestTrace(unittest.TestCase):
@greentest.skipOnPurePython("Locks can be traced in Pure Python")
def test_untraceable_lock(self):
# Untraceable locks were part of the solution to https://bugs.python.org/issue1733757
# which details a deadlock that could happen if a trace function invoked
......@@ -58,13 +60,12 @@ class TestTrace(unittest.TestCase):
old = sys.gettrace()
else:
old = None
PYPY = hasattr(sys, 'pypy_version_info')
lst = []
try:
def trace(frame, ev, arg):
def trace(frame, ev, _arg):
lst.append((frame.f_code.co_filename, frame.f_lineno, ev))
if not PYPY: # because we expect to trace on PyPy
print("TRACE: %s:%s %s" % lst[-1])
print("TRACE: %s:%s %s" % lst[-1])
return trace
with allocate_lock():
......@@ -72,28 +73,24 @@ class TestTrace(unittest.TestCase):
finally:
sys.settrace(old)
if not PYPY:
self.assertEqual(lst, [], "trace not empty")
else:
# Have an assert so that we know if we miscompile
self.assertTrue(len(lst) > 0, "should not compile on pypy")
self.assertEqual(lst, [], "trace not empty")
@greentest.skipOnPurePython("Locks can be traced in Pure Python")
def test_untraceable_lock_uses_different_lock(self):
if hasattr(sys, 'gettrace'):
old = sys.gettrace()
else:
old = None
PYPY = hasattr(sys, 'pypy_version_info')
PY3 = sys.version_info[0] > 2
lst = []
# we should be able to use unrelated locks from within the trace function
l = allocate_lock()
try:
def trace(frame, ev, arg):
def trace(frame, ev, _arg):
with l:
lst.append((frame.f_code.co_filename, frame.f_lineno, ev))
if not PYPY: # because we expect to trace on PyPy
print("TRACE: %s:%s %s" % lst[-1])
print("TRACE: %s:%s %s" % lst[-1])
return trace
l2 = allocate_lock()
......@@ -105,20 +102,20 @@ class TestTrace(unittest.TestCase):
finally:
sys.settrace(old)
if not PYPY and not PY3:
if not PY3:
# Py3 overrides acquire in Python to do argument checking
self.assertEqual(lst, [], "trace not empty")
else:
# Have an assert so that we know if we miscompile
self.assertTrue(len(lst) > 0, "should not compile on pypy")
self.assertTrue(lst, "should not compile on pypy")
@greentest.skipOnPurePython("Locks can be traced in Pure Python")
def test_untraceable_lock_uses_same_lock(self):
from gevent.hub import LoopExit
if hasattr(sys, 'gettrace'):
old = sys.gettrace()
else:
old = None
PYPY = hasattr(sys, 'pypy_version_info')
PY3 = sys.version_info[0] > 2
lst = []
e = None
......@@ -126,7 +123,7 @@ class TestTrace(unittest.TestCase):
# because it's over acquired but instead of deadlocking it raises an exception
l = allocate_lock()
try:
def trace(frame, ev, arg):
def trace(frame, ev, _arg):
with l:
lst.append((frame.f_code.co_filename, frame.f_lineno, ev))
return trace
......@@ -140,12 +137,12 @@ class TestTrace(unittest.TestCase):
finally:
sys.settrace(old)
if not PYPY and not PY3:
if not PY3:
# Py3 overrides acquire in Python to do argument checking
self.assertEqual(lst, [], "trace not empty")
else:
# Have an assert so that we know if we miscompile
self.assertTrue(len(lst) > 0, "should not compile on pypy")
self.assertTrue(lst, "should not compile on pypy")
self.assertTrue(isinstance(e, LoopExit))
def run_script(self, more_args=()):
......@@ -163,5 +160,4 @@ class TestTrace(unittest.TestCase):
if __name__ == "__main__":
import greentest
greentest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment