Honor PURE_PYTHON at runtime.

Fixes #1118.

Also some speed improvements for semaphores.

CHANGES.rst

@@ -109,6 +109,11 @@
   before. See :pr:`1117`. If there are any compatibility
   problems, please open issues.
 
+- On CPython, allow the pure-Python implementations of
+  ``gevent.greenlet``, ``gevent.local`` and ``gevent.sempahore`` to be
+  used when the environment variable ``PURE_PYTHON`` is set. This is
+  not recommended except for debugging and testing. See :issue:`1118`.
+
 
 1.3a1 (2018-01-27)
 ==================

Makefile

@@ -101,6 +101,9 @@ leaktest: test_prelim
 	@${PYTHON} scripts/travis.py fold_start leaktest "Running leak tests"
 	cd src/greentest && GEVENT_RESOLVER=thread GEVENTTEST_LEAKCHECK=1 ${PYTHON} testrunner.py --config known_failures.py --quiet --ignore tests_that_dont_do_leakchecks.txt
 	@${PYTHON} scripts/travis.py fold_end leaktest
+	@${PYTHON} scripts/travis.py fold_start default "Testing default backend pure python"
+	PURE_PYTHON=1 GEVENTTEST_COVERAGE=1 make basictest
+	@${PYTHON} scripts/travis.py fold_end default
 
 bench:
 	${PYTHON} src/greentest/bench_sendall.py
@@ -179,7 +182,7 @@ develop:
 	@${PYTHON} scripts/travis.py fold_end install
 
 test-py27: $(PY27)
-	PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop lint leaktest allbackendtest
+	PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop lint leaktest cffibackendtest coverage_combine
 
 test-py34: $(PY34)
 	PYTHON=python3.4.7 PATH=$(BUILD_RUNTIMES)/versions/python3.4.7/bin:$(PATH) make develop basictest

ci-requirements.txt

 setuptools
 wheel
-# Python 3.7b1 requires at least this version.
-# 0.28 is preferred due to 3.7 specific changes.
-cython>=0.27.3
+
+# Python 3.7 requires at least Cython 0.27.3.
+# 0.28 is faster, and (important!) lets us specify the target module
+# name to be created so that we can have both foo.py and _foo.so
+# at the same time.
+# This is an arbitrary commit that seems to work well.
+-e git+https://github.com/cython/cython.git@63cd3bbb5eac22b92808eeb90b512359e3def20a#egg=cython
+
 # Python 3.7b1 requires this.
 greenlet>=0.4.13
 pylint>=1.8.0

setup.py

@@ -50,11 +50,6 @@ from _setuplibev import CORE
 from _setupares import ARES
 
 
-SEMAPHORE = Extension(name="gevent._semaphore",
-                      sources=["src/gevent/_semaphore.py"],
-                      depends=['src/gevent/_semaphore.pxd'])
-SEMAPHORE = cythonize1(SEMAPHORE)
-
 # The sysconfig dir is not enough if we're in a virtualenv
 # See https://github.com/pypa/pip/issues/4610
 include_dirs = [sysconfig.get_path("include")]
@@ -64,32 +59,41 @@ venv_include_dir = os.path.abspath(venv_include_dir)
 if os.path.exists(venv_include_dir):
     include_dirs.append(venv_include_dir)
 
+SEMAPHORE = Extension(name="gevent.__semaphore",
+                      sources=["src/gevent/_semaphore.py"],
+                      depends=['src/gevent/__semaphore.pxd'],
+                      include_dirs=include_dirs)
 
-LOCAL = Extension(name="gevent.local",
+
+LOCAL = Extension(name="gevent._local",
                   sources=["src/gevent/local.py"],
-                  depends=['src/gevent/local.pxd'],
+                  depends=['src/gevent/_local.pxd'],
                   include_dirs=include_dirs)
-LOCAL = cythonize1(LOCAL)
 
 
-GREENLET = Extension(name="gevent.greenlet",
+GREENLET = Extension(name="gevent._greenlet",
                      sources=[
                          "src/gevent/greenlet.py",
                      ],
                      depends=[
-                         'src/gevent/greenlet.pxd',
-                         'src/gevent/_ident.pxd',
+                         'src/gevent/_greenlet.pxd',
+                         'src/gevent/__ident.pxd',
                          'src/gevent/_ident.py'
                      ],
                      include_dirs=include_dirs)
-GREENLET = cythonize1(GREENLET)
 
 
-IDENT = Extension(name="gevent._ident",
+IDENT = Extension(name="gevent.__ident",
                   sources=["src/gevent/_ident.py"],
-                  depends=['src/gevent/_ident.pxd'],
+                  depends=['src/gevent/__ident.pxd'],
                   include_dirs=include_dirs)
-IDENT = cythonize1(IDENT)
+
+_to_cythonize = [
+    SEMAPHORE,
+    LOCAL,
+    GREENLET,
+    IDENT,
+]
 
 EXT_MODULES = [
     CORE,
@@ -120,28 +124,33 @@ if not SKIP_LIBUV:
     # but manylinux1 has only 2.5, so we set SKIP_LIBUV in the script make-manylinux
     cffi_modules.append(LIBUV_CFFI_MODULE)
 
+install_requires = [
+    # We need to watch our greenlet version fairly carefully,
+    # since we compile cython code that extends the greenlet object.
+    # Binary compatibility would break if the greenlet struct changes.
+    'greenlet >= 0.4.13; platform_python_implementation=="CPython"',
+]
+
+setup_requires = [
+]
+
 if PYPY:
-    install_requires = []
-    setup_requires = []
-    EXT_MODULES.remove(CORE)
+    # These use greenlet/greenlet.h, which doesn't exist on PyPy
     EXT_MODULES.remove(LOCAL)
     EXT_MODULES.remove(GREENLET)
-    EXT_MODULES.remove(IDENT)
     EXT_MODULES.remove(SEMAPHORE)
-    # By building the semaphore with Cython under PyPy, we get
-    # atomic operations (specifically, exiting/releasing), at the
-    # cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
-    # at around 1s and the compiled version at around 4s). Some clever subclassing
-    # and having only the bare minimum be in cython might help reduce that penalty.
-    # NOTE: You must use version 0.23.4 or later to avoid a memory leak.
-    # https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
-    # However, that's all for naught on up to and including PyPy 4.0.1 which
-    # have some serious crashing bugs with GC interacting with cython,
-    # so this is disabled
-else:
-    install_requires = ['greenlet >= 0.4.10'] # TODO: Replace this with platform markers?
-    setup_requires = []
 
+    # As of PyPy 5.10, this builds, but won't import (missing _Py_ReprEnter)
+    EXT_MODULES.remove(CORE)
+
+    _to_cythonize.remove(LOCAL)
+    _to_cythonize.remove(GREENLET)
+    _to_cythonize.remove(SEMAPHORE)
+
+for mod in _to_cythonize:
+    EXT_MODULES.remove(mod)
+    EXT_MODULES.append(cythonize1(mod))
+del _to_cythonize
 
 try:
     cffi = __import__('cffi')

src/gevent/_ident.pxd → src/gevent/__ident.pxd

@@ -7,6 +7,8 @@ cdef extern from "Python.h":
 
 cdef heappop
 cdef heappush
+cdef object WeakKeyDictionary
+cdef type ref
 
 @cython.internal
 @cython.final

src/gevent/_semaphore.pxd → src/gevent/__semaphore.pxd

 # cython: auto_pickle=False
+
+cdef Timeout
+cdef get_hub
+
+cdef bint _greenlet_imported
+
+cdef extern from "greenlet/greenlet.h":
+
+    ctypedef class greenlet.greenlet [object PyGreenlet]:
+        pass
+
+    # These are actually macros and so much be included
+    # (defined) in each .pxd, as are the two functions
+    # that call them.
+    greenlet PyGreenlet_GetCurrent()
+    void PyGreenlet_Import()
+
+cdef inline greenlet getcurrent():
+    return PyGreenlet_GetCurrent()
+
+cdef inline void greenlet_init():
+    global _greenlet_imported
+    if not _greenlet_imported:
+        PyGreenlet_Import()
+        _greenlet_imported = True
+
+
+cdef void _init()
+
+
 cdef class Semaphore:
     cdef public int counter
-    cdef readonly object _links
+    cdef readonly list _links
     cdef readonly object _notifier
     cdef public int _dirty
     cdef object __weakref__

src/gevent/_compat.py

@@ -6,6 +6,7 @@ internal gevent python 2/python 3 bridges. Not for external use.
 from __future__ import print_function, absolute_import, division
 
 import sys
+import os
 
 
 PY2 = sys.version_info[0] == 2
@@ -13,6 +14,8 @@ PY3 = sys.version_info[0] >= 3
 PYPY = hasattr(sys, 'pypy_version_info')
 WIN = sys.platform.startswith("win")
 
+PURE_PYTHON = PYPY or os.getenv('PURE_PYTHON')
+
 ## Types
 
 if PY3:

src/gevent/greenlet.pxd → src/gevent/_greenlet.pxd

 # cython: auto_pickle=False
 
 cimport cython
-from gevent._ident cimport IdentRegistry
+from gevent.__ident cimport IdentRegistry
 cdef bint _greenlet_imported
 cdef bint _PYPY
 

src/gevent/_ident.py

@@ -74,3 +74,7 @@ class IdentRegistry(object):
 
     def __len__(self):
         return len(self._registry)
+
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent.__ident')

src/gevent/_semaphore.py

-# cython: auto_pickle=False
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
+from __future__ import print_function, absolute_import, division
 import sys
-from gevent.hub import get_hub, getcurrent
+
+from gevent.hub import get_hub
 from gevent.timeout import Timeout
 
 
-__all__ = ['Semaphore', 'BoundedSemaphore']
+__all__ = [
+    'Semaphore',
+    'BoundedSemaphore',
+]
+
+# In Cython, we define these as 'cdef inline' functions. The
+# compilation unit cannot have a direct assignment to them (import
+# is assignment) without generating a 'lvalue is not valid target'
+# error.
+locals()['getcurrent'] = __import__('greenlet').getcurrent
+locals()['greenlet_init'] = lambda: None
 
 
 class Semaphore(object):
@@ -101,7 +113,7 @@ class Semaphore(object):
                     try:
                         link(self) # Must use Cython >= 0.23.4 on PyPy else this leaks memory
                     except: # pylint:disable=bare-except
-                        getcurrent().handle_error((link, self), *sys.exc_info())
+                        getcurrent().handle_error((link, self), *sys.exc_info()) # pylint:disable=undefined-variable
                     if self._dirty:
                         # We mutated self._links so we need to start over
                         break
@@ -158,7 +170,7 @@ class Semaphore(object):
         elapses, return the exception. Otherwise, return None.
         Raises timeout if a different timer expires.
         """
-        switch = getcurrent().switch
+        switch = getcurrent().switch # pylint:disable=undefined-variable
         self.rawlink(switch)
         try:
             timer = Timeout._start_new_or_dummy(timeout)
@@ -267,4 +279,25 @@ class BoundedSemaphore(Semaphore):
     def release(self):
         if self.counter >= self._initial_value:
             raise self._OVER_RELEASE_ERROR("Semaphore released too many times")
-        return Semaphore.release(self)
+        Semaphore.release(self)
+
+
+def _init():
+    greenlet_init() # pylint:disable=undefined-variable
+
+_init()
+
+# By building the semaphore with Cython under PyPy, we get
+# atomic operations (specifically, exiting/releasing), at the
+# cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
+# at around 1s and the compiled version at around 4s). Some clever subclassing
+# and having only the bare minimum be in cython might help reduce that penalty.
+# NOTE: You must use version 0.23.4 or later to avoid a memory leak.
+# https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
+# However, that's all for naught on up to and including PyPy 4.0.1 which
+# have some serious crashing bugs with GC interacting with cython.
+# It hasn't been tested since then, and PURE_PYTHON is assumed to be true
+# for PyPy in all cases anyway, so this does nothing.
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent.__semaphore')

src/gevent/_util.py

@@ -71,6 +71,36 @@ def copy_globals(source,
 
     return copied
 
+def import_c_accel(globs, cname):
+    """
+    Import the C-accelerator for the __name__
+    and copy its globals.
+    """
+
+    name = globs.get('__name__')
+
+    if not name or name == cname:
+        # Do nothing if we're being exec'd as a file (no name)
+        # or we're running from the C extension
+        return
+
+    import importlib
+    from gevent._compat import PURE_PYTHON
+    if PURE_PYTHON:
+        return
+
+    mod = importlib.import_module(cname)
+
+    # By adopting the entire __dict__, we get a more accurate
+    # __file__ and module repr, plus we don't leak any imported
+    # things we no longer need.
+    globs.clear()
+    globs.update(mod.__dict__)
+
+    if 'import_c_accel' in globs:
+        del globs['import_c_accel']
+
+
 class Lazy(object):
     """
     A non-data descriptor used just like @property. The

src/gevent/greenlet.py

 # Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details.
 # cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function, division
 
 import sys
 from weakref import ref as wref
@@ -476,9 +476,9 @@ class Greenlet(greenlet):
 
         .. versionadded:: 1.1
         """
-        e = self._exc_info
-        if e is not None and e[0] is not None:
-            return (e[0], e[1], load_traceback(e[2]))
+        exc_info = self._exc_info
+        if exc_info is not None and exc_info[0] is not None:
+            return (exc_info[0], exc_info[1], load_traceback(exc_info[2]))
 
     def throw(self, *args):
         """Immediately switch into the greenlet and raise an exception in it.
@@ -926,3 +926,6 @@ def _init():
     greenlet_init() # pylint:disable=undefined-variable
 
 _init()
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent._greenlet')

src/gevent/local.py

@@ -532,3 +532,6 @@ finally:
     del sys
 
 _init()
+
+from gevent._util import import_c_accel
+import_c_accel(globals(), 'gevent._local')

src/greentest/greentest/__init__.py

@@ -73,6 +73,8 @@ from greentest.skipping import skipOnLibuvOnWin
 from greentest.skipping import skipOnLibuvOnCI
 from greentest.skipping import skipOnLibuvOnCIOnPyPy
 from greentest.skipping import skipOnLibuvOnPyPyOnWin
+from greentest.skipping import skipOnPurePython
+from greentest.skipping import skipWithCExtensions
 
 
 from greentest.exception import ExpectedException

src/greentest/greentest/skipping.py

@@ -40,6 +40,9 @@ skipOnPyPyOnCI = _do_not_skip
 skipOnPyPy3OnCI = _do_not_skip
 skipOnPyPy3 = _do_not_skip
 
+skipOnPurePython = unittest.skip if sysinfo.PURE_PYTHON else _do_not_skip
+skipWithCExtensions = unittest.skip if not sysinfo.PURE_PYTHON else _do_not_skip
+
 skipOnLibuv = _do_not_skip
 skipOnLibuvOnWin = _do_not_skip
 skipOnLibuvOnCI = _do_not_skip

src/greentest/greentest/sysinfo.py

@@ -22,13 +22,16 @@ import os
 import sys
 
 import gevent.core
+from gevent import _compat as gsysinfo
 
-PYPY = hasattr(sys, 'pypy_version_info')
+PYPY = gsysinfo.PYPY
 VERBOSE = sys.argv.count('-v') > 1
-WIN = sys.platform.startswith("win")
+WIN = gsysinfo.WIN
 LINUX = sys.platform.startswith('linux')
 OSX = sys.platform == 'darwin'
 
+PURE_PYTHON = gsysinfo.PURE_PYTHON
+
 # XXX: Formalize this better
 LIBUV = 'libuv' in gevent.core.loop.__module__ # pylint:disable=no-member
 CFFI_BACKEND = PYPY or LIBUV or 'cffi' in os.getenv('GEVENT_LOOP', '')

src/greentest/test___ident.py

@@ -66,6 +66,14 @@ class TestIdent(greentest.TestCase):
             self.assertLessEqual(self.reg.get_ident(target), keep_count)
 
 
+@greentest.skipOnPurePython("Needs C extension")
+class TestCExt(greentest.TestCase):
+
+    def test_c_extension(self):
+        self.assertEqual(IdentRegistry.__module__,
+                         'gevent.__ident')
+
+
 
 
 if __name__ == '__main__':

src/greentest/test__greenlet.py

@@ -741,6 +741,25 @@ class TestRef(greentest.TestCase):
             g.kill()
 
 
+@greentest.skipOnPurePython("Needs C extension")
+class TestCExt(greentest.TestCase):
+
+    def test_c_extension(self):
+        self.assertEqual(greenlet.Greenlet.__module__,
+                         'gevent._greenlet')
+        self.assertEqual(greenlet.SpawnedLink.__module__,
+                         'gevent._greenlet')
+
+@greentest.skipWithCExtensions("Needs pure python")
+class TestPure(greentest.TestCase):
+
+    def test_pure(self):
+        self.assertEqual(greenlet.Greenlet.__module__,
+                         'gevent.greenlet')
+        self.assertEqual(greenlet.SpawnedLink.__module__,
+                         'gevent.greenlet')
+
+
 X = object()
 
 del AbstractGenericGetTestCase

src/greentest/test__local.py

@@ -308,6 +308,20 @@ class GeventLocalTestCase(greentest.TestCase):
 
         self.assertEqual(count, len(deleted_sentinels))
 
+@greentest.skipOnPurePython("Needs C extension")
+class TestCExt(greentest.TestCase):
+
+    def test_c_extension(self):
+        self.assertEqual(local.__module__,
+                         'gevent._local')
+
+@greentest.skipWithCExtensions("Needs pure-python")
+class TestPure(greentest.TestCase):
+
+    def test_extension(self):
+        self.assertEqual(local.__module__,
+                         'gevent.local')
+
 
 if __name__ == '__main__':
     greentest.main()

src/greentest/test__semaphore.py

@@ -70,5 +70,13 @@ class TestLock(greentest.TestCase):
         self.assertIsInstance(g_exc, type(std_exc))
 
 
+@greentest.skipOnPurePython("Needs C extension")
+class TestCExt(greentest.TestCase):
+
+    def test_c_extension(self):
+        self.assertEqual(Semaphore.__module__,
+                         'gevent.__semaphore')
+
+
 if __name__ == '__main__':
     greentest.main()

src/greentest/test__threading_vs_settrace.py

@@ -3,6 +3,7 @@ import sys
 import subprocess
 import unittest
 from gevent.thread import allocate_lock
+import greentest
 
 script = """
 from gevent import monkey
@@ -47,6 +48,7 @@ sys.stdout.write("..finishing..")
 
 
 class TestTrace(unittest.TestCase):
+    @greentest.skipOnPurePython("Locks can be traced in Pure Python")
     def test_untraceable_lock(self):
         # Untraceable locks were part of the solution to https://bugs.python.org/issue1733757
         # which details a deadlock that could happen if a trace function invoked
@@ -61,7 +63,7 @@ class TestTrace(unittest.TestCase):
         PYPY = hasattr(sys, 'pypy_version_info')
         lst = []
         try:
-            def trace(frame, ev, arg):
+            def trace(frame, ev, _arg):
                 lst.append((frame.f_code.co_filename, frame.f_lineno, ev))
                 if not PYPY: # because we expect to trace on PyPy
                     print("TRACE: %s:%s %s" % lst[-1])
@@ -72,24 +74,21 @@ class TestTrace(unittest.TestCase):
         finally:
             sys.settrace(old)
 
-        if not PYPY:
-            self.assertEqual(lst, [], "trace not empty")
-        else:
-            # Have an assert so that we know if we miscompile
-            self.assertTrue(len(lst) > 0, "should not compile on pypy")
+        self.assertEqual(lst, [], "trace not empty")
 
+    @greentest.skipOnPurePython("Locks can be traced in Pure Python")
     def test_untraceable_lock_uses_different_lock(self):
         if hasattr(sys, 'gettrace'):
             old = sys.gettrace()
         else:
             old = None
-        PYPY = hasattr(sys, 'pypy_version_info')
+
         PY3 = sys.version_info[0] > 2
         lst = []
         # we should be able to use unrelated locks from within the trace function
         l = allocate_lock()
         try:
-            def trace(frame, ev, arg):
+            def trace(frame, ev, _arg):
                 with l:
                     lst.append((frame.f_code.co_filename, frame.f_lineno, ev))
                 if not PYPY: # because we expect to trace on PyPy
@@ -105,20 +104,20 @@ class TestTrace(unittest.TestCase):
         finally:
             sys.settrace(old)
 
-        if not PYPY and not PY3:
+        if not PY3:
             # Py3 overrides acquire in Python to do argument checking
             self.assertEqual(lst, [], "trace not empty")
         else:
             # Have an assert so that we know if we miscompile
-            self.assertTrue(len(lst) > 0, "should not compile on pypy")
+            self.assertTrue(lst, "should not compile on pypy")
 
+    @greentest.skipOnPurePython("Locks can be traced in Pure Python")
     def test_untraceable_lock_uses_same_lock(self):
         from gevent.hub import LoopExit
         if hasattr(sys, 'gettrace'):
             old = sys.gettrace()
         else:
             old = None
-        PYPY = hasattr(sys, 'pypy_version_info')
         PY3 = sys.version_info[0] > 2
         lst = []
         e = None
@@ -126,7 +125,7 @@ class TestTrace(unittest.TestCase):
         # because it's over acquired but instead of deadlocking it raises an exception
         l = allocate_lock()
         try:
-            def trace(frame, ev, arg):
+            def trace(frame, ev, _arg):
                 with l:
                     lst.append((frame.f_code.co_filename, frame.f_lineno, ev))
                 return trace
@@ -140,12 +139,12 @@ class TestTrace(unittest.TestCase):
         finally:
             sys.settrace(old)
 
-        if not PYPY and not PY3:
+        if not PY3:
             # Py3 overrides acquire in Python to do argument checking
             self.assertEqual(lst, [], "trace not empty")
         else:
             # Have an assert so that we know if we miscompile
-            self.assertTrue(len(lst) > 0, "should not compile on pypy")
+            self.assertTrue(lst, "should not compile on pypy")
             self.assertTrue(isinstance(e, LoopExit))
 
     def run_script(self, more_args=()):