Experimental/WIP PyPy support #248

Some tests fail, consult known_failures.py for the list.

stdlib's signal module is not supported. Depends on https://bitbucket.org/cffi/cffi/issue/152/handling-errors-from-signal-handlers-in

On PyPy, sys.exc_info is shared between greenlets apparently https://bugs.pypy.org/issue1743.

based on gevent-on-pypy/pypycore@f04a41c written by Ralf Schmitt and Lucas Clemente Vella with patches from hasenj and Armin Rigo

.travis.yml

@@ -3,9 +3,12 @@ python:
  - "2.6"
  - "2.7"
  - "3.3"
+ - "pypy"
 matrix:
   allow_failures:
     - python: "3.3"
-script: make travis
+script:
+ - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then PYTHON=pypy make travis_pypy; fi
+ - if [[ $TRAVIS_PYTHON_VERSION != 'pypy' ]]; then PYTHON=python$TRAVIS_PYTHON_VERSION make travis_cpython; fi
 notifications:
   email: false

Makefile

@@ -4,13 +4,13 @@
 PYTHON ?= python${TRAVIS_PYTHON_VERSION}
 CYTHON ?= cython
 
-all: gevent/gevent.core.c gevent/gevent.ares.c gevent/gevent._semaphore.c gevent/gevent._util.c
+all: gevent/gevent.corecext.c gevent/gevent.ares.c gevent/gevent._semaphore.c gevent/gevent._util.c
 
-gevent/gevent.core.c: gevent/core.ppyx gevent/libev.pxd
-	$(PYTHON) util/cythonpp.py -o gevent.core.c gevent/core.ppyx
-	echo                          >> gevent.core.c
-	echo '#include "callbacks.c"' >> gevent.core.c
-	mv gevent.core.* gevent/
+gevent/gevent.corecext.c: gevent/core.ppyx gevent/libev.pxd
+	$(PYTHON) util/cythonpp.py -o gevent.corecext.c gevent/core.ppyx
+	echo                          >> gevent.corecext.c
+	echo '#include "callbacks.c"' >> gevent.corecext.c
+	mv gevent.corecext.* gevent/
 
 gevent/gevent.ares.c: gevent/ares.pyx gevent/*.pxd
 	$(CYTHON) -o gevent.ares.c gevent/ares.pyx
@@ -26,6 +26,7 @@ gevent/gevent._util.c: gevent/_util.pyx
 
 clean:
 	rm -f gevent.core.c gevent.core.h core.pyx gevent/gevent.core.c gevent/gevent.core.h gevent/core.pyx
+	rm -f gevent.corecext.c gevent.corecext.h gevent/gevent.corecext.c gevent/gevent.corecext.h
 	rm -f gevent.ares.c gevent.ares.h gevent/gevent.ares.c gevent/gevent.ares.h
 	rm -f gevent._semaphore.c gevent._semaphore.h gevent/gevent._semaphore.c gevent/gevent._semaphore.h
 	rm -f gevent._util.c gevent._util.h gevent/gevent._util.c gevent/gevent._util.h
@@ -57,7 +58,14 @@ travistest:
 	cd greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config ../known_failures.py --ignore tests_that_dont_use_resolver.txt
 	cd greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config ../known_failures.py `grep -l subprocess test_*.py`
 
-travis:
+travis_pypy:
+	# no need to repeat linters here
+	which ${PYTHON}
+	${PYTHON} --version
+	${PYTHON} setup.py install
+	NWORKERS=1 cd greentest && ${PYTHON} testrunner.py --config ../known_failures.py
+
+travis_cpython:
 	make whitespace
 
 	pip install -q pep8

gevent/__init__.py

@@ -32,7 +32,7 @@ __all__ = ['get_hub',
            'reinit']
 
 
-from gevent.hub import get_hub, iwait, wait
+from gevent.hub import get_hub, iwait, wait, PYPY
 from gevent.greenlet import Greenlet, joinall, killall
 spawn = Greenlet.spawn
 spawn_later = Greenlet.spawn_later

gevent/_socket2.py

@@ -2,6 +2,7 @@
 
 import time
 from gevent import _socketcommon
+from gevent.hub import PYPY
 
 for key in _socketcommon.__dict__:
     if key.startswith('__'):
@@ -36,6 +37,15 @@ class _closedsocket(object):
         raise error(EBADF, 'Bad file descriptor')
     # All _delegate_methods must also be initialized here.
     send = recv = recv_into = sendto = recvfrom = recvfrom_into = _dummy
+
+    if PYPY:
+
+        def _drop(self):
+            pass
+
+        def _reuse(self):
+            pass
+
     __getattr__ = _dummy
 
 
@@ -57,6 +67,8 @@ class socket(object):
             else:
                 self._sock = _sock
                 self.timeout = _socket.getdefaulttimeout()
+            if PYPY:
+                self._sock._reuse()
         self._sock.setblocking(0)
         fileno = self._sock.fileno()
         self.hub = get_hub()
@@ -133,13 +145,19 @@ class socket(object):
                     raise
                 sys.exc_clear()
             self._wait(self._read_event)
-        return socket(_sock=client_socket), address
+        sockobj = socket(_sock=client_socket)
+        if PYPY:
+            client_socket._drop()
+        return sockobj, address
 
     def close(self, _closedsocket=_closedsocket, cancel_wait_ex=cancel_wait_ex):
         # This function should not reference any globals. See Python issue #808164.
         self.hub.cancel_wait(self._read_event, cancel_wait_ex)
         self.hub.cancel_wait(self._write_event, cancel_wait_ex)
+        s = self._sock
         self._sock = _closedsocket()
+        if PYPY:
+            s._drop()
 
     @property
     def closed(self):
@@ -196,7 +214,10 @@ class socket(object):
         #    socket (hence creating a new instance)
         # 2) The resulting fileobject must keep the timeout in order
         #    to be compatible with the stdlib's socket.makefile.
-        return _fileobject(type(self)(_sock=self), mode, bufsize)
+        fobj = _fileobject(type(self)(_sock=self._sock), mode, bufsize)
+        if PYPY:
+            self._sock._drop()
+        return fobj
 
     def recv(self, *args):
         sock = self._sock  # keeping the reference so that fd is not closed during waiting
@@ -340,20 +361,38 @@ class socket(object):
         exec(_s % (_m, _m, _m, _m))
     del _m, _s
 
+    if PYPY:
+
+        def _reuse(self):
+            self._sock._reuse()
+
+        def _drop(self):
+            self._sock._drop()
+
+
 SocketType = socket
 
 if hasattr(_socket, 'socketpair'):
 
     def socketpair(*args):
         one, two = _socket.socketpair(*args)
-        return socket(_sock=one), socket(_sock=two)
+        result = socket(_sock=one), socket(_sock=two)
+        if PYPY:
+            one._drop()
+            two._drop()
+        return result
 else:
     __implements__.remove('socketpair')
 
 if hasattr(_socket, 'fromfd'):
 
     def fromfd(*args):
-        return socket(_sock=_socket.fromfd(*args))
+        s = _socket.fromfd(*args)
+        result = socket(_sock=s)
+        if PYPY:
+            s._drop()
+        return result
+
 else:
     __implements__.remove('fromfd')
 

gevent/_ssl2.py

@@ -18,8 +18,8 @@ except AttributeError:
 import sys
 import errno
 from gevent.socket import socket, _fileobject, timeout_default
-from gevent.socket import error as socket_error
-from gevent.hub import string_types
+from gevent.socket import error as socket_error, EWOULDBLOCK
+from gevent.hub import string_types, PYPY
 
 
 __implements__ = ['SSLSocket',
@@ -65,6 +65,9 @@ class SSLSocket(socket):
                  ciphers=None):
         socket.__init__(self, _sock=sock)
 
+        if PYPY:
+            sock._drop()
+
         if certfile and not keyfile:
             keyfile = certfile
         # see if it's connected
@@ -289,6 +292,17 @@ class SSLSocket(socket):
         else:
             self._makefile_refs -= 1
 
+    if PYPY:
+
+        def _reuse(self):
+            self._makefile_refs += 1
+
+        def _drop(self):
+            if self._makefile_refs < 1:
+                self.close()
+            else:
+                self._makefile_refs -= 1
+
     def do_handshake(self):
         """Perform a TLS/SSL handshake."""
         while True:
@@ -331,8 +345,18 @@ class SSLSocket(socket):
         """Accepts a new connection from a remote client, and returns
         a tuple containing that new connection wrapped with a server-side
         SSL channel, and the address of the remote client."""
-        newsock, addr = socket.accept(self)
-        return (SSLSocket(newsock._sock,
+        sock = self._sock
+        while True:
+            try:
+                client_socket, address = sock.accept()
+                break
+            except socket_error as ex:
+                if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0:
+                    raise
+                sys.exc_clear()
+            self._wait(self._read_event)
+
+        sslobj = SSLSocket(client_socket,
                            keyfile=self.keyfile,
                            certfile=self.certfile,
                            server_side=True,
@@ -341,13 +365,15 @@ class SSLSocket(socket):
                            ca_certs=self.ca_certs,
                            do_handshake_on_connect=self.do_handshake_on_connect,
                            suppress_ragged_eofs=self.suppress_ragged_eofs,
-                          ciphers=self.ciphers),
-                addr)
+                           ciphers=self.ciphers)
+
+        return sslobj, address
 
     def makefile(self, mode='r', bufsize=-1):
         """Make and return a file-like object that
         works with the SSL connection.  Just use the code
         from the socket module."""
+        if not PYPY:
             self._makefile_refs += 1
         # close=True so as to decrement the reference count when done with
         # the file-like object.

gevent/baseserver.py

@@ -126,10 +126,17 @@ class BaseServer(object):
 
     def do_handle(self, *args):
         spawn = self._spawn
+        try:
             if spawn is None:
                 self._handle(*args)
             else:
                 spawn(self._handle, *args)
+        except:
+            self.do_close(*args)
+            raise
+
+    def do_close(self, *args):
+        pass
 
     def _do_read(self):
         for _ in xrange(self.max_accept):

gevent/core.py

+from gevent.hub import PYPY
+
+if PYPY:
+    from gevent import corecffi as _core
+else:
+    from gevent import corecext as _core
+
+
+for item in dir(_core):
+    if item.startswith('__'):
+        continue
+    globals()[item] = getattr(_core, item)
+
+
+__all__ = _core.__all__

gevent/event.py

@@ -150,10 +150,11 @@ class AsyncResult(object):
         >>> import gevent
         >>> result = AsyncResult()
         >>> gevent.spawn(lambda : 1/0).link(result)
-        >>> result.get()
-        Traceback (most recent call last):
-         ...
-        ZeroDivisionError: integer division or modulo by zero
+        >>> try:
+        ...     result.get()
+        ... except ZeroDivisionError:
+        ...     print 'ZeroDivisionError'
+        ZeroDivisionError
     """
     def __init__(self):
         self._links = deque()

gevent/fileobject.py

@@ -8,6 +8,9 @@ from gevent.os import _read, _write, ignored_errors
 from gevent.lock import Semaphore, DummySemaphore
 
 
+PYPY = hasattr(sys, 'pypy_version_info')
+
+
 try:
     from fcntl import fcntl
 except ImportError:
@@ -64,6 +67,7 @@ else:
             io = self.hub.loop.io
             self._read_event = io(fileno, 1)
             self._write_event = io(fileno, 2)
+            self._refcount = 1
 
         def __repr__(self):
             if self._fileno is None:
@@ -86,7 +90,18 @@ else:
             self._fileno = None
             return x
 
+        def _reuse(self):
+            self._refcount += 1
+
+        def _drop(self):
+            self._refcount -= 1
+            if self._refcount <= 0:
+                self._realclose()
+
         def close(self):
+            self._drop()
+
+        def _realclose(self):
             self.hub.cancel_wait(self._read_event, cancel_wait_ex)
             self.hub.cancel_wait(self._write_event, cancel_wait_ex)
             fileno = self._fileno
@@ -161,6 +176,8 @@ else:
             self._fobj = fobj
             self._closed = False
             _fileobject.__init__(self, sock, mode=mode, bufsize=bufsize, close=close)
+            if PYPY:
+                sock._drop()
 
         def __repr__(self):
             if self._sock is None:
@@ -184,6 +201,8 @@ else:
             finally:
                 if self._fobj is not None or not self._close:
                     sock.detach()
+                else:
+                    sock._drop()
                 self._sock = None
                 self._fobj = None
 

gevent/greenlet.py

 # Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details.
 
 import sys
-from gevent.hub import greenlet, getcurrent, get_hub, GreenletExit, Waiter, PY3, iwait, wait
+from gevent.hub import greenlet, getcurrent, get_hub, GreenletExit, Waiter, PY3, iwait, wait, PYPY
 from gevent.timeout import Timeout
 from collections import deque
 
@@ -11,6 +11,11 @@ __all__ = ['Greenlet',
            'killall']
 
 
+if PYPY:
+    import _continuation
+    _continulet = _continuation.continulet
+
+
 class SpawnedLink(object):
     """A wrapper around link that calls it in another greenlet.
 
@@ -96,6 +101,14 @@ class Greenlet(greenlet):
         def __nonzero__(self):
             return self._start_event is not None and self._exception is _NONE
 
+    if PYPY:
+
+        # oops - pypy's .dead relies on __nonzero__ which we overriden above
+
+        @property
+        def dead(self):
+            return self._greenlet__started and not (self._greenlet__main or _continulet.is_pending(self))
+
     @property
     def started(self):
         # DEPRECATED

gevent/hub.py

@@ -28,6 +28,7 @@ __all__ = ['getcurrent',
 
 
 PY3 = sys.version_info[0] >= 3
+PYPY = hasattr(sys, 'pypy_version_info')
 
 
 if PY3:

gevent/local.py

@@ -128,7 +128,7 @@ affects what we see:
 """
 from weakref import WeakKeyDictionary
 from copy import copy
-from gevent.hub import getcurrent
+from gevent.hub import getcurrent, PYPY
 from gevent.lock import RLock
 
 __all__ = ["local"]
@@ -144,7 +144,8 @@ class _localbase(object):
         dicts = WeakKeyDictionary()
         object.__setattr__(self, '_local__dicts', dicts)
 
-        if (args or kw) and (cls.__init__ is object.__init__):
+        if args or kw:
+            if (PYPY and cls.__init__ == object.__init__) or (not PYPY and cls.__init__ is object.__init__):
                 raise TypeError("Initialization arguments are not supported")
 
         # We need to create the greenlet dict in anticipation of

gevent/server.py

@@ -4,6 +4,7 @@ import sys
 import _socket
 from gevent.baseserver import BaseServer
 from gevent.socket import EWOULDBLOCK, socket
+from gevent.hub import PYPY
 
 
 __all__ = ['StreamServer', 'DatagramServer']
@@ -95,7 +96,13 @@ class StreamServer(BaseServer):
             if err.args[0] == EWOULDBLOCK:
                 return
             raise
-        return socket(_sock=client_socket), address
+        sockobj = socket(_sock=client_socket)
+        if PYPY:
+            client_socket._drop()
+        return sockobj, address
+
+    def do_close(self, socket, *args):
+        socket.close()
 
     def wrap_socket_and_handle(self, client_socket, address):
         # used in case of ssl sockets

gevent/threading.py

@@ -32,3 +32,6 @@ class _DummyThread(_DummyThread_):
         rawlink = getattr(g, 'rawlink', None)
         if rawlink is not None:
             rawlink(_cleanup)
+
+    def _Thread__stop(self):
+        pass

greentest/greentest.py

@@ -32,7 +32,9 @@ from gevent.hub import _get_hub
 from functools import wraps
 import contextlib
 import gc
+import six
 
+PYPY = hasattr(sys, 'pypy_version_info')
 VERBOSE = sys.argv.count('-v') > 1
 
 if '--debug-greentest' in sys.argv:
@@ -42,6 +44,7 @@ else:
     DEBUG = False
 
 gettotalrefcount = getattr(sys, 'gettotalrefcount', None)
+OPTIONAL_MODULES = ['resolver_ares']
 
 
 def wrap_switch_count_check(method):
@@ -369,7 +372,9 @@ class ExpectedException(Exception):
     """An exception whose traceback should be ignored"""
 
 
-def walk_modules(basedir=None, modpath=None, include_so=False):
+def walk_modules(basedir=None, modpath=None, include_so=False, recursive=False):
+    if PYPY:
+        include_so = False
     if basedir is None:
         basedir = os.path.dirname(gevent.__file__)
         if modpath is None:
@@ -380,6 +385,8 @@ def walk_modules(basedir=None, modpath=None, include_so=False):
     for fn in sorted(os.listdir(basedir)):
         path = os.path.join(basedir, fn)
         if os.path.isdir(path):
+            if not recursive:
+                continue
             pkg_init = os.path.join(path, '__init__.py')
             if os.path.exists(pkg_init):
                 yield pkg_init, modpath + fn
@@ -390,7 +397,13 @@ def walk_modules(basedir=None, modpath=None, include_so=False):
             x = fn[:-3]
             if x.endswith('_d'):
                 x = x[:-2]
-            if x not in ['__init__', 'core', 'ares', '_util', '_semaphore']:
+            if x in ['__init__', 'core', 'ares', '_util', '_semaphore', 'corecffi']:
+                continue
+            if x in OPTIONAL_MODULES:
+                try:
+                    six.exec_("import %s" % x, {})
+                except ImportError:
+                    continue
             yield path, modpath + x
         elif include_so and fn.endswith('.so'):
             if fn.endswith('_d.so'):
@@ -433,3 +446,14 @@ def get_number_open_files():
     if os.path.exists('/proc/'):
         fd_directory = '/proc/%d/fd' % os.getpid()
         return len(os.listdir(fd_directory))
+
+
+if PYPY:
+
+    def getrefcount(*args):
+        pass
+
+else:
+
+    def getrefcount(*args):
+        return sys.getrefcount(*args)

greentest/test___monkey_patching.py

@@ -8,6 +8,8 @@ import atexit
 
 TIMEOUT = 60
 directory = '%s.%s' % sys.version_info[:2]
+if hasattr(sys, 'pypy_version_info'):
+    directory += 'pypy'
 version = '%s.%s.%s' % sys.version_info[:3]
 
 

greentest/test__api_timeout.py

@@ -23,6 +23,7 @@ import sys
 import greentest
 import weakref
 import time
+import gc
 from gevent import sleep, Timeout
 DELAY = 0.04
 
@@ -104,6 +105,7 @@ class Test(greentest.TestCase):
         with Timeout(DELAY * 2, err):
             sleep(DELAY)
         del err
+        gc.collect()
         assert not err_ref(), repr(err_ref())
 
     def test_nested_timeout(self):

greentest/test__core_watcher.py

@@ -30,6 +30,9 @@ class Test(greentest.TestCase):
             pass
         # test that args can be changed later
         io.args = (1, 2, 3)
+        if greentest.PYPY:
+            pass  # on PYPY .args is just a normal property
+        else:
             # test that only tuple and None are accepted by 'args' attribute
             try:
                 io.args = 5
@@ -43,7 +46,11 @@ class Test(greentest.TestCase):
             except TypeError:
                 pass
         self.assertEqual(io.args, (1, 2, 3))
+        if greentest.PYPY:
+            io.args = ()
+        else:
             # None also works, means empty tuple
+            # XXX why?
             io.args = None
         start = core.time()
         loop.run()

greentest/test__fileobject.py

 import os
+import sys
 import greentest
 import gevent
 from gevent.fileobject import FileObject, FileObjectThread
 
 
+PYPY = hasattr(sys, 'pypy_version_info')
+
+
 class Test(greentest.TestCase):
 
     def _test_del(self, **kwargs):
@@ -11,6 +15,9 @@ class Test(greentest.TestCase):
         s = FileObject(w, 'wb')
         s.write('x')
         s.flush()
+        if PYPY:
+            s.close()
+        else:
             del s
         try:
             os.close(w)
@@ -33,6 +40,9 @@ class Test(greentest.TestCase):
             s = FileObject(w, 'wb', close=False)
             s.write('x')
             s.flush()
+            if PYPY:
+                s.close()
+            else:
                 del s
             os.close(w)
             self.assertEqual(FileObject(r).read(), 'x')
@@ -67,6 +77,9 @@ else:
             r, w = os.pipe()
             s = SocketAdapter(w)
             s.sendall('x')
+            if PYPY:
+                s.close()
+            else:
                 del s
             try:
                 os.close(w)

greentest/test__greenio.py

@@ -20,6 +20,10 @@
 from greentest import TestCase, main, tcp_listener
 import gevent
 from gevent import socket
+import sys
+
+
+PYPY = hasattr(sys, 'pypy_version_info')
 
 
 class TestGreenIo(TestCase):
@@ -76,6 +80,8 @@ class TestGreenIo(TestCase):
         server_greenlet.kill()
 
     def test_del_closes_socket(self):
+        if PYPY:
+            return
         timer = gevent.Timeout.start_new(0.5)
 
         def accept_once(listener):

greentest/test__pool.py

@@ -77,6 +77,9 @@ class TestCoroutinePool(unittest.TestCase):
         evt.wait()
 
     def test_stderr_raising(self):
+        if greentest.PYPY:
+            # Does not work on PyPy
+            return
         # testing that really egregious errors in the error handling code
         # (that prints tracebacks to stderr) don't cause the pool to lose
         # any members

greentest/test__server.py

@@ -11,6 +11,7 @@ class SimpleStreamServer(StreamServer):
 
     def handle(self, client_socket, address):
         fd = client_socket.makefile()
+        try:
             request_line = fd.readline()
             if not request_line:
                 return
@@ -29,6 +30,8 @@ class SimpleStreamServer(StreamServer):
                         break
             else:
                 client_socket.sendall('HTTP/1.0 404 WTF?\r\n\r\n')
+        finally:
+            fd.close()
 
 
 class Settings:
@@ -81,6 +84,7 @@ class TestCase(greentest.TestCase):
         sock.connect((self.server.server_host, self.server.server_port))
         fobj = sock.makefile(bufsize=bufsize)
         fobj._sock.settimeout(timeout)
+        sock.close()
         return fobj
 
     def send_request(self, url='/', timeout=0.1, bufsize=1):

greentest/test__socket.py

@@ -150,6 +150,7 @@ class TestTCP(greentest.TestCase):
             fd = conn.makefile(mode='w')
             fd.write('hello\n')
             fd.close()
+            conn.close()  # for pypy
 
         acceptor = Thread(target=accept_once)
         client = self.create_connection()

greentest/test__socketpair.py

+from gevent import monkey; monkey.patch_all()
+import socket
+import unittest
+
+
+class Test(unittest.TestCase):
+
+    def test(self):
+        msg = 'hello world'
+        x, y = socket.socketpair()
+        x.sendall(msg)
+        x.close()
+        read = y.makefile().read()
+        self.assertEqual(msg, read)
+
+    def test_fromfd(self):
+        msg = 'hello world'
+        x, y = socket.socketpair()
+        xx = socket.fromfd(x.fileno(), x.family, socket.SOCK_STREAM)
+        x.close()
+        yy = socket.fromfd(y.fileno(), y.family, socket.SOCK_STREAM)
+        y.close()
+
+        xx.sendall(msg)
+        xx.close()
+        read = yy.makefile().read()
+        self.assertEqual(msg, read)
+
+
+if __name__ == '__main__':
+    unittest.main()

greentest/test__subprocess.py

@@ -6,6 +6,10 @@ import greentest
 import gevent
 from gevent import subprocess
 import time
+import gc
+
+
+PYPY = hasattr(sys, 'pypy_version_info')
 
 
 if subprocess.mswindows:
@@ -19,6 +23,10 @@ python_universal_newlines = hasattr(sys.stdout, 'newlines')
 
 class Test(greentest.TestCase):
 
+    def setUp(self):
+        gc.collect()
+        gc.collect()
+
     def test_exit(self):
         popen = subprocess.Popen([sys.executable, '-c', 'import sys; sys.exit(10)'])
         self.assertEqual(popen.wait(), 10)
@@ -42,6 +50,9 @@ class Test(greentest.TestCase):
                              stdout=subprocess.PIPE)
         p.wait()
         del p
+        if PYPY:
+            gc.collect()
+            gc.collect()
         num_after = greentest.get_number_open_files()
         self.assertEqual(num_before, num_after)
 

greentest/test__threadpool.py

@@ -6,6 +6,10 @@ import greentest
 from gevent.threadpool import ThreadPool
 import gevent
 import six
+import gc
+
+
+PYPY = hasattr(sys, 'pypy_version_info')
 
 
 class TestCase(greentest.TestCase):
@@ -133,6 +137,22 @@ class TestPool(TestCase):
             self.assertEqual(six.advance_iterator(it), i * i)
         self.assertRaises(StopIteration, lambda: six.advance_iterator(it))
 
+    def test_imap_gc(self):
+        it = self.pool.imap(sqr, range(10))
+        for i in range(10):
+            self.assertEqual(six.advance_iterator(it), i * i)
+            gc.collect()
+        self.assertRaises(StopIteration, lambda: six.advance_iterator(it))
+
+    def test_imap_unordered_gc(self):
+        it = self.pool.imap_unordered(sqr, range(10))
+        result = []
+        for i in range(10):
+            result.append(six.advance_iterator(it))
+            gc.collect()
+        self.assertRaises(StopIteration, lambda: six.advance_iterator(it))
+        self.assertEqual(sorted(result), [x * x for x in range(10)])
+
     def test_imap_random(self):
         it = self.pool.imap(sqr_random_sleep, range(10))
         self.assertEqual(list(it), list(map(sqr, range(10))))
@@ -327,8 +347,11 @@ class TestRef(TestCase):
 
             refs.append(weakref.ref(func))
             del func, result
+            if PYPY:
+                gc.collect()
+                gc.collect()
             for index, r in enumerate(refs):
-                assert r() is None, (index, r(), sys.getrefcount(r()), refs)
+                assert r() is None, (index, r(), greentest.getrefcount(r()), refs)
             assert len(refs) == 4, refs
 
 

greentest/test_ares_timeout.py

 from __future__ import print_function
 import sys
 import gevent
-from gevent.resolver_ares import Resolver
+try:
+    from gevent.resolver_ares import Resolver
+except ImportError as ex:
+    print(ex)
+    sys.exit(0)
 from gevent import socket
 print(gevent.__file__)
 

greentest/test_threading_2.py

@@ -374,7 +374,7 @@ class ThreadTests(unittest.TestCase):
         finally:
             sys.setcheckinterval(old_interval)
 
-    if sys.version_info[:2] > (2, 5):
+    if sys.version_info[:2] > (2, 5) and not hasattr(sys, 'pypy_version_info'):
         def test_no_refcycle_through_target(self):
             class RunSelfFunction(object):
                 def __init__(self, should_raise):

known_failures.py

@@ -44,8 +44,42 @@ if CPYTHON_DBG:
 
 
 if PYPY:
+    FAILING_TESTS += [
+        # Not implemented:
+
         # stat watchers are not implemented on pypy
-    FAILING_TESTS += ['test__core_stat.py']
+        'test__core_stat.py',
+
+        # ares not supported on PyPy yet
+        'test__ares_host_result.py',
+
+        # ---
+
+        # BUGS:
+
+        # https://bugs.pypy.org/issue1743
+        'test__real_greenlet.py',
+        'test__exc_info.py',
+
+        # in CPython we compile _semaphore.py with Cython to make its operation atomic
+        # how to do atomic operations on PyPy?
+        'test__threading_vs_settrace.py',
+
+
+        # check_sendall_interrupted and testInterruptedTimeout fail due to
+        # https://bitbucket.org/cffi/cffi/issue/152/handling-errors-from-signal-handlers-in
+        'test_socket.py',
+
+        # No idea!
+        'test_threading_2.py',
+        'test_threading.py',
+        'test__example_portforwarder.py',
+        'test__pywsgi.py',
+        'test__backdoor.py',
+        'test__refcount.py',
+        'test__server.py',
+        'test_subprocess.py',  # test_executable_without_cwd
+    ]
 
 
 if __name__ == '__main__':

setup.py

@@ -9,9 +9,14 @@ import traceback
 from os.path import join, abspath, basename, dirname
 from glob import glob
 
+PYPY = hasattr(sys, 'pypy_version_info')
+
 try:
     from setuptools import Extension, setup
 except ImportError:
+    if PYPY:
+        # need setuptools for include_package_data to work
+        raise
     from distutils.core import Extension, setup
 from distutils.command.build_ext import build_ext
 from distutils.command.sdist import sdist as _sdist
@@ -19,9 +24,6 @@ from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatfo
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, IOError)
 
 
-# XXX make all env variables that setup.py parses start with GEVENTSETUP_
-
-
 __version__ = re.search("__version__\s*=\s*'(.*)'", open('gevent/__init__.py').read(), re.M).group(1)
 assert __version__
 
@@ -52,8 +54,8 @@ CARES_EMBED = get_config_value('CARES_EMBED', 'EMBED', 'c-ares')
 
 define_macros = []
 libraries = []
-libev_configure_command = ["/bin/sh", abspath('libev/configure'), '> configure-output.txt']
-ares_configure_command = ["/bin/sh", abspath('c-ares/configure'), 'CONFIG_COMMANDS= CONFIG_FILES= > configure-output.txt']
+libev_configure_command = ' '.join(["/bin/sh", abspath('libev/configure'), '> configure-output.txt'])
+ares_configure_command = ' '.join(["/bin/sh", abspath('c-ares/configure'), 'CONFIG_COMMANDS= CONFIG_FILES= > configure-output.txt'])
 
 
 if sys.platform == 'win32':
@@ -70,7 +72,7 @@ def expand(*lst):
 
 
 CORE = Extension(name='gevent.core',
-                 sources=['gevent/gevent.core.c'],
+                 sources=['gevent/gevent.corecext.c'],
                  include_dirs=['libev'] if LIBEV_EMBED else [],
                  libraries=libraries,
                  define_macros=define_macros,
@@ -86,14 +88,6 @@ ARES = Extension(name='gevent.ares',
 ARES.optional = True
 
 
-ext_modules = [CORE,
-               ARES,
-               Extension(name="gevent._semaphore",
-                         sources=["gevent/gevent._semaphore.c"]),
-               Extension(name="gevent._util",
-                         sources=["gevent/gevent._util.c"])]
-
-
 def make_universal_header(filename, *defines):
     defines = [('#define %s ' % define, define) for define in defines]
     lines = open(filename, 'r').read().split('\n')
@@ -114,11 +108,15 @@ def make_universal_header(filename, *defines):
 
 
 def _system(cmd):
-    cmd = ' '.join(cmd)
     sys.stdout.write('Running %r in %s\n' % (cmd, os.getcwd()))
     return os.system(cmd)
 
 
+def system(cmd):
+    if _system(cmd):
+        sys.exit(1)
+
+
 def configure_libev(bext, ext):
     if sys.platform == "win32":
         CORE.define_macros.append(('EV_STANDALONE', '1'))
@@ -170,7 +168,6 @@ if LIBEV_EMBED:
     CORE.define_macros += [('LIBEV_EMBED', '1'),
                            ('EV_COMMON', ''),  # we don't use void* data
                            # libev watchers that we don't use currently:
-                           ('EV_CHECK_ENABLE', '0'),
                            ('EV_CLEANUP_ENABLE', '0'),
                            ('EV_EMBED_ENABLE', '0'),
                            ("EV_PERIODIC_ENABLE", '0')]
@@ -204,8 +201,7 @@ def make(done=[]):
         if os.path.exists('Makefile'):
             if "PYTHON" not in os.environ:
                 os.environ["PYTHON"] = sys.executable
-            if os.system('make'):
-                sys.exit(1)
+            system('make')
         done.append(1)
 
 
@@ -227,7 +223,6 @@ class sdist(_sdist):
 class my_build_ext(build_ext):
 
     def gevent_prepare(self, ext):
-        make()
         configure = getattr(ext, 'configure', None)
         if configure:
             configure(self, ext)
@@ -241,6 +236,11 @@ class my_build_ext(build_ext):
                 raise BuildFailed
             else:
                 raise
+        if not PYPY:
+            self.gevent_symlink(ext)
+        return result
+
+    def gevent_symlink(self, ext):
         # hack: create a symlink from build/../core.so to gevent/core.so
         # to prevent "ImportError: cannot import name core" failures
         try:
@@ -255,7 +255,6 @@ class my_build_ext(build_ext):
                 link(path_to_build_core_so, path_to_core_so)
         except Exception:
             traceback.print_exc()
-        return result
 
 
 def link(source, dest):
@@ -286,7 +285,32 @@ def read(name, *args):
         return ''
 
 
-def run_setup(ext_modules):
+if PYPY:
+    sys.path.insert(0, '.')
+    # XXX ugly - need to find a better way
+    system('cp -r libev gevent/libev')
+    system('touch gevent/libev/__init__.py')
+    system('cd gevent/libev && ./configure > configure_output.txt')
+    from gevent import corecffi
+    ext_modules = [corecffi.ffi.verifier.get_extension()]
+    install_requires = []
+    include_package_data = True
+    run_make = False
+else:
+    ext_modules = [CORE,
+                   ARES,
+                   Extension(name="gevent._semaphore",
+                             sources=["gevent/gevent._semaphore.c"]),
+                   Extension(name="gevent._util",
+                             sources=["gevent/gevent._util.c"])]
+    install_requires = ['greenlet']
+    include_package_data = False
+    run_make = True
+
+
+def run_setup(ext_modules, run_make):
+    if run_make:
+        make()
     setup(
         name='gevent',
         version=__version__,
@@ -296,9 +320,11 @@ def run_setup(ext_modules):
         author_email='denis.bilenko@gmail.com',
         url='http://www.gevent.org/',
         packages=['gevent'],
+        include_package_data=include_package_data,
         ext_modules=ext_modules,
         cmdclass=dict(build_ext=my_build_ext, sdist=sdist),
-        install_requires=['greenlet'],
+        install_requires=install_requires,
+        zip_safe=False,
         classifiers=[
             "License :: OSI Approved :: MIT License",
             "Programming Language :: Python :: 2.6",
@@ -314,11 +340,11 @@ def run_setup(ext_modules):
 
 if __name__ == '__main__':
     try:
-        run_setup(ext_modules)
+        run_setup(ext_modules, run_make=run_make)
     except BuildFailed:
         if ARES not in ext_modules:
             raise
         ext_modules.remove(ARES)
-        run_setup(ext_modules)
+        run_setup(ext_modules, run_make=run_make)
     if ARES not in ext_modules:
         sys.stderr.write('\nWARNING: The gevent.ares extension has been disabled.\n')

util/pyflakes.py

@@ -24,6 +24,8 @@ gevent/wsgi.py:1: 'from gevent.pywsgi import *' used; unable to detect undefined
 examples/webchat/urls.py:1: 'from django.conf.urls.defaults import *' used; unable to detect undefined names
 greentest/test__queue.py:\d+: undefined name 'GenericGetTestCase'
 greentest/test__server_pywsgi.py:
+gevent/core.py:\d+: 'from gevent.corecffi import *' used; unable to detect undefined names
+gevent/core.py:\d+: 'from gevent.corecext import *' used; unable to detect undefined names
 '''
 
 IGNORED = IGNORED.strip().replace(' *', ' \\*').split('\n')