Note dependency on Cython 0.23.4 or above, and revert semaphore workaround for older versions.

README.rst

@@ -41,8 +41,9 @@ Development
 
 To install the latest development version::
 
-  pip install cython git+git://github.com/gevent/gevent.git#egg=gevent
+  pip install 'cython>=0.23.4' git+git://github.com/gevent/gevent.git#egg=gevent
 
+.. note:: You must have Cython installed to build a checkout.
 
 Running Tests
 -------------
@@ -50,6 +51,7 @@ Running Tests
 There are a few different ways to run the tests. To simply run the
 tests on one version of Python during development, try this::
 
+  pip install cython>=0.23.4
   python setup.py build
   cd greentest
   PYTHONPATH=.. python testrunner.py --config ../known_failures.py

changelog.rst

@@ -11,10 +11,10 @@
   ``gevent.lock.Semaphore`` subclasses. If monkey-patched, this could
   also apply to ``threading.Semaphore`` objects. Reported in
   :issue:`660` by Jay Oster.
-- PyPy: Work around the worst case of a bug in Cython 0.23.3 that
-  caused tuples to leak when releasing ``gevent.lock.Semaphore``
-  objects that were linked to (e.g., when a StreamServer had a spawn
-  limit).
+- PyPy: Cython version 0.23.4 or later must be used to avoid a memory
+  leak (`details`_). Thanks to Jay Oster.
+
+.. _details: https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
 
 1.1b5 (Sep 18, 2015)
 ====================

gevent/_semaphore.pxd

@@ -9,7 +9,7 @@ cdef class Semaphore:
     cpdef rawlink(self, object callback)
     cpdef unlink(self, object callback)
     cpdef _start_notify(self)
-    cdef _notify_links(self)
+    cpdef _notify_links(self)
     cdef _do_wait(self, object timeout)
     cpdef int wait(self, object timeout=*) except -1000
     cpdef bint acquire(self, int blocking=*, object timeout=*) except -1000

gevent/_semaphore.pyx

@@ -63,15 +63,10 @@ class Semaphore(object):
             # if needed. (it has a __bool__ method that tells whether it has
             # been run; once it's run once---at the end of the loop---it becomes
             # false.)
-            # Note that we pass Semaphore.__notify_links and not self._notify_links.
-            # This avoids having Cython create a bound method, which on PyPy 2.6.1,
-            # shows up as a leak, at least in the short term (possibly due to the cycles?)
-            # Simply passing it to Python, where Python doesn't keep a reference to it,
-            # shows as a leak. (It's probably this is the same leak as the CallOneArg leak documented in _notify_links.)
-            # Note2: The method and tuple objects still show as a leak until the event
-            # loop is allowed to run, e.g., with gevent.sleep(). Manually running the callbacks
-            # isn't enough.
-            self._notifier = get_hub().loop.run_callback(Semaphore._notify_links, self)
+            # NOTE: Passing the bound method will cause a memory leak on PyPy
+            # with Cython <= 0.23.3. You must use >= 0.23.4.
+            # See  https://bitbucket.org/pypy/pypy/issues/2149/memory-leak-for-python-subclass-of-cpyext#comment-22371546
+            self._notifier = get_hub().loop.run_callback(self._notify_links)
 
     def _notify_links(self):
         # Subclasses CANNOT override. This is a cdef method.
@@ -83,11 +78,6 @@ class Semaphore(object):
         # no need to keep it around until that point (making it potentially climb
         # into older GC generations, notably on PyPy)
         notifier = self._notifier
-        # XXX Cython 0.23.3 and before, at least, have a bug calling arguments of one method on PyPy,
-        # such that a tuple leaks every time.
-        # See https://bitbucket.org/pypy/pypy/issues/2149/memory-leak-for-python-subclass-of-cpyext#comment-22371546
-        # So we do a workaround and force tuple packing/unpacking
-        args = (self,)
         try:
             while True:
                 self._dirty = False
@@ -99,7 +89,7 @@ class Semaphore(object):
                     if self.counter <= 0:
                         return
                     try:
-                        link(*args)
+                        link(self) # Must use Cython >= 0.23.4 on PyPy else this leaks memory
                     except:
                         getcurrent().handle_error((link, self), *sys.exc_info())
                     if self._dirty:
@@ -128,7 +118,7 @@ class Semaphore(object):
         if self._links is None:
             self._links = [callback]
         else:
-            self._links.append(callback) # XXX might leak a tuple on PyPy. See _notify_links
+            self._links.append(callback)
         self._dirty = True
 
     def unlink(self, callback):
@@ -138,7 +128,7 @@ class Semaphore(object):
         Remove the callback set by :meth:`rawlink`
         """
         try:
-            self._links.remove(callback) # XXX might leak a tuple on PyPy. See _notify_links
+            self._links.remove(callback)
             self._dirty = True
         except (ValueError, AttributeError):
             pass
@@ -157,7 +147,6 @@ class Semaphore(object):
         try:
             # As a tiny efficiency optimization, avoid allocating a timer
             # if not needed.
-            # XXX might leak a tuple on PyPy. See _notify_links
             timer = Timeout.start_new(timeout) if timeout is not None else None
             try:
                 try:

rtd-requirements.txt

-cython
+cython >= 0.23.4

setup.py

@@ -345,6 +345,8 @@ elif PYPY:
                    # cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
                    # at around 1s and the compiled version at around 4s). Some clever subclassing
                    # and having only the bare minimum be in cython might help reduce that penalty.
+                   # NOTE: You must use version 0.23.4 or later to avoid a memory leak.
+                   # https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
                    Extension(name="gevent._semaphore",
                              sources=["gevent/gevent._semaphore.c"])]
     include_package_data = True
@@ -360,7 +362,10 @@ else:
     run_make = True
 
 if run_make and os.path.exists("Makefile"):
-    setup_requires = ['cython']
+    # This is effectively pointless and serves only for
+    # documentation/metadata, because we run 'make' *before* we run
+    # setup(), so installing cython happens too late.
+    setup_requires = ['cython >= 0.23.4']
 else:
     setup_requires = []