Merge pull request #1115 from gevent/issue755

Add spawn_tree_locals, spawning_greenlet and spawning_stack to gevent.greenlet.Greenlet

.gitignore

@@ -9,6 +9,7 @@ gevent.*.[ch]
 src/gevent/__pycache__
 src/gevent/_semaphore.c
 src/gevent/local.c
+src/gevent/greenlet.c
 src/gevent/libev/corecext.c
 src/gevent/libev/corecext.h
 src/gevent/libev/_corecffi.c

CHANGES.rst

@@ -85,6 +85,15 @@
 - Signal handling under PyPy with libuv is more reliable. See
   :issue:`1112`.
 
+- Greenlet objects now keep track of their spawning parent greenlet
+  and the code location that spawned them, in addition to maintaining
+  a "spawn tree local" mapping. Based on a proposal from PayPal and
+  comments by Mahmoud Hashemi and Kurt Rose. See :issue:`755` and
+  :pr:`1115`.
+
+- The :mod:`gevent.greenlet` module is now compiled with Cython to
+  offset any performance decrease due to :issue:`755`. Please open
+  issues for any compatibility concerns. See :pr:`1115`.
 
 1.3a1 (2018-01-27)
 ==================

doc/conf.py

@@ -223,7 +223,6 @@ latex_documents = [
 # prevent some stuff from showing up in docs
 import socket
 import gevent.socket
-del gevent.Greenlet.throw
 for item in gevent.socket.__all__[:]:
     if getattr(gevent.socket, item) is getattr(socket, item, None):
         gevent.socket.__all__.remove(item)

doc/gevent.rst

@@ -36,18 +36,7 @@ generated.
 
 .. automethod:: Greenlet.__init__
 
-.. attribute:: Greenlet.value
-
-    Holds the value returned by the function if the greenlet has
-    finished successfully. Until then, or if it finished in error, ``None``.
-
-    .. tip:: Recall that a greenlet killed with the default
-             :class:`GreenletExit` is considered to have finished
-             successfully, and the ``GreenletExit`` exception will be
-             its value.
-
 .. autoattribute:: Greenlet.exception
-
 .. automethod:: Greenlet.ready
 .. automethod:: Greenlet.successful
 .. automethod:: Greenlet.start
@@ -114,6 +103,8 @@ Spawn helpers
 Useful general functions
 ========================
 
+.. seealso:: :mod:`gevent.util`
+
 .. function:: getcurrent()
 
    Return the currently executing greenlet (the one that called this

setup.py

@@ -4,6 +4,7 @@ from __future__ import print_function
 import sys
 import os
 import os.path
+import sysconfig
 
 # setuptools is *required* on Windows
 # (https://bugs.python.org/issue23246) and for PyPy. No reason not to
@@ -59,11 +60,28 @@ LOCAL = Extension(name="gevent.local",
                   depends=['src/gevent/local.pxd'])
 LOCAL = cythonize1(LOCAL)
 
+# The sysconfig dir is not enough if we're in a virtualenv
+# See https://github.com/pypa/pip/issues/4610
+include_dirs = [sysconfig.get_path("include")]
+venv_include_dir = os.path.join(sys.prefix, 'include', 'site',
+                                'python' + sysconfig.get_python_version())
+venv_include_dir = os.path.abspath(venv_include_dir)
+if os.path.exists(venv_include_dir):
+    include_dirs.append(venv_include_dir)
+
+
+GREENLET = Extension(name="gevent.greenlet",
+                     sources=["src/gevent/greenlet.py"],
+                     depends=['src/gevent/greenlet.pxd'],
+                     include_dirs=include_dirs)
+GREENLET = cythonize1(GREENLET)
+
 EXT_MODULES = [
     CORE,
     ARES,
     SEMAPHORE,
     LOCAL,
+    GREENLET,
 ]
 
 LIBEV_CFFI_MODULE = 'src/gevent/libev/_corecffi_build.py:ffi'
@@ -91,6 +109,7 @@ if PYPY:
     setup_requires = []
     EXT_MODULES.remove(CORE)
     EXT_MODULES.remove(LOCAL)
+    EXT_MODULES.remove(GREENLET)
     EXT_MODULES.remove(SEMAPHORE)
     # By building the semaphore with Cython under PyPy, we get
     # atomic operations (specifically, exiting/releasing), at the

src/gevent/greenlet.pxd

+# cython: auto_pickle=False
+
+cimport cython
+
+cdef extern from "greenlet/greenlet.h":
+
+  ctypedef class greenlet.greenlet [object PyGreenlet]:
+      pass
+
+cdef class SpawnedLink:
+    cdef public object callback
+
+
+@cython.final
+cdef class SuccessSpawnedLink(SpawnedLink):
+    pass
+
+@cython.final
+cdef class FailureSpawnedLink(SpawnedLink):
+    pass
+
+@cython.final
+@cython.internal
+cdef class _Frame:
+    cdef readonly object f_code
+    cdef readonly int f_lineno
+    cdef public _Frame f_back
+
+
+@cython.final
+@cython.locals(
+               previous=_Frame,
+               first=_Frame,
+               next_frame=_Frame)
+cdef _Frame _extract_stack(int limit, _Frame f_back)
+
+
+
+cdef class Greenlet(greenlet):
+    cdef readonly object value
+    cdef readonly args
+    cdef readonly object spawning_greenlet
+    cdef public dict spawn_tree_locals
+
+    cdef readonly _Frame spawning_stack
+
+    cdef list _links
+    cdef tuple _exc_info
+    cdef object _notifier
+    cdef object _start_event
+    cdef dict _kwargs
+    cdef str _formatted_info
+
+    cpdef bint has_links(self)
+    cpdef join(self, timeout=*)
+    cpdef bint ready(self)
+    cpdef bint successful(self)
+    cpdef rawlink(self, object callback)
+    cpdef str _formatinfo(self)
+
+    cdef bint __started_but_aborted(self)
+    cdef bint __start_cancelled_by_kill(self)
+    cdef bint __start_pending(self)
+    cdef bint __never_started_or_killed(self)
+    cdef bint __start_completed(self)
+    cdef __handle_death_before_start(self, tuple args)
+
+    cdef __cancel_start(self)
+
+    cdef _report_result(self, object result)
+    cdef _report_error(self, tuple exc_info)
+    # This is used as the target of a callback
+    # from the loop, and so needs to be a cpdef
+    cpdef _notify_links(self)
+    # IMapUnordered greenlets in pools need to access this
+    # method
+    cpdef _raise_exception(self)
+
+# Declare a bunch of imports as cdefs so they can
+# be accessed directly as static vars without
+# doing a module global lookup. This is especially important
+# for spawning greenlets.
+cdef _greenlet__init__
+cdef get_hub
+cdef wref
+cdef getcurrent
+
+cdef Timeout
+cdef dump_traceback
+cdef load_traceback
+cdef Waiter
+cdef wait
+cdef iwait
+cdef reraise
+cdef InvalidSwitchError
+
+
+@cython.final
+@cython.internal
+cdef class _dummy_event:
+    cdef readonly bint pending
+    cdef readonly bint active
+
+    cpdef stop(self)
+    cpdef start(self, cb)
+    cpdef close(self)
+
+cdef _dummy_event _cancelled_start_event
+cdef _dummy_event _start_completed_event
+
+
+@cython.locals(diehards=list)
+cdef _killall3(list greenlets, object exception, object waiter)
+cdef _killall(list greenlets, object exception)
+
+@cython.locals(done=list)
+cpdef joinall(greenlets, timeout=*, raise_error=*, count=*)

src/gevent/greenlet.py

 # Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details.
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
 from __future__ import absolute_import
+
 import sys
+from weakref import ref as wref
+
+
 from greenlet import greenlet
-from gevent._compat import PY3
+from greenlet import getcurrent
+
 from gevent._compat import PYPY
 from gevent._compat import reraise
-from gevent._util import Lazy
 from gevent._tblib import dump_traceback
 from gevent._tblib import load_traceback
 from gevent.hub import GreenletExit
 from gevent.hub import InvalidSwitchError
 from gevent.hub import Waiter
 from gevent.hub import get_hub
-from gevent.hub import getcurrent
 from gevent.hub import iwait
 from gevent.hub import wait
 from gevent.timeout import Timeout
-from collections import deque
-
 
 __all__ = [
     'Greenlet',
@@ -32,7 +34,8 @@ if PYPY:
 
 
 class SpawnedLink(object):
-    """A wrapper around link that calls it in another greenlet.
+    """
+    A wrapper around link that calls it in another greenlet.
 
     Can be called only from main loop.
     """
@@ -87,28 +90,51 @@ class FailureSpawnedLink(SpawnedLink):
         if not source.successful():
             return SpawnedLink.__call__(self, source)
 
+class _Frame(object):
+
+    __slots__ = ('f_code', 'f_lineno', 'f_back')
+
+    def __init__(self, f_code, f_lineno):
+        self.f_code = f_code
+        self.f_lineno = f_lineno
+        self.f_back = None
+
+    f_globals = property(lambda _self: None)
+
+def _extract_stack(limit, f_back):
+    previous = None
+    frame = sys._getframe()
+    first = None
+
+    first = previous = _Frame(frame.f_code, frame.f_lineno)
+    limit -= 1
+    frame = frame.f_back
+
+    while limit and frame is not None:
+        limit -= 1
+        next_frame = _Frame(frame.f_code, frame.f_lineno)
+        previous.f_back = next_frame
+        previous = next_frame
+        frame = frame.f_back
+
+    previous.f_back = f_back
+    return first
+
+
+_greenlet__init__ = greenlet.__init__
+
 class Greenlet(greenlet):
-    """A light-weight cooperatively-scheduled execution unit.
+    """
+    A light-weight cooperatively-scheduled execution unit.
     """
     # pylint:disable=too-many-public-methods,too-many-instance-attributes
 
-    value = None
-    _exc_info = ()
-    _notifier = None
-
-    #: An event, such as a timer or a callback that fires. It is established in
-    #: start() and start_later() as those two objects, respectively.
-    #: Once this becomes non-None, the Greenlet cannot be started again. Conversely,
-    #: kill() and throw() check for non-None to determine if this object has ever been
-    #: scheduled for starting. A placeholder _dummy_event is assigned by them to prevent
-    #: the greenlet from being started in the future, if necessary.
-    _start_event = None
-    args = ()
-    _kwargs = None
+    spawning_stack_limit = 10
 
-    def __init__(self, run=None, *args, **kwargs): # pylint:disable=keyword-arg-before-vararg
+    # pylint:disable=keyword-arg-before-vararg,super-init-not-called
+    def __init__(self, run=None, *args, **kwargs):
         """
-        Greenlet constructor.
+        Greenlet(run=None, *args, **kwargs) -> Greenlet
 
         :param args: The arguments passed to the ``run`` function.
         :param kwargs: The keyword arguments passed to the ``run`` function.
@@ -119,6 +145,53 @@ class Greenlet(greenlet):
             The ``run`` argument to the constructor is now verified to be a callable
             object. Previously, passing a non-callable object would fail after the greenlet
             was spawned.
+
+        .. attribute:: value
+
+            Holds the value returned by the function if the greenlet has
+            finished successfully. Until then, or if it finished in error, ``None``.
+
+            .. tip:: Recall that a greenlet killed with the default
+                     :class:`GreenletExit` is considered to have finished
+                     successfully, and the ``GreenletExit`` exception will be
+                     its value.
+
+
+        .. attribute:: spawn_tree_locals
+
+            A dictionary that is shared between all the greenlets
+            in a "spawn tree", that is, a spawning greenlet and all
+            its descendent greenlets. All children of the main (root)
+            greenlet start their own spawn trees. Assign a new dictionary
+            to this attribute on an instance of this class to create a new
+            spawn tree (as far as locals are concerned).
+
+            .. versionadded:: 1.3a2
+
+        .. attribute:: spawning_greenlet
+
+            A weak-reference to the greenlet that was current when this object
+            was created. Note that the :attr:`parent` attribute is always the
+            hub.
+
+            .. versionadded:: 1.3a2
+
+        .. attribute:: spawning_stack
+
+           A lightweight frame-like object capturing the stack when
+           this greenlet was created as well as the stack when the spawning
+           greenlet was created (if applicable). This can be passed to
+           :func:`traceback.print_stack`.
+
+            .. versionadded:: 1.3a2
+
+        .. attribute:: spawning_stack_limit
+
+            A class attribute specifying how many levels of the spawning
+            stack will be kept. Specify a smaller number for higher performance,
+            specify a larger value for improved debugging.
+
+            .. versionadded:: 1.3a2
         """
         # greenlet.greenlet(run=None, parent=None)
         # Calling it with both positional arguments instead of a keyword
@@ -128,7 +201,27 @@ class Greenlet(greenlet):
         # Python 3.4: 2.32usec with keywords vs 1.74usec with positional
         # Python 3.3: 2.55usec with keywords vs 1.92usec with positional
         # Python 2.7: 1.73usec with keywords vs 1.40usec with positional
-        greenlet.__init__(self, None, get_hub())
+
+        # Timings taken Feb 21 2018 prior to integration of #755
+        # python -m perf timeit -s 'import gevent' 'gevent.Greenlet()'
+        # 3.6.4       : Mean +- std dev: 1.08 us +- 0.05 us
+        # 2.7.14      : Mean +- std dev: 1.44 us +- 0.06 us
+        # PyPy2 5.10.0: Mean +- std dev: 2.14 ns +- 0.08 ns
+
+        # After the integration of spawning_stack, spawning_greenlet,
+        # and spawn_tree_locals on that same date:
+        # 3.6.4       : Mean +- std dev: 8.92 us +- 0.36 us ->  8.2x
+        # 2.7.14      : Mean +- std dev: 14.8 us +- 0.5 us  -> 10.2x
+        # PyPy2 5.10.0: Mean +- std dev: 3.24 us +- 0.17 us ->  1.5x
+
+        # Compiling with Cython gets us to these numbers:
+        # 3.6.4        : Mean +- std dev: 3.63 us +- 0.14 us
+        # 2.7.14       : Mean +- std dev: 3.37 us +- 0.20 us
+        # PyPy2 5.10.0 : Mean +- std dev: 4.44 us +- 0.28 us
+
+
+        _greenlet__init__(self, None, get_hub())
+        #super(Greenlet, self).__init__(None, get_hub())
 
         if run is not None:
             self._run = run
@@ -139,22 +232,45 @@ class Greenlet(greenlet):
         if not callable(self._run):
             raise TypeError("The run argument or self._run must be callable")
 
-        if args:
-            self.args = args
-        if kwargs:
-            self._kwargs = kwargs
+        #: An event, such as a timer or a callback that fires. It is established in
+        #: start() and start_later() as those two objects, respectively.
+        #: Once this becomes non-None, the Greenlet cannot be started again. Conversely,
+        #: kill() and throw() check for non-None to determine if this object has ever been
+        #: scheduled for starting. A placeholder _dummy_event is assigned by them to prevent
+        #: the greenlet from being started in the future, if necessary.
+        self._start_event = None
+        self.args = args
+        self._kwargs = kwargs
+        self.value = None
+        self._notifier = None
+        self._formatted_info = None
+        self._links = []
+
+        # Initial state: None.
+        # Completed successfully: (None, None, None)
+        # Failed with exception: (t, v, dump_traceback(tb)))
+        self._exc_info = None
+
+        spawner = getcurrent()
+        self.spawning_greenlet = wref(spawner)
+        try:
+            self.spawn_tree_locals = spawner.spawn_tree_locals
+        except AttributeError:
+            self.spawn_tree_locals = {}
+            if spawner.parent is not None:
+                # The main greenlet has no parent.
+                # Its children get separate locals.
+                spawner.spawn_tree_locals = self.spawn_tree_locals
+
+
+        self.spawning_stack = _extract_stack(self.spawning_stack_limit,
+                                             getattr(spawner, 'spawning_stack', None))
+
 
     @property
     def kwargs(self):
         return self._kwargs or {}
 
-    @Lazy
-    def _links(self):
-        return deque()
-
-    def _has_links(self):
-        return '_links' in self.__dict__ and self._links
-
     def _raise_exception(self):
         reraise(*self.exc_info)
 
@@ -163,9 +279,14 @@ class Greenlet(greenlet):
         # needed by killall
         return self.parent.loop
 
-    def __bool__(self):
-        return self._start_event is not None and self._exc_info is Greenlet._exc_info
-    __nonzero__ = __bool__
+    def __nonzero__(self):
+        return self._start_event is not None and self._exc_info is None
+    try:
+        __bool__ = __nonzero__ # Python 3
+    except NameError: # pragma: no cover
+        # When we're compiled with Cython, the __nonzero__ function
+        # goes directly into the slot and can't be accessed by name.
+        pass
 
     ### Lifecycle
 
@@ -175,38 +296,33 @@ class Greenlet(greenlet):
         def dead(self):
             if self._greenlet__main:
                 return False
-            if self.__start_cancelled_by_kill or self.__started_but_aborted:
+            if self.__start_cancelled_by_kill() or self.__started_but_aborted():
                 return True
 
             return self._greenlet__started and not _continulet.is_pending(self)
     else:
         @property
         def dead(self):
-            return self.__start_cancelled_by_kill or self.__started_but_aborted or greenlet.dead.__get__(self)
+            return self.__start_cancelled_by_kill() or self.__started_but_aborted() or greenlet.dead.__get__(self)
 
-    @property
     def __never_started_or_killed(self):
         return self._start_event is None
 
-    @property
     def __start_pending(self):
         return (self._start_event is not None
                 and (self._start_event.pending or getattr(self._start_event, 'active', False)))
 
-    @property
     def __start_cancelled_by_kill(self):
         return self._start_event is _cancelled_start_event
 
-    @property
     def __start_completed(self):
         return self._start_event is _start_completed_event
 
-    @property
     def __started_but_aborted(self):
-        return (not self.__never_started_or_killed # we have been started or killed
-                and not self.__start_cancelled_by_kill # we weren't killed, so we must have been started
-                and not self.__start_completed # the start never completed
-                and not self.__start_pending) # and we're not pending, so we must have been aborted
+        return (not self.__never_started_or_killed() # we have been started or killed
+                and not self.__start_cancelled_by_kill() # we weren't killed, so we must have been started
+                and not self.__start_completed() # the start never completed
+                and not self.__start_pending()) # and we're not pending, so we must have been aborted
 
     def __cancel_start(self):
         if self._start_event is None:
@@ -221,9 +337,9 @@ class Greenlet(greenlet):
         self._start_event.stop()
         self._start_event.close()
 
-    def __handle_death_before_start(self, *args):
+    def __handle_death_before_start(self, args):
         # args is (t, v, tb) or simply t or v
-        if self._exc_info is Greenlet._exc_info and self.dead:
+        if self._exc_info is None and self.dead:
             # the greenlet was never switched to before and it will never be, _report_error was not called
             # the result was not set and the links weren't notified. let's do it here.
             # checking that self.dead is true is essential, because throw() does not necessarily kill the greenlet
@@ -253,7 +369,7 @@ class Greenlet(greenlet):
             This function is only guaranteed to return true or false *values*, not
             necessarily the literal constants ``True`` or ``False``.
         """
-        return self.dead or self._exc_info
+        return self.dead or self._exc_info is not None
 
     def successful(self):
         """
@@ -267,7 +383,7 @@ class Greenlet(greenlet):
         .. note:: This function is only guaranteed to return true or false *values*,
               not necessarily the literal constants ``True`` or ``False``.
         """
-        return self._exc_info and self._exc_info[1] is None
+        return self._exc_info is not None and self._exc_info[1] is None
 
     def __repr__(self):
         classname = self.__class__.__name__
@@ -277,19 +393,24 @@ class Greenlet(greenlet):
             result += ': ' + formatted
         return result + '>'
 
-    _formatted_info = None
 
     def _formatinfo(self):
         info = self._formatted_info
         if info is not None:
             return info
 
-        try:
-            result = getfuncname(self.__dict__['_run'])
-        except Exception: # pylint:disable=broad-except
-            # Don't cache
-            return ''
+        # Are we running an arbitrary function provided to the constructor,
+        # or did a subclass override _run?
+        func = self._run
+        im_self = getattr(func, '__self__', None)
+        if im_self is self:
+            funcname = '_run'
+        elif im_self is not None:
+            funcname = repr(func)
+        else:
+            funcname = getattr(func, '__name__', '') or repr(func)
 
+        result = funcname
         args = []
         if self.args:
             args = [repr(x)[:50] for x in self.args]
@@ -303,10 +424,11 @@ class Greenlet(greenlet):
 
     @property
     def exception(self):
-        """Holds the exception instance raised by the function if the greenlet has finished with an error.
-        Otherwise ``None``.
         """
-        return self._exc_info[1] if self._exc_info else None
+        Holds the exception instance raised by the function if the
+        greenlet has finished with an error. Otherwise ``None``.
+        """
+        return self._exc_info[1] if self._exc_info is not None else None
 
     @property
     def exc_info(self):
@@ -319,7 +441,7 @@ class Greenlet(greenlet):
         .. versionadded:: 1.1
         """
         e = self._exc_info
-        if e and e[0] is not None:
+        if e is not None and e[0] is not None:
             return (e[0], e[1], load_traceback(e[2]))
 
     def throw(self, *args):
@@ -342,7 +464,7 @@ class Greenlet(greenlet):
                 # LoopExit.
                 greenlet.throw(self, *args)
         finally:
-            self.__handle_death_before_start(*args)
+            self.__handle_death_before_start(args)
 
     def start(self):
         """Schedule the greenlet to run in this loop iteration"""
@@ -350,7 +472,12 @@ class Greenlet(greenlet):
             self._start_event = self.parent.loop.run_callback(self.switch)
 
     def start_later(self, seconds):
-        """Schedule the greenlet to run in the future loop iteration *seconds* later"""
+        """
+        start_later(seconds) -> None
+
+        Schedule the greenlet to run in the future loop iteration
+        *seconds* later
+        """
         if self._start_event is None:
             self._start_event = self.parent.loop.timer(seconds)
             self._start_event.start(self.switch)
@@ -434,7 +561,7 @@ class Greenlet(greenlet):
         self.__cancel_start()
 
         if self.dead:
-            self.__handle_death_before_start(exception)
+            self.__handle_death_before_start((exception,))
         else:
             waiter = Waiter() if block else None
             self.parent.loop.run_callback(_kill, self, exception, waiter)
@@ -445,11 +572,17 @@ class Greenlet(greenlet):
         # thus it should not raise when the greenlet is already killed (= not started)
 
     def get(self, block=True, timeout=None):
-        """Return the result the greenlet has returned or re-raise the exception it has raised.
+        """
+        get(block=True, timeout=None) -> object
 
-        If block is ``False``, raise :class:`gevent.Timeout` if the greenlet is still alive.
-        If block is ``True``, unschedule the current greenlet until the result is available
-        or the timeout expires. In the latter case, :class:`gevent.Timeout` is raised.
+        Return the result the greenlet has returned or re-raise the
+        exception it has raised.
+
+        If block is ``False``, raise :class:`gevent.Timeout` if the
+        greenlet is still alive. If block is ``True``, unschedule the
+        current greenlet until the result is available or the timeout
+        expires. In the latter case, :class:`gevent.Timeout` is
+        raised.
         """
         if self.ready():
             if self.successful():
@@ -483,8 +616,11 @@ class Greenlet(greenlet):
             self._raise_exception()
 
     def join(self, timeout=None):
-        """Wait until the greenlet finishes or *timeout* expires.
-        Return ``None`` regardless.
+        """
+        join(timeout=None) -> None
+
+        Wait until the greenlet finishes or *timeout* expires. Return
+        ``None`` regardless.
         """
         if self.ready():
             return
@@ -510,7 +646,7 @@ class Greenlet(greenlet):
     def _report_result(self, result):
         self._exc_info = (None, None, None)
         self.value = result
-        if self._has_links() and not self._notifier:
+        if self._links and not self._notifier:
             self._notifier = self.parent.loop.run_callback(self._notify_links)
 
     def _report_error(self, exc_info):
@@ -520,7 +656,7 @@ class Greenlet(greenlet):
 
         self._exc_info = exc_info[0], exc_info[1], dump_traceback(exc_info[2])
 
-        if self._has_links() and not self._notifier:
+        if self._links and not self._notifier:
             self._notifier = self.parent.loop.run_callback(self._notify_links)
 
         try:
@@ -541,8 +677,8 @@ class Greenlet(greenlet):
             self._report_result(result)
         finally:
             self.__dict__.pop('_run', None)
-            self.__dict__.pop('args', None)
-            self.__dict__.pop('kwargs', None)
+            self.args = ()
+            self._kwargs = None
 
     def _run(self):
         """Subclasses may override this method to take any number of arguments and keyword arguments.
@@ -555,6 +691,9 @@ class Greenlet(greenlet):
         # pylint: disable=method-hidden
         return
 
+    def has_links(self):
+        return len(self._links)
+
     def rawlink(self, callback):
         """Register a callable to be executed when the greenlet finishes execution.
 
@@ -589,6 +728,14 @@ class Greenlet(greenlet):
         except ValueError:
             pass
 
+    def unlink_all(self):
+        """
+        Remove all the callbacks.
+
+        .. versionadded:: 1.3a2
+        """
+        del self._links[:]
+
     def link_value(self, callback, SpawnedLink=SuccessSpawnedLink):
         """
         Like :meth:`link` but *callback* is only notified when the greenlet
@@ -598,13 +745,23 @@ class Greenlet(greenlet):
         self.link(callback, SpawnedLink=SpawnedLink)
 
     def link_exception(self, callback, SpawnedLink=FailureSpawnedLink):
-        """Like :meth:`link` but *callback* is only notified when the greenlet dies because of an unhandled exception."""
+        """
+        Like :meth:`link` but *callback* is only notified when the
+        greenlet dies because of an unhandled exception.
+        """
         # pylint:disable=redefined-outer-name
         self.link(callback, SpawnedLink=SpawnedLink)
 
     def _notify_links(self):
         while self._links:
-            link = self._links.popleft() # pylint:disable=no-member
+            # Early links are allowed to remove later links
+            # before we get to them, and they're also allowed to
+            # add new links, so we have to be careful about iterating.
+
+            # We don't expect this list to be very large, so the time spent
+            # manipulating it should be small. a deque is probably not justified.
+            # Cython has optimizations to transform this into a memmove anyway.
+            link = self._links.pop(0)
             try:
                 link(self)
             except: # pylint:disable=bare-except
@@ -612,8 +769,10 @@ class Greenlet(greenlet):
 
 
 class _dummy_event(object):
-    pending = False
-    active = False
+    __slots__ = ('pending', 'active')
+
+    def __init__(self):
+        self.pending = self.active = False
 
     def stop(self):
         pass
@@ -621,11 +780,11 @@ class _dummy_event(object):
     def start(self, cb): # pylint:disable=unused-argument
         raise AssertionError("Cannot start the dummy event")
 
-    close = stop
+    def close(self):
+        pass
 
 _cancelled_start_event = _dummy_event()
 _start_completed_event = _dummy_event()
-del _dummy_event
 
 
 def _kill(glet, exception, waiter):
@@ -726,21 +885,3 @@ def killall(greenlets, exception=GreenletExit, block=True, timeout=None):
             t.cancel()
     else:
         loop.run_callback(_killall, greenlets, exception)
-
-
-if PY3:
-    _meth_self = "__self__"
-else:
-    _meth_self = "im_self"
-
-
-def getfuncname(func):
-    if not hasattr(func, _meth_self):
-        try:
-            funcname = func.__name__
-        except AttributeError:
-            pass
-        else:
-            if funcname != '<lambda>':
-                return funcname
-    return repr(func)

src/gevent/hub.py

@@ -9,6 +9,7 @@ from functools import partial as _functools_partial
 import os
 import sys
 import traceback
+from weakref import ref as wref
 
 from greenlet import greenlet as RawGreenlet, getcurrent, GreenletExit
 
@@ -110,6 +111,10 @@ def spawn_raw(function, *args, **kwargs):
     occasionally be useful as an optimization if there are many
     greenlets involved.
 
+    .. versionchanged:: 1.1a3
+        Verify that ``function`` is callable, raising a TypeError if not. Previously,
+        the spawned greenlet would have failed the first time it was switched to.
+
     .. versionchanged:: 1.1b1
        If *function* is not callable, immediately raise a :exc:`TypeError`
        instead of spawning a greenlet that will raise an uncaught TypeError.
@@ -118,12 +123,15 @@ def spawn_raw(function, *args, **kwargs):
         Accept keyword arguments for ``function`` as previously (incorrectly)
         documented. Note that this may incur an additional expense.
 
-    .. versionchanged:: 1.1a3
-        Verify that ``function`` is callable, raising a TypeError if not. Previously,
-        the spawned greenlet would have failed the first time it was switched to.
+    .. versionchanged:: 1.3a2
+       Populate the ``spawning_greenlet`` and ``spawn_tree_locals``
+       attributes of the returned greenlet.
+
     """
     if not callable(function):
         raise TypeError("function must be callable")
+
+    # The hub is always the parent.
     hub = get_hub()
 
     # The callback class object that we use to run this doesn't
@@ -136,6 +144,19 @@ def spawn_raw(function, *args, **kwargs):
     else:
         g = RawGreenlet(function, hub)
         hub.loop.run_callback(g.switch, *args)
+
+    # See greenlet.py's Greenlet class. We capture the cheap
+    # parts to maintain the tree structure, but we do not capture
+    # the stack because that's too expensive.
+    current = getcurrent()
+    g.spawning_greenlet = wref(current)
+    # See Greenlet for how trees are maintained.
+    try:
+        g.spawn_tree_locals = current.spawn_tree_locals
+    except AttributeError:
+        g.spawn_tree_locals = {}
+        if current.parent:
+            current.spawn_tree_locals = g.spawn_tree_locals
     return g
 
 

src/gevent/util.py

@@ -3,11 +3,14 @@
 Low-level utilities.
 """
 
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function, division
 
 import functools
 
-__all__ = ['wrap_errors']
+__all__ = [
+    'wrap_errors',
+    'format_run_info',
+]
 
 
 class wrap_errors(object):
@@ -59,7 +62,7 @@ class wrap_errors(object):
     def __getattr__(self, name):
         return getattr(self.__func, name)
 
-def dump_stacks():
+def format_run_info():
     """
     Request information about the running threads of the current process.
 
@@ -68,39 +71,82 @@ def dump_stacks():
 
     :return: A sequence of text lines detailing the stacks of running
             threads and greenlets. (One greenlet will duplicate one thread,
-            the current thread and greenlet.)
+            the current thread and greenlet.) Extra information about
+            :class:`gevent.greenlet.Greenlet` object will also be returned.
 
     .. versionadded:: 1.3a1
+    .. versionchanged:: 1.3a2
+       Renamed from ``dump_stacks`` to reflect the fact that this
+       prints additional information about greenlets, including their
+       spawning stack, parent, and any spawn tree locals.
     """
-    dump = []
 
-    # threads
-    import threading  # Late import this stuff because it may get monkey-patched
-    import traceback
-    import sys
-    import gc
+    lines = []
 
-    from greenlet import greenlet
+    _format_thread_info(lines)
+    _format_greenlet_info(lines)
+    return lines
+
+def _format_thread_info(lines):
+    import threading
+    import sys
+    import traceback
 
     threads = {th.ident: th.name for th in threading.enumerate()}
 
+    lines.append('*' * 80)
+    lines.append('* Threads')
+
+    thread = None
+    frame = None
     for thread, frame in sys._current_frames().items():
-        dump.append('Thread 0x%x (%s)\n' % (thread, threads.get(thread)))
-        dump.append(''.join(traceback.format_stack(frame)))
-        dump.append('\n')
+        lines.append("*" * 80)
+        lines.append('Thread 0x%x (%s)\n' % (thread, threads.get(thread)))
+        lines.append(''.join(traceback.format_stack(frame)))
+
+    # We may have captured our own frame, creating a reference
+    # cycle, so clear it out.
+    del thread
+    del frame
+    del lines
+    del threads
+
+def _format_greenlet_info(lines):
+    from greenlet import greenlet
+    import pprint
+    import traceback
+    import gc
 
-    # greenlets
+    def _noop():
+        return None
 
-    # if greenlet is present, let's dump each greenlet stack
     # Use the gc module to inspect all objects to find the greenlets
     # since there isn't a global registry
+    lines.append('*' * 80)
+    lines.append('* Greenlets')
+    seen_locals = set() # {id}
     for ob in gc.get_objects():
         if not isinstance(ob, greenlet):
             continue
         if not ob:
             continue  # not running anymore or not started
-        dump.append('Greenlet %s\n' % ob)
-        dump.append(''.join(traceback.format_stack(ob.gr_frame)))
-        dump.append('\n')
-
-    return dump
+        lines.append('*' * 80)
+        lines.append('Greenlet %s\n' % ob)
+        lines.append(''.join(traceback.format_stack(ob.gr_frame)))
+        spawning_stack = getattr(ob, 'spawning_stack', None)
+        if spawning_stack:
+            lines.append("Spawned at: ")
+            lines.append(''.join(traceback.format_stack(spawning_stack)))
+        parent = getattr(ob, 'spawning_greenlet', _noop)()
+        if parent is not None:
+            lines.append("Parent greenlet: %s\n" % (parent,))
+        spawn_tree_locals = getattr(ob, 'spawn_tree_locals', None)
+        if spawn_tree_locals and id(spawn_tree_locals) not in seen_locals:
+            seen_locals.add(id(spawn_tree_locals))
+            lines.append("Spawn Tree Locals:\n")
+            lines.append(pprint.pformat(spawn_tree_locals))
+
+
+    del lines
+
+dump_stacks = format_run_info

src/greentest/test__greenlet.py

@@ -96,14 +96,17 @@ class TestUnlink(greentest.TestCase):
 
     def _test_func(self, p, link):
         link(dummy_test_func)
-        assert len(p._links) == 1, p._links
+        self.assertEqual(1, p.has_links())
+
         p.unlink(dummy_test_func)
-        assert not p._links, p._links
+        self.assertEqual(0, p.has_links())
 
         link(self.setUp)
-        assert len(p._links) == 1, p._links
+        self.assertEqual(1, p.has_links())
+
         p.unlink(self.setUp)
-        assert not p._links, p._links
+        self.assertEqual(0, p.has_links())
+
         p.kill()
 
     def test_func_link(self):
@@ -170,8 +173,7 @@ class TestReturn_link(LinksTestCase):
     p = None
 
     def cleanup(self):
-        while self.p._links:
-            self.p._links.pop()
+        self.p.unlink_all()
         self.p = None
 
     def test_return(self):
@@ -374,7 +376,8 @@ class TestStuff(greentest.TestCase):
         link(results.listener2)
         link(results.listener3)
         sleep(DELAY * 10)
-        assert results.results == [5], results.results
+        self.assertEqual([5], results.results)
+
 
     def test_multiple_listeners_error_unlink_Greenlet_link(self):
         p = gevent.spawn(lambda: 5)
@@ -402,6 +405,8 @@ class A(object):
 
 hexobj = re.compile('-?0x[0123456789abcdef]+L?', re.I)
 
+class Subclass(gevent.Greenlet):
+    pass
 
 class TestStr(greentest.TestCase):
 
@@ -425,6 +430,17 @@ class TestStr(greentest.TestCase):
         str_g = str_g.replace(__name__, 'module')
         self.assertEqual(str_g, '<Greenlet at X: <bound method A.method of <module.A object at X>>>')
 
+    def test_subclass(self):
+        g = Subclass()
+        str_g = hexobj.sub('X', str(g))
+        str_g = str_g.replace(__name__, 'module')
+        self.assertEqual(str_g, '<Subclass at X: _run>')
+
+        g = Subclass(None, 'question', answer=42)
+        str_g = hexobj.sub('X', str(g))
+        str_g = str_g.replace(__name__, 'module')
+        self.assertEqual(str_g, "<Subclass at X: _run('question', answer=42)>")
+
 
 class TestJoin(AbstractGenericWaitTestCase):
 
@@ -515,14 +531,14 @@ class TestBasic(greentest.TestCase):
         assert g.exception is None
 
         gevent.sleep(0.001)
-        assert g
-        assert not g.dead
-        assert g.started
-        assert not g.ready()
-        assert not g.successful()
-        assert g.value is None
-        assert g.exception is None
-        assert not link_test
+        self.assertTrue(g)
+        self.assertFalse(g.dead, g)
+        self.assertTrue(g.started, g)
+        self.assertFalse(g.ready(), g)
+        self.assertFalse(g.successful(), g)
+        self.assertIsNone(g.value, g)
+        self.assertIsNone(g.exception, g)
+        self.assertFalse(link_test)
 
         gevent.sleep(0.02)
         assert not g
@@ -649,6 +665,29 @@ class TestBasic(greentest.TestCase):
         g.join()
         self.assertFalse(g.exc_info)
 
+    def test_tree_locals(self):
+        g = g2 = None
+        def func():
+            child = greenlet.Greenlet()
+            self.assertIs(child.spawn_tree_locals, getcurrent().spawn_tree_locals)
+            self.assertIs(child.spawning_greenlet(), getcurrent())
+        g = greenlet.Greenlet(func)
+        g2 = greenlet.Greenlet(func)
+        # Creating those greenlets did not give the main greenlet
+        # a locals dict.
+        self.assertFalse(hasattr(getcurrent(), 'spawn_tree_locals'),
+                         getcurrent())
+        self.assertIsNot(g.spawn_tree_locals, g2.spawn_tree_locals)
+        g.start()
+        g.join()
+
+        raw = gevent.spawn_raw(func)
+        self.assertIsNotNone(raw.spawn_tree_locals)
+        self.assertIsNot(raw.spawn_tree_locals, g.spawn_tree_locals)
+        self.assertIs(raw.spawning_greenlet(), getcurrent())
+        while not raw.dead:
+            gevent.sleep(0.01)
+
 
 class TestStart(greentest.TestCase):
 

src/greentest/test__issue330.py

 # A greenlet that's killed before it is ever started
 # should never be switched to
 import gevent
+import greentest
 
-switched_to = [False, False]
 
-
-def runner(i):
-    switched_to[i] = True
-
-
-def check(g, g2):
-    gevent.joinall((g, g2))
-    assert switched_to == [False, False], switched_to
-
-    # They both have a GreenletExit as their value
-    assert isinstance(g.value, gevent.GreenletExit)
-    assert isinstance(g2.value, gevent.GreenletExit)
-    # They both have no reported exc_info
-    assert g._exc_info == (None, None, None)
-    assert g2._exc_info == (None, None, None)
-    assert g._exc_info is not type(g)._exc_info
-    assert g2._exc_info is not type(g2)._exc_info
-
-    switched_to[:] = [False, False]
-
-g = gevent.spawn(runner, 0) # create but do not switch to
-g2 = gevent.spawn(runner, 1) # create but do not switch to
-# Using gevent.kill
-gevent.kill(g)
-gevent.kill(g2)
-check(g, g2)
-
-# killing directly
-g = gevent.spawn(runner, 0)
-g2 = gevent.spawn(runner, 1)
-g.kill()
-g2.kill()
-check(g, g2)
-
-# throwing
-g = gevent.spawn(runner, 0)
-g2 = gevent.spawn(runner, 1)
-g.throw(gevent.GreenletExit)
-g2.throw(gevent.GreenletExit)
-check(g, g2)
-
-
-# Killing with gevent.kill gets the right exception
 class MyException(Exception):
     pass
 
-
-def catcher():
-    try:
-        while True:
-            gevent.sleep(0)
-    except Exception as e:
-        switched_to[0] = e
-
-g = gevent.spawn(catcher)
-g.start()
-gevent.sleep()
-gevent.kill(g, MyException())
-gevent.sleep()
-
-assert isinstance(switched_to[0], MyException), switched_to
+class TestSwitch(greentest.TestCase):
+
+    def setUp(self):
+        self.switched_to = [False, False]
+        self.caught = None
+
+    def runner(self, i):
+        self.switched_to[i] = True
+
+    def check(self, g, g2):
+        gevent.joinall((g, g2))
+        self.assertEqual([False, False], self.switched_to)
+
+        # They both have a GreenletExit as their value
+        self.assertIsInstance(g.value, gevent.GreenletExit)
+        self.assertIsInstance(g2.value, gevent.GreenletExit)
+
+        # They both have no reported exc_info
+        self.assertIsNone(g.exc_info)
+        self.assertIsNone(g2.exc_info)
+        self.assertIsNone(g.exception)
+        self.assertIsNone(g2.exception)
+
+
+    def test_gevent_kill(self):
+        g = gevent.spawn(self.runner, 0) # create but do not switch to
+        g2 = gevent.spawn(self.runner, 1) # create but do not switch to
+        # Using gevent.kill
+        gevent.kill(g)
+        gevent.kill(g2)
+        self.check(g, g2)
+
+    def test_greenlet_kill(self):
+        # killing directly
+        g = gevent.spawn(self.runner, 0)
+        g2 = gevent.spawn(self.runner, 1)
+        g.kill()
+        g2.kill()
+        self.check(g, g2)
+
+    def test_throw(self):
+        # throwing
+        g = gevent.spawn(self.runner, 0)
+        g2 = gevent.spawn(self.runner, 1)
+        g.throw(gevent.GreenletExit)
+        g2.throw(gevent.GreenletExit)
+        self.check(g, g2)
+
+
+    def catcher(self):
+        try:
+            while True:
+                gevent.sleep(0)
+        except MyException as e:
+            self.caught = e
+
+    def test_kill_exception(self):
+        # Killing with gevent.kill gets the right exception
+
+        g = gevent.spawn(self.catcher)
+        g.start()
+        gevent.sleep()
+        gevent.kill(g, MyException())
+        gevent.sleep()
+
+        self.assertIsInstance(self.caught, MyException)
+        self.assertIsNone(g.exception, MyException)
+
+
+if __name__ == '__main__':
+    greentest.main()

src/greentest/test__local.py

@@ -273,7 +273,7 @@ class GeventLocalTestCase(greentest.TestCase):
             self.assertEqual(count, len(deleted_sentinels))
 
             # The links were removed as well.
-            self.assertEqual(list(running_greenlet._links), [])
+            self.assertFalse(running_greenlet.has_links())
 
 
         running_greenlet = gevent.spawn(demonstrate_my_local)

src/greentest/test__util.py

+# -*- coding: utf-8 -*-
+# Copyright 2018 gevent contributes
+# See LICENSE for details.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import greentest
+
+import gevent
+from gevent import util
+
+@greentest.skipOnPyPy("5.10.x is *very* slow formatting stacks")
+class TestFormat(greentest.TestCase):
+
+    def test_basic(self):
+        lines = util.format_run_info()
+
+        value = '\n'.join(lines)
+        self.assertIn('Threads', value)
+        self.assertIn('Greenlets', value)
+
+        # because it's a raw greenlet, we have no data for it.
+        self.assertNotIn("Spawned at", value)
+        self.assertNotIn("Parent greenlet", value)
+        self.assertNotIn("Spawn Tree Locals", value)
+
+    def test_with_Greenlet(self):
+        def root():
+            gevent.getcurrent().spawn_tree_locals['a value'] = 42
+            g = gevent.spawn(util.format_run_info)
+            g.join()
+            return g.value
+
+        g = gevent.spawn(root)
+        g.join()
+        value = '\n'.join(g.value)
+
+        self.assertIn("Spawned at", value)
+        self.assertIn("Parent greenlet", value)
+        self.assertIn("Spawn Tree Locals", value)
+
+
+if __name__ == '__main__':
+    greentest.main()

src/greentest/tests_that_dont_monkeypatch.txt

@@ -13,3 +13,4 @@ test__socket_errors.py
 test__socket_send_memoryview.py
 test__socket_timeout.py
 test__examples.py
+test__issue330.py

src/greentest/tests_that_dont_use_resolver.txt

@@ -126,3 +126,4 @@ test_asyncore.py
 
 test___config.py
 test__destroy_default_loop.py
+test__util.py