Merge pull request #1115 from gevent/issue755

Add spawn_tree_locals, spawning_greenlet and spawning_stack to gevent.greenlet.Greenlet

.gitignore

@@ -9,6 +9,7 @@ gevent.*.[ch]
 src/gevent/__pycache__
 src/gevent/_semaphore.c
 src/gevent/local.c
+src/gevent/greenlet.c
 src/gevent/libev/corecext.c
 src/gevent/libev/corecext.h
 src/gevent/libev/_corecffi.c

CHANGES.rst

@@ -85,6 +85,15 @@
 - Signal handling under PyPy with libuv is more reliable. See
   :issue:`1112`.
 
+- Greenlet objects now keep track of their spawning parent greenlet
+  and the code location that spawned them, in addition to maintaining
+  a "spawn tree local" mapping. Based on a proposal from PayPal and
+  comments by Mahmoud Hashemi and Kurt Rose. See :issue:`755` and
+  :pr:`1115`.
+
+- The :mod:`gevent.greenlet` module is now compiled with Cython to
+  offset any performance decrease due to :issue:`755`. Please open
+  issues for any compatibility concerns. See :pr:`1115`.
 
 1.3a1 (2018-01-27)
 ==================

doc/conf.py

@@ -223,7 +223,6 @@ latex_documents = [
 # prevent some stuff from showing up in docs
 import socket
 import gevent.socket
-del gevent.Greenlet.throw
 for item in gevent.socket.__all__[:]:
     if getattr(gevent.socket, item) is getattr(socket, item, None):
         gevent.socket.__all__.remove(item)

doc/gevent.rst

@@ -36,18 +36,7 @@ generated.
 
 .. automethod:: Greenlet.__init__
 
-.. attribute:: Greenlet.value
-
-    Holds the value returned by the function if the greenlet has
-    finished successfully. Until then, or if it finished in error, ``None``.
-
-    .. tip:: Recall that a greenlet killed with the default
-             :class:`GreenletExit` is considered to have finished
-             successfully, and the ``GreenletExit`` exception will be
-             its value.
-
 .. autoattribute:: Greenlet.exception
-
 .. automethod:: Greenlet.ready
 .. automethod:: Greenlet.successful
 .. automethod:: Greenlet.start
@@ -114,6 +103,8 @@ Spawn helpers
 Useful general functions
 ========================
 
+.. seealso:: :mod:`gevent.util`
+
 .. function:: getcurrent()
 
    Return the currently executing greenlet (the one that called this

setup.py

@@ -4,6 +4,7 @@ from __future__ import print_function
 import sys
 import os
 import os.path
+import sysconfig
 
 # setuptools is *required* on Windows
 # (https://bugs.python.org/issue23246) and for PyPy. No reason not to
@@ -59,11 +60,28 @@ LOCAL = Extension(name="gevent.local",
                   depends=['src/gevent/local.pxd'])
 LOCAL = cythonize1(LOCAL)
 
+# The sysconfig dir is not enough if we're in a virtualenv
+# See https://github.com/pypa/pip/issues/4610
+include_dirs = [sysconfig.get_path("include")]
+venv_include_dir = os.path.join(sys.prefix, 'include', 'site',
+                                'python' + sysconfig.get_python_version())
+venv_include_dir = os.path.abspath(venv_include_dir)
+if os.path.exists(venv_include_dir):
+    include_dirs.append(venv_include_dir)
+
+
+GREENLET = Extension(name="gevent.greenlet",
+                     sources=["src/gevent/greenlet.py"],
+                     depends=['src/gevent/greenlet.pxd'],
+                     include_dirs=include_dirs)
+GREENLET = cythonize1(GREENLET)
+
 EXT_MODULES = [
     CORE,
     ARES,
     SEMAPHORE,
     LOCAL,
+    GREENLET,
 ]
 
 LIBEV_CFFI_MODULE = 'src/gevent/libev/_corecffi_build.py:ffi'
@@ -91,6 +109,7 @@ if PYPY:
     setup_requires = []
     EXT_MODULES.remove(CORE)
     EXT_MODULES.remove(LOCAL)
+    EXT_MODULES.remove(GREENLET)
     EXT_MODULES.remove(SEMAPHORE)
     # By building the semaphore with Cython under PyPy, we get
     # atomic operations (specifically, exiting/releasing), at the

src/gevent/greenlet.pxd

+# cython: auto_pickle=False
+
+cimport cython
+
+cdef extern from "greenlet/greenlet.h":
+
+  ctypedef class greenlet.greenlet [object PyGreenlet]:
+      pass
+
+cdef class SpawnedLink:
+    cdef public object callback
+
+
+@cython.final
+cdef class SuccessSpawnedLink(SpawnedLink):
+    pass
+
+@cython.final
+cdef class FailureSpawnedLink(SpawnedLink):
+    pass
+
+@cython.final
+@cython.internal
+cdef class _Frame:
+    cdef readonly object f_code
+    cdef readonly int f_lineno
+    cdef public _Frame f_back
+
+
+@cython.final
+@cython.locals(
+               previous=_Frame,
+               first=_Frame,
+               next_frame=_Frame)
+cdef _Frame _extract_stack(int limit, _Frame f_back)
+
+
+
+cdef class Greenlet(greenlet):
+    cdef readonly object value
+    cdef readonly args
+    cdef readonly object spawning_greenlet
+    cdef public dict spawn_tree_locals
+
+    cdef readonly _Frame spawning_stack
+
+    cdef list _links
+    cdef tuple _exc_info
+    cdef object _notifier
+    cdef object _start_event
+    cdef dict _kwargs
+    cdef str _formatted_info
+
+    cpdef bint has_links(self)
+    cpdef join(self, timeout=*)
+    cpdef bint ready(self)
+    cpdef bint successful(self)
+    cpdef rawlink(self, object callback)
+    cpdef str _formatinfo(self)
+
+    cdef bint __started_but_aborted(self)
+    cdef bint __start_cancelled_by_kill(self)
+    cdef bint __start_pending(self)
+    cdef bint __never_started_or_killed(self)
+    cdef bint __start_completed(self)
+    cdef __handle_death_before_start(self, tuple args)
+
+    cdef __cancel_start(self)
+
+    cdef _report_result(self, object result)
+    cdef _report_error(self, tuple exc_info)
+    # This is used as the target of a callback
+    # from the loop, and so needs to be a cpdef
+    cpdef _notify_links(self)
+    # IMapUnordered greenlets in pools need to access this
+    # method
+    cpdef _raise_exception(self)
+
+# Declare a bunch of imports as cdefs so they can
+# be accessed directly as static vars without
+# doing a module global lookup. This is especially important
+# for spawning greenlets.
+cdef _greenlet__init__
+cdef get_hub
+cdef wref
+cdef getcurrent
+
+cdef Timeout
+cdef dump_traceback
+cdef load_traceback
+cdef Waiter
+cdef wait
+cdef iwait
+cdef reraise
+cdef InvalidSwitchError
+
+
+@cython.final
+@cython.internal
+cdef class _dummy_event:
+    cdef readonly bint pending
+    cdef readonly bint active
+
+    cpdef stop(self)
+    cpdef start(self, cb)
+    cpdef close(self)
+
+cdef _dummy_event _cancelled_start_event
+cdef _dummy_event _start_completed_event
+
+
+@cython.locals(diehards=list)
+cdef _killall3(list greenlets, object exception, object waiter)
+cdef _killall(list greenlets, object exception)
+
+@cython.locals(done=list)
+cpdef joinall(greenlets, timeout=*, raise_error=*, count=*)

src/gevent/hub.py

@@ -9,6 +9,7 @@ from functools import partial as _functools_partial
 import os
 import sys
 import traceback
+from weakref import ref as wref
 
 from greenlet import greenlet as RawGreenlet, getcurrent, GreenletExit
 
@@ -110,6 +111,10 @@ def spawn_raw(function, *args, **kwargs):
     occasionally be useful as an optimization if there are many
     greenlets involved.
 
+    .. versionchanged:: 1.1a3
+        Verify that ``function`` is callable, raising a TypeError if not. Previously,
+        the spawned greenlet would have failed the first time it was switched to.
+
     .. versionchanged:: 1.1b1
        If *function* is not callable, immediately raise a :exc:`TypeError`
        instead of spawning a greenlet that will raise an uncaught TypeError.
@@ -118,12 +123,15 @@ def spawn_raw(function, *args, **kwargs):
         Accept keyword arguments for ``function`` as previously (incorrectly)
         documented. Note that this may incur an additional expense.
 
-    .. versionchanged:: 1.1a3
-        Verify that ``function`` is callable, raising a TypeError if not. Previously,
-        the spawned greenlet would have failed the first time it was switched to.
+    .. versionchanged:: 1.3a2
+       Populate the ``spawning_greenlet`` and ``spawn_tree_locals``
+       attributes of the returned greenlet.
+
     """
     if not callable(function):
         raise TypeError("function must be callable")
+
+    # The hub is always the parent.
     hub = get_hub()
 
     # The callback class object that we use to run this doesn't
@@ -136,6 +144,19 @@ def spawn_raw(function, *args, **kwargs):
     else:
         g = RawGreenlet(function, hub)
         hub.loop.run_callback(g.switch, *args)
+
+    # See greenlet.py's Greenlet class. We capture the cheap
+    # parts to maintain the tree structure, but we do not capture
+    # the stack because that's too expensive.
+    current = getcurrent()
+    g.spawning_greenlet = wref(current)
+    # See Greenlet for how trees are maintained.
+    try:
+        g.spawn_tree_locals = current.spawn_tree_locals
+    except AttributeError:
+        g.spawn_tree_locals = {}
+        if current.parent:
+            current.spawn_tree_locals = g.spawn_tree_locals
     return g
 
 

src/gevent/util.py

@@ -3,11 +3,14 @@
 Low-level utilities.
 """
 
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function, division
 
 import functools
 
-__all__ = ['wrap_errors']
+__all__ = [
+    'wrap_errors',
+    'format_run_info',
+]
 
 
 class wrap_errors(object):
@@ -59,7 +62,7 @@ class wrap_errors(object):
     def __getattr__(self, name):
         return getattr(self.__func, name)
 
-def dump_stacks():
+def format_run_info():
     """
     Request information about the running threads of the current process.
 
@@ -68,39 +71,82 @@ def dump_stacks():
 
     :return: A sequence of text lines detailing the stacks of running
             threads and greenlets. (One greenlet will duplicate one thread,
-            the current thread and greenlet.)
+            the current thread and greenlet.) Extra information about
+            :class:`gevent.greenlet.Greenlet` object will also be returned.
 
     .. versionadded:: 1.3a1
+    .. versionchanged:: 1.3a2
+       Renamed from ``dump_stacks`` to reflect the fact that this
+       prints additional information about greenlets, including their
+       spawning stack, parent, and any spawn tree locals.
     """
-    dump = []
 
-    # threads
-    import threading  # Late import this stuff because it may get monkey-patched
-    import traceback
-    import sys
-    import gc
+    lines = []
 
-    from greenlet import greenlet
+    _format_thread_info(lines)
+    _format_greenlet_info(lines)
+    return lines
+
+def _format_thread_info(lines):
+    import threading
+    import sys
+    import traceback
 
     threads = {th.ident: th.name for th in threading.enumerate()}
 
+    lines.append('*' * 80)
+    lines.append('* Threads')
+
+    thread = None
+    frame = None
     for thread, frame in sys._current_frames().items():
-        dump.append('Thread 0x%x (%s)\n' % (thread, threads.get(thread)))
-        dump.append(''.join(traceback.format_stack(frame)))
-        dump.append('\n')
+        lines.append("*" * 80)
+        lines.append('Thread 0x%x (%s)\n' % (thread, threads.get(thread)))
+        lines.append(''.join(traceback.format_stack(frame)))
+
+    # We may have captured our own frame, creating a reference
+    # cycle, so clear it out.
+    del thread
+    del frame
+    del lines
+    del threads
+
+def _format_greenlet_info(lines):
+    from greenlet import greenlet
+    import pprint
+    import traceback
+    import gc
 
-    # greenlets
+    def _noop():
+        return None
 
-    # if greenlet is present, let's dump each greenlet stack
     # Use the gc module to inspect all objects to find the greenlets
     # since there isn't a global registry
+    lines.append('*' * 80)
+    lines.append('* Greenlets')
+    seen_locals = set() # {id}
     for ob in gc.get_objects():
         if not isinstance(ob, greenlet):
             continue
         if not ob:
             continue  # not running anymore or not started
-        dump.append('Greenlet %s\n' % ob)
-        dump.append(''.join(traceback.format_stack(ob.gr_frame)))
-        dump.append('\n')
-
-    return dump
+        lines.append('*' * 80)
+        lines.append('Greenlet %s\n' % ob)
+        lines.append(''.join(traceback.format_stack(ob.gr_frame)))
+        spawning_stack = getattr(ob, 'spawning_stack', None)
+        if spawning_stack:
+            lines.append("Spawned at: ")
+            lines.append(''.join(traceback.format_stack(spawning_stack)))
+        parent = getattr(ob, 'spawning_greenlet', _noop)()
+        if parent is not None:
+            lines.append("Parent greenlet: %s\n" % (parent,))
+        spawn_tree_locals = getattr(ob, 'spawn_tree_locals', None)
+        if spawn_tree_locals and id(spawn_tree_locals) not in seen_locals:
+            seen_locals.add(id(spawn_tree_locals))
+            lines.append("Spawn Tree Locals:\n")
+            lines.append(pprint.pformat(spawn_tree_locals))
+
+
+    del lines
+
+dump_stacks = format_run_info

src/greentest/test__greenlet.py

@@ -96,14 +96,17 @@ class TestUnlink(greentest.TestCase):
 
     def _test_func(self, p, link):
         link(dummy_test_func)
-        assert len(p._links) == 1, p._links
+        self.assertEqual(1, p.has_links())
+
         p.unlink(dummy_test_func)
-        assert not p._links, p._links
+        self.assertEqual(0, p.has_links())
 
         link(self.setUp)
-        assert len(p._links) == 1, p._links
+        self.assertEqual(1, p.has_links())
+
         p.unlink(self.setUp)
-        assert not p._links, p._links
+        self.assertEqual(0, p.has_links())
+
         p.kill()
 
     def test_func_link(self):
@@ -170,8 +173,7 @@ class TestReturn_link(LinksTestCase):
     p = None
 
     def cleanup(self):
-        while self.p._links:
-            self.p._links.pop()
+        self.p.unlink_all()
         self.p = None
 
     def test_return(self):
@@ -374,7 +376,8 @@ class TestStuff(greentest.TestCase):
         link(results.listener2)
         link(results.listener3)
         sleep(DELAY * 10)
-        assert results.results == [5], results.results
+        self.assertEqual([5], results.results)
+
 
     def test_multiple_listeners_error_unlink_Greenlet_link(self):
         p = gevent.spawn(lambda: 5)
@@ -402,6 +405,8 @@ class A(object):
 
 hexobj = re.compile('-?0x[0123456789abcdef]+L?', re.I)
 
+class Subclass(gevent.Greenlet):
+    pass
 
 class TestStr(greentest.TestCase):
 
@@ -425,6 +430,17 @@ class TestStr(greentest.TestCase):
         str_g = str_g.replace(__name__, 'module')
         self.assertEqual(str_g, '<Greenlet at X: <bound method A.method of <module.A object at X>>>')
 
+    def test_subclass(self):
+        g = Subclass()
+        str_g = hexobj.sub('X', str(g))
+        str_g = str_g.replace(__name__, 'module')
+        self.assertEqual(str_g, '<Subclass at X: _run>')
+
+        g = Subclass(None, 'question', answer=42)
+        str_g = hexobj.sub('X', str(g))
+        str_g = str_g.replace(__name__, 'module')
+        self.assertEqual(str_g, "<Subclass at X: _run('question', answer=42)>")
+
 
 class TestJoin(AbstractGenericWaitTestCase):
 
@@ -515,14 +531,14 @@ class TestBasic(greentest.TestCase):
         assert g.exception is None
 
         gevent.sleep(0.001)
-        assert g
-        assert not g.dead
-        assert g.started
-        assert not g.ready()
-        assert not g.successful()
-        assert g.value is None
-        assert g.exception is None
-        assert not link_test
+        self.assertTrue(g)
+        self.assertFalse(g.dead, g)
+        self.assertTrue(g.started, g)
+        self.assertFalse(g.ready(), g)
+        self.assertFalse(g.successful(), g)
+        self.assertIsNone(g.value, g)
+        self.assertIsNone(g.exception, g)
+        self.assertFalse(link_test)
 
         gevent.sleep(0.02)
         assert not g
@@ -649,6 +665,29 @@ class TestBasic(greentest.TestCase):
         g.join()
         self.assertFalse(g.exc_info)
 
+    def test_tree_locals(self):
+        g = g2 = None
+        def func():
+            child = greenlet.Greenlet()
+            self.assertIs(child.spawn_tree_locals, getcurrent().spawn_tree_locals)
+            self.assertIs(child.spawning_greenlet(), getcurrent())
+        g = greenlet.Greenlet(func)
+        g2 = greenlet.Greenlet(func)
+        # Creating those greenlets did not give the main greenlet
+        # a locals dict.
+        self.assertFalse(hasattr(getcurrent(), 'spawn_tree_locals'),
+                         getcurrent())
+        self.assertIsNot(g.spawn_tree_locals, g2.spawn_tree_locals)
+        g.start()
+        g.join()
+
+        raw = gevent.spawn_raw(func)
+        self.assertIsNotNone(raw.spawn_tree_locals)
+        self.assertIsNot(raw.spawn_tree_locals, g.spawn_tree_locals)
+        self.assertIs(raw.spawning_greenlet(), getcurrent())
+        while not raw.dead:
+            gevent.sleep(0.01)
+
 
 class TestStart(greentest.TestCase):
 

src/greentest/test__issue330.py

 # A greenlet that's killed before it is ever started
 # should never be switched to
 import gevent
+import greentest
 
-switched_to = [False, False]
 
+class MyException(Exception):
+    pass
+
+class TestSwitch(greentest.TestCase):
 
-def runner(i):
-    switched_to[i] = True
+    def setUp(self):
+        self.switched_to = [False, False]
+        self.caught = None
 
+    def runner(self, i):
+        self.switched_to[i] = True
 
-def check(g, g2):
+    def check(self, g, g2):
         gevent.joinall((g, g2))
-    assert switched_to == [False, False], switched_to
+        self.assertEqual([False, False], self.switched_to)
 
         # They both have a GreenletExit as their value
-    assert isinstance(g.value, gevent.GreenletExit)
-    assert isinstance(g2.value, gevent.GreenletExit)
-    # They both have no reported exc_info
-    assert g._exc_info == (None, None, None)
-    assert g2._exc_info == (None, None, None)
-    assert g._exc_info is not type(g)._exc_info
-    assert g2._exc_info is not type(g2)._exc_info
-
-    switched_to[:] = [False, False]
-
-g = gevent.spawn(runner, 0) # create but do not switch to
-g2 = gevent.spawn(runner, 1) # create but do not switch to
-# Using gevent.kill
-gevent.kill(g)
-gevent.kill(g2)
-check(g, g2)
-
-# killing directly
-g = gevent.spawn(runner, 0)
-g2 = gevent.spawn(runner, 1)
-g.kill()
-g2.kill()
-check(g, g2)
-
-# throwing
-g = gevent.spawn(runner, 0)
-g2 = gevent.spawn(runner, 1)
-g.throw(gevent.GreenletExit)
-g2.throw(gevent.GreenletExit)
-check(g, g2)
-
-
-# Killing with gevent.kill gets the right exception
-class MyException(Exception):
-    pass
+        self.assertIsInstance(g.value, gevent.GreenletExit)
+        self.assertIsInstance(g2.value, gevent.GreenletExit)
 
-
-def catcher():
+        # They both have no reported exc_info
+        self.assertIsNone(g.exc_info)
+        self.assertIsNone(g2.exc_info)
+        self.assertIsNone(g.exception)
+        self.assertIsNone(g2.exception)
+
+
+    def test_gevent_kill(self):
+        g = gevent.spawn(self.runner, 0) # create but do not switch to
+        g2 = gevent.spawn(self.runner, 1) # create but do not switch to
+        # Using gevent.kill
+        gevent.kill(g)
+        gevent.kill(g2)
+        self.check(g, g2)
+
+    def test_greenlet_kill(self):
+        # killing directly
+        g = gevent.spawn(self.runner, 0)
+        g2 = gevent.spawn(self.runner, 1)
+        g.kill()
+        g2.kill()
+        self.check(g, g2)
+
+    def test_throw(self):
+        # throwing
+        g = gevent.spawn(self.runner, 0)
+        g2 = gevent.spawn(self.runner, 1)
+        g.throw(gevent.GreenletExit)
+        g2.throw(gevent.GreenletExit)
+        self.check(g, g2)
+
+
+    def catcher(self):
         try:
             while True:
                 gevent.sleep(0)
-    except Exception as e:
-        switched_to[0] = e
+        except MyException as e:
+            self.caught = e
+
+    def test_kill_exception(self):
+        # Killing with gevent.kill gets the right exception
+
+        g = gevent.spawn(self.catcher)
+        g.start()
+        gevent.sleep()
+        gevent.kill(g, MyException())
+        gevent.sleep()
+
+        self.assertIsInstance(self.caught, MyException)
+        self.assertIsNone(g.exception, MyException)
 
-g = gevent.spawn(catcher)
-g.start()
-gevent.sleep()
-gevent.kill(g, MyException())
-gevent.sleep()
 
-assert isinstance(switched_to[0], MyException), switched_to
+if __name__ == '__main__':
+    greentest.main()

src/greentest/test__local.py

@@ -273,7 +273,7 @@ class GeventLocalTestCase(greentest.TestCase):
             self.assertEqual(count, len(deleted_sentinels))
 
             # The links were removed as well.
-            self.assertEqual(list(running_greenlet._links), [])
+            self.assertFalse(running_greenlet.has_links())
 
 
         running_greenlet = gevent.spawn(demonstrate_my_local)

src/greentest/test__util.py

+# -*- coding: utf-8 -*-
+# Copyright 2018 gevent contributes
+# See LICENSE for details.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import greentest
+
+import gevent
+from gevent import util
+
+@greentest.skipOnPyPy("5.10.x is *very* slow formatting stacks")
+class TestFormat(greentest.TestCase):
+
+    def test_basic(self):
+        lines = util.format_run_info()
+
+        value = '\n'.join(lines)
+        self.assertIn('Threads', value)
+        self.assertIn('Greenlets', value)
+
+        # because it's a raw greenlet, we have no data for it.
+        self.assertNotIn("Spawned at", value)
+        self.assertNotIn("Parent greenlet", value)
+        self.assertNotIn("Spawn Tree Locals", value)
+
+    def test_with_Greenlet(self):
+        def root():
+            gevent.getcurrent().spawn_tree_locals['a value'] = 42
+            g = gevent.spawn(util.format_run_info)
+            g.join()
+            return g.value
+
+        g = gevent.spawn(root)
+        g.join()
+        value = '\n'.join(g.value)
+
+        self.assertIn("Spawned at", value)
+        self.assertIn("Parent greenlet", value)
+        self.assertIn("Spawn Tree Locals", value)
+
+
+if __name__ == '__main__':
+    greentest.main()

src/greentest/tests_that_dont_monkeypatch.txt

@@ -13,3 +13,4 @@ test__socket_errors.py
 test__socket_send_memoryview.py
 test__socket_timeout.py
 test__examples.py
+test__issue330.py

src/greentest/tests_that_dont_use_resolver.txt

@@ -126,3 +126,4 @@ test_asyncore.py
 
 test___config.py
 test__destroy_default_loop.py
+test__util.py