Add spawn_tree_locals, spawning_greenlet and spawning_stack to Greenlet

Based on #755.

A comment in the code goes into detail about the timing. Here it is
again:

 Timings taken Feb 21 2018 prior to integration of #755
 python -m perf timeit -s 'import gevent' 'gevent.Greenlet()'
 3.6.4       : Mean +- std dev: 1.08 us +- 0.05 us
 2.7.14      : Mean +- std dev: 1.44 us +- 0.06 us
 PyPy2 5.10.0: Mean +- std dev: 2.14 ns +- 0.08 ns

 After the integration of spawning_stack, spawning_greenlet,
 and spawn_tree_locals on that same date:
 3.6.4       : Mean +- std dev: 8.92 us +- 0.36 us ->  8.2x
 2.7.14      : Mean +- std dev: 14.8 us +- 0.5 us  -> 10.2x
 PyPy2 5.10.0: Mean +- std dev: 3.24 us +- 0.17 us ->  1.5x

Selected bench_spawn output on 3.6.4 before:

//gevent36/bin/python src/greentest/bench_spawn.py eventlet --ignore-import-errors
using eventlet from //gevent36/lib/python3.6/site-packages/eventlet/__init__.py
spawning: 11.93 microseconds per greenlet
sleep(0): 23.49 microseconds per greenlet

//gevent36/bin/python src/greentest/bench_spawn.py gevent --ignore-import-errors
using gevent from //src/gevent/__init__.py
spawning: 3.39 microseconds per greenlet
sleep(0): 17.59 microseconds per greenlet

//gevent36/bin/python src/greentest/bench_spawn.py geventpool --ignore-import-errors
using gevent from //src/gevent/__init__.py
spawning: 8.71 microseconds per greenlet

//gevent36/bin/python src/greentest/bench_spawn.py geventraw --ignore-import-errors
using gevent from //src/gevent/__init__.py
spawning: 2.09 microseconds per greenlet

//gevent36/bin/python src/greentest/bench_spawn.py none --ignore-import-errors
    noop: 0.33 microseconds per greenlet

And after:

//gevent36/bin/python bench_spawn.py gevent --ignore-import-errors
using gevent from //src/gevent/__init__.py
spawning: 12.99 microseconds per greenlet -> 3.8x

//gevent36/bin/python bench_spawn.py geventpool --ignore-import-errors
using gevent from //src/gevent/__init__.py
spawning: 19.49 microseconds per greenlet -> 2.2x

//gevent36/bin/python bench_spawn.py geventraw --ignore-import-errors
using gevent from //src/gevent/__init__.py
spawning: 4.57 microseconds per greenlet -> 2.2x

We're approximately the speed of eventlet now.

Refs #755

CHANGES.rst

@@ -85,6 +85,10 @@
 - Signal handling under PyPy with libuv is more reliable. See
   :issue:`1112`.
 
+- Greenlet objects now keep track of their spawning parent greenlet
+  and the code location that spawned them, in addition to maintaining
+  a "spawn tree local" mapping. Based on a proposal from PayPal and
+  comments by Mahmoud Hashemi and Kurt Rose. See :issue:`755`.
 
 1.3a1 (2018-01-27)
 ==================

doc/gevent.rst

@@ -47,6 +47,13 @@ generated.
              its value.
 
 .. autoattribute:: Greenlet.exception
+.. autoattribute:: Greenlet.spawn_tree_locals
+   :annotation: = {}
+.. autoattribute:: Greenlet.spawning_greenlet
+   :annotation: = weakref.ref()
+.. autoattribute:: Greenlet.spawning_stack
+   :annotation: = <Frame>
+.. autoattribute:: Greenlet.spawning_stack_limit
 
 .. automethod:: Greenlet.ready
 .. automethod:: Greenlet.successful
@@ -114,6 +121,8 @@ Spawn helpers
 Useful general functions
 ========================
 
+.. seealso:: :mod:`gevent.util`
+
 .. function:: getcurrent()
 
    Return the currently executing greenlet (the one that called this

src/gevent/greenlet.py

 # Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details.
 from __future__ import absolute_import
+
+from collections import deque
 import sys
+from weakref import ref as wref
+
+
 from greenlet import greenlet
+from greenlet import getcurrent
+
 from gevent._compat import PY3
 from gevent._compat import PYPY
 from gevent._compat import reraise
@@ -12,11 +19,10 @@ from gevent.hub import GreenletExit
 from gevent.hub import InvalidSwitchError
 from gevent.hub import Waiter
 from gevent.hub import get_hub
-from gevent.hub import getcurrent
 from gevent.hub import iwait
 from gevent.hub import wait
 from gevent.timeout import Timeout
-from collections import deque
+
 
 
 __all__ = [
@@ -87,11 +93,56 @@ class FailureSpawnedLink(SpawnedLink):
         if not source.successful():
             return SpawnedLink.__call__(self, source)
 
+
+class _Frame(object):
+
+    __slots__ = ('f_code', 'f_lineno', 'f_back')
+
+    def __init__(self, f_code, f_lineno):
+        self.f_code = f_code
+        self.f_lineno = f_lineno
+        self.f_back = None
+
+    f_globals = property(lambda _self: None)
+
 class Greenlet(greenlet):
-    """A light-weight cooperatively-scheduled execution unit.
+    """
+    A light-weight cooperatively-scheduled execution unit.
     """
     # pylint:disable=too-many-public-methods,too-many-instance-attributes
 
+    #: A dictionary that is shared between all the greenlets
+    #: in a "spawn tree", that is, a spawning greenlet and all
+    #: its descendent greenlets. All children of the main (root)
+    #: greenlet start their own spawn trees. Assign a new dictionary
+    #: to this attribute on an instance of this class to create a new
+    #: spawn tree (as far as locals are concerned).
+    #:
+    #: .. versionadded:: 1.3a2
+    spawn_tree_locals = None
+
+    #: A weak-reference to the greenlet that was current when this object
+    #: was created. Note that the :attr:`parent` attribute is always the
+    #: hub.
+    #:
+    #: .. versionadded:: 1.3a2
+    spawning_greenlet = None
+
+    #: A lightweight frame-like object capturing the stack when
+    #: this greenlet was created as well as the stack when the spawning
+    #: greenlet was created (if applicable). This can be passed to
+    #: :func:`traceback.print_stack`.
+    #:
+    #: .. versionadded:: 1.3a2
+    spawning_stack = None
+
+    #: A class attribute specifying how many levels of the spawning
+    #: stack will be kept. Specify a smaller number for higher performance,
+    #: specify a larger value for improved debugging.
+    #:
+    #: .. versionadded:: 1.3a2
+    spawning_stack_limit = 10
+
     value = None
     _exc_info = ()
     _notifier = None
@@ -128,6 +179,20 @@ class Greenlet(greenlet):
         # Python 3.4: 2.32usec with keywords vs 1.74usec with positional
         # Python 3.3: 2.55usec with keywords vs 1.92usec with positional
         # Python 2.7: 1.73usec with keywords vs 1.40usec with positional
+
+        # Timings taken Feb 21 2018 prior to integration of #755
+        # python -m perf timeit -s 'import gevent' 'gevent.Greenlet()'
+        # 3.6.4       : Mean +- std dev: 1.08 us +- 0.05 us
+        # 2.7.14      : Mean +- std dev: 1.44 us +- 0.06 us
+        # PyPy2 5.10.0: Mean +- std dev: 2.14 ns +- 0.08 ns
+
+        # After the integration of spawning_stack, spawning_greenlet,
+        # and spawn_tree_locals on that same date:
+        # 3.6.4       : Mean +- std dev: 8.92 us +- 0.36 us ->  8.2x
+        # 2.7.14      : Mean +- std dev: 14.8 us +- 0.5 us  -> 10.2x
+        # PyPy2 5.10.0: Mean +- std dev: 3.24 us +- 0.17 us ->  1.5x
+
+
         greenlet.__init__(self, None, get_hub())
 
         if run is not None:
@@ -144,6 +209,35 @@ class Greenlet(greenlet):
         if kwargs:
             self._kwargs = kwargs
 
+        spawner = getcurrent()
+        self.spawning_greenlet = wref(spawner)
+        try:
+            self.spawn_tree_locals = spawner.spawn_tree_locals
+        except AttributeError:
+            self.spawn_tree_locals = {}
+            if spawner.parent:
+                # The main greenlet has no parent.
+                # Its children get separate locals.
+                spawner.spawn_tree_locals = self.spawn_tree_locals
+
+
+        frame = sys._getframe()
+        previous = None
+        limit = self.spawning_stack_limit
+        while limit and frame:
+            limit -= 1
+            next_frame = _Frame(frame.f_code, frame.f_lineno)
+            if previous:
+                previous.f_back = next_frame
+            else:
+                self.spawning_stack = next_frame
+            previous = next_frame
+
+            frame = frame.f_back
+
+        previous.f_back = getattr(spawner, 'spawning_stack', None)
+
+
     @property
     def kwargs(self):
         return self._kwargs or {}

src/gevent/hub.py

@@ -9,6 +9,7 @@ from functools import partial as _functools_partial
 import os
 import sys
 import traceback
+from weakref import ref as wref
 
 from greenlet import greenlet as RawGreenlet, getcurrent, GreenletExit
 
@@ -110,6 +111,10 @@ def spawn_raw(function, *args, **kwargs):
     occasionally be useful as an optimization if there are many
     greenlets involved.
 
+    .. versionchanged:: 1.1a3
+        Verify that ``function`` is callable, raising a TypeError if not. Previously,
+        the spawned greenlet would have failed the first time it was switched to.
+
     .. versionchanged:: 1.1b1
        If *function* is not callable, immediately raise a :exc:`TypeError`
        instead of spawning a greenlet that will raise an uncaught TypeError.
@@ -118,12 +123,15 @@ def spawn_raw(function, *args, **kwargs):
         Accept keyword arguments for ``function`` as previously (incorrectly)
         documented. Note that this may incur an additional expense.
 
-    .. versionchanged:: 1.1a3
-        Verify that ``function`` is callable, raising a TypeError if not. Previously,
-        the spawned greenlet would have failed the first time it was switched to.
+    .. versionchanged:: 1.3a2
+       Populate the ``spawning_greenlet`` and ``spawn_tree_locals``
+       attributes of the returned greenlet.
+
     """
     if not callable(function):
         raise TypeError("function must be callable")
+
+    # The hub is always the parent.
     hub = get_hub()
 
     # The callback class object that we use to run this doesn't
@@ -136,6 +144,19 @@ def spawn_raw(function, *args, **kwargs):
     else:
         g = RawGreenlet(function, hub)
         hub.loop.run_callback(g.switch, *args)
+
+    # See greenlet.py's Greenlet class. We capture the cheap
+    # parts to maintain the tree structure, but we do not capture
+    # the stack because that's too expensive.
+    current = getcurrent()
+    g.spawning_greenlet = wref(current)
+    # See Greenlet for how trees are maintained.
+    try:
+        g.spawn_tree_locals = current.spawn_tree_locals
+    except AttributeError:
+        g.spawn_tree_locals = {}
+        if current.parent:
+            current.spawn_tree_locals = g.spawn_tree_locals
     return g
 
 

src/gevent/util.py

@@ -3,11 +3,14 @@
 Low-level utilities.
 """
 
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function, division
 
 import functools
 
-__all__ = ['wrap_errors']
+__all__ = [
+    'wrap_errors',
+    'format_run_info',
+]
 
 
 class wrap_errors(object):
@@ -59,7 +62,7 @@ class wrap_errors(object):
     def __getattr__(self, name):
         return getattr(self.__func, name)
 
-def dump_stacks():
+def format_run_info():
     """
     Request information about the running threads of the current process.
 
@@ -68,39 +71,82 @@ def dump_stacks():
 
     :return: A sequence of text lines detailing the stacks of running
             threads and greenlets. (One greenlet will duplicate one thread,
-            the current thread and greenlet.)
+            the current thread and greenlet.) Extra information about
+            :class:`gevent.greenlet.Greenlet` object will also be returned.
 
     .. versionadded:: 1.3a1
+    .. versionchanged:: 1.3a2
+       Renamed from ``dump_stacks`` to reflect the fact that this
+       prints additional information about greenlets, including their
+       spawning stack, parent, and any spawn tree locals.
     """
-    dump = []
 
-    # threads
-    import threading  # Late import this stuff because it may get monkey-patched
-    import traceback
-    import sys
-    import gc
+    lines = []
 
-    from greenlet import greenlet
+    _format_thread_info(lines)
+    _format_greenlet_info(lines)
+    return lines
+
+def _format_thread_info(lines):
+    import threading
+    import sys
+    import traceback
 
     threads = {th.ident: th.name for th in threading.enumerate()}
 
+    lines.append('*' * 80)
+    lines.append('* Threads')
+
+    thread = None
+    frame = None
     for thread, frame in sys._current_frames().items():
-        dump.append('Thread 0x%x (%s)\n' % (thread, threads.get(thread)))
-        dump.append(''.join(traceback.format_stack(frame)))
-        dump.append('\n')
+        lines.append("*" * 80)
+        lines.append('Thread 0x%x (%s)\n' % (thread, threads.get(thread)))
+        lines.append(''.join(traceback.format_stack(frame)))
+
+    # We may have captured our own frame, creating a reference
+    # cycle, so clear it out.
+    del thread
+    del frame
+    del lines
+    del threads
+
+def _format_greenlet_info(lines):
+    from greenlet import greenlet
+    import pprint
+    import traceback
+    import gc
 
-    # greenlets
+    def _noop():
+        return None
 
-    # if greenlet is present, let's dump each greenlet stack
     # Use the gc module to inspect all objects to find the greenlets
     # since there isn't a global registry
+    lines.append('*' * 80)
+    lines.append('* Greenlets')
+    seen_locals = set() # {id}
     for ob in gc.get_objects():
         if not isinstance(ob, greenlet):
             continue
         if not ob:
             continue  # not running anymore or not started
-        dump.append('Greenlet %s\n' % ob)
-        dump.append(''.join(traceback.format_stack(ob.gr_frame)))
-        dump.append('\n')
-
-    return dump
+        lines.append('*' * 80)
+        lines.append('Greenlet %s\n' % ob)
+        lines.append(''.join(traceback.format_stack(ob.gr_frame)))
+        spawning_stack = getattr(ob, 'spawning_stack', None)
+        if spawning_stack:
+            lines.append("Spawned at: ")
+            lines.append(''.join(traceback.format_stack(spawning_stack)))
+        parent = getattr(ob, 'spawning_greenlet', _noop)()
+        if parent is not None:
+            lines.append("Parent greenlet: %s\n" % (parent,))
+        spawn_tree_locals = getattr(ob, 'spawn_tree_locals', None)
+        if spawn_tree_locals and id(spawn_tree_locals) not in seen_locals:
+            seen_locals.add(id(spawn_tree_locals))
+            lines.append("Spawn Tree Locals:\n")
+            lines.append(pprint.pformat(spawn_tree_locals))
+
+
+    del lines
+
+dump_stacks = format_run_info

src/greentest/test__greenlet.py

@@ -649,6 +649,29 @@ class TestBasic(greentest.TestCase):
         g.join()
         self.assertFalse(g.exc_info)
 
+    def test_tree_locals(self):
+        g = g2 = None
+        def func():
+            child = greenlet.Greenlet()
+            self.assertIs(child.spawn_tree_locals, getcurrent().spawn_tree_locals)
+            self.assertIs(child.spawning_greenlet(), getcurrent())
+        g = greenlet.Greenlet(func)
+        g2 = greenlet.Greenlet(func)
+        # Creating those greenlets did not give the main greenlet
+        # a locals dict.
+        self.assertFalse(hasattr(getcurrent(), 'spawn_tree_locals'),
+                         getcurrent())
+        self.assertIsNot(g.spawn_tree_locals, g2.spawn_tree_locals)
+        g.start()
+        g.join()
+
+        raw = gevent.spawn_raw(func)
+        self.assertIsNotNone(raw.spawn_tree_locals)
+        self.assertIsNot(raw.spawn_tree_locals, g.spawn_tree_locals)
+        self.assertIs(raw.spawning_greenlet(), getcurrent())
+        while not raw.dead:
+            gevent.sleep(0.01)
+
 
 class TestStart(greentest.TestCase):
 

src/greentest/test__util.py

+# -*- coding: utf-8 -*-
+# Copyright 2018 gevent contributes
+# See LICENSE for details.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import greentest
+
+import gevent
+from gevent import util
+
+
+class TestFormat(greentest.TestCase):
+
+    def test_basic(self):
+        lines = util.format_run_info()
+
+        value = '\n'.join(lines)
+        self.assertIn('Threads', value)
+        self.assertIn('Greenlets', value)
+
+        # because it's a raw greenlet, we have no data for it.
+        self.assertNotIn("Spawned at", value)
+        self.assertNotIn("Parent greenlet", value)
+        self.assertNotIn("Spawn Tree Locals", value)
+
+    def test_with_Greenlet(self):
+        def root():
+            gevent.getcurrent().spawn_tree_locals['a value'] = 42
+            g = gevent.spawn(util.format_run_info)
+            g.join()
+            return g.value
+
+        g = gevent.spawn(root)
+        g.join()
+        value = '\n'.join(g.value)
+
+        self.assertIn("Spawned at", value)
+        self.assertIn("Parent greenlet", value)
+        self.assertIn("Spawn Tree Locals", value)
+
+
+if __name__ == '__main__':
+    greentest.main()

src/greentest/tests_that_dont_use_resolver.txt

@@ -126,3 +126,4 @@ test_asyncore.py
 
 test___config.py
 test__destroy_default_loop.py
+test__util.py