Add a GreenletTree for more organized, clearer output of greenlets

It organizes things by the spawning greenlet, if possible.

Example output::

```
<greenlet.greenlet object at 0x10753a9b0>
 :    Running:
 :      File "/tmp/t.py", line 38, in <module>
 :        print("\n".join(format_run_info()))
 :      File "//src/gevent/util.py", line 99, in format_run_info
 :        _format_greenlet_info(lines)
 :      File "//src/gevent/util.py", line 132, in _format_greenlet_info
 :        lines.extend(tree.format_lines(details=True))
 :      File "//src/gevent/util.py", line 265, in format_lines
 :        for l in self._render(tree)]
 :      File "/-main/src/gevent/util.py", line 310, in _render
 :        self.__render_tb(tree, 'Running:', self.greenlet.gr_frame)
 :      File "//src/gevent/util.py", line 281, in __render_tb
 :        tb = ''.join(traceback.format_stack(frame))
 :    Greenlet Locals:
 :      Local <type 'gevent._local.local'> at 0x10759cec0
 :        {'foo': 42}
 +--- <Greenlet "Greenlet-0" at 0x107405cb0: _run>; finished with value <Greenlet "Greenlet-4" at 0x10
 :          Parent: <Hub at 0x10753a550 select default pending=0 ref=0>
 :          Spawned at:
 :            File "/tmp/t.py", line 1, in <module>
 :              from gevent.util import GreenletTree
 |    +--- <Greenlet "Greenlet-4" at 0x10780b260: _run>; finished
 :                Parent: <Hub at 0x10753a550 select default pending=0 ref=0>
 :                Spawned at:
 :                  File "/tmp/t.py", line 1, in <module>
 :                    from gevent.util import GreenletTree
 :                  File "/tmp/t.py", line 12, in t2
 :                    def t2():
 +--- <Hub at 0x10753a550 select default pending=0 ref=0>
 :          Parent: <greenlet.greenlet object at 0x10753a9b0>
 :          Running:
 :            File "/Users/jmadden/Projects/GithubSources/gevent-main/src/gevent/hub.py", line 673, in run
 :              loop.run()
 +--- <Greenlet "Greenlet-1" at 0x10780b368: _run>; finished with value <Greenlet "Greenlet-5" at 0x10
 :          Parent: <Hub at 0x10753a550 select default pending=0 ref=0>
 :          Spawned at:
 :            File "/tmp/t.py", line 1, in <module>
 :              from gevent.util import GreenletTree
 |    +--- <Greenlet "Greenlet-5" at 0x10780b578: _run>; finished
 :                Parent: <Hub at 0x10753a550 select default pending=0 ref=0>
 :                Spawned at:
 :                  File "/tmp/t.py", line 1, in <module>
 :                    from gevent.util import GreenletTree
 :                  File "/tmp/t.py", line 12, in t2
 :                    def t2():
 +--- <Greenlet "Greenlet-2" at 0x10780b470: _run>; finished with value <Greenlet "Greenlet-6" at 0x10
 :          Parent: <Hub at 0x10753a550 select default pending=0 ref=0>
 :          Spawned at:
 :            File "/tmp/t.py", line 1, in <module>
 :              from gevent.util import GreenletTree
 :          Spawn Tree Locals
 :          {'stl': 'STL'}
 |    +--- <Greenlet "Greenlet-6" at 0x10780b680: _run>; finished with value <Greenlet "Greenlet-7" at 0x10
 :                Parent: <Hub at 0x10753a550 select default pending=0 ref=0>
 :                Spawned at:
 :                  File "/tmp/t.py", line 1, in <module>
 :                    from gevent.util import GreenletTree
 :                  File "/tmp/t.py", line 21, in t3
 :                    def t3():
 |         +--- <Greenlet "Greenlet-7" at 0x10780b788: _run>; finished
 :                      Parent: <Hub at 0x10753a550 select default pending=0 ref=0>
 :                      Spawned at:
 :                        File "/tmp/t.py", line 1, in <module>
 :                          from gevent.util import GreenletTree
 :                        File "/tmp/t.py", line 21, in t3
 :                          def t3():
 :                        File "/tmp/t.py", line 12, in t2
 :                          def t2():
 +--- <Greenlet "Greenlet-3" at 0x10780b890: _run>; finished with value [<gevent.util.GreenletTree obj
            Parent: <Hub at 0x10753a550 select default pending=0 ref=0>
            Spawned at:
              File "/tmp/t.py", line 1, in <module>
                from gevent.util import GreenletTree
```

CHANGES.rst

@@ -123,6 +123,8 @@
 - `gevent.Greenlet` objects now have a `gevent.Greenlet.name`
   attribute that is included in the default repr.
 
+- Add `gevent.util.GreenletTree` to visualize the greenlet tree. This
+  is used by `gevent.util.format_run_info`.
 
 1.3a1 (2018-01-27)
 ==================

src/gevent/util.py

@@ -5,13 +5,27 @@ Low-level utilities.
 
 from __future__ import absolute_import, print_function, division
 
+import gc
 import functools
+import pprint
+import traceback
+
+from greenlet import getcurrent
+from greenlet import greenlet as RawGreenlet
+
+from gevent.local import all_local_dicts_for_greenlet
 
 __all__ = [
     'wrap_errors',
     'format_run_info',
+    'GreenletTree',
 ]
 
+def _noop():
+    return None
+
+def _ready():
+    return False
 
 class wrap_errors(object):
     """
@@ -90,7 +104,6 @@ def format_run_info():
 def _format_thread_info(lines):
     import threading
     import sys
-    import traceback
 
     threads = {th.ident: th.name for th in threading.enumerate()}
 
@@ -112,49 +125,297 @@ def _format_thread_info(lines):
     del threads
 
 def _format_greenlet_info(lines):
-    # pylint:disable=too-many-locals
-    from greenlet import greenlet
-    import pprint
-    import traceback
-    import gc
-    from gevent.local import all_local_dicts_for_greenlet
-
-    def _noop():
-        return None
-
     # Use the gc module to inspect all objects to find the greenlets
     # since there isn't a global registry
     lines.append('*' * 80)
     lines.append('* Greenlets')
-    seen_locals = set() # {id}
-    for ob in gc.get_objects():
-        if not isinstance(ob, greenlet):
-            continue
-        if not ob:
-            continue  # not running anymore or not started
-        lines.append('*' * 80)
-        lines.append('Greenlet %s\n' % ob)
-        lines.append(''.join(traceback.format_stack(ob.gr_frame)))
-        spawning_stack = getattr(ob, 'spawning_stack', None)
-        if spawning_stack:
-            lines.append("Spawned at: ")
-            lines.append(''.join(traceback.format_stack(spawning_stack)))
-        parent = getattr(ob, 'spawning_greenlet', _noop)()
-        if parent is not None:
-            lines.append("Parent greenlet: %s\n" % (parent,))
-        spawn_tree_locals = getattr(ob, 'spawn_tree_locals', None)
-        if spawn_tree_locals and id(spawn_tree_locals) not in seen_locals:
-            seen_locals.add(id(spawn_tree_locals))
-            lines.append("Spawn Tree Locals:\n")
-            lines.append(pprint.pformat(spawn_tree_locals))
-        gr_locals = all_local_dicts_for_greenlet(ob)
+    lines.append('*' * 80)
+    for tree in GreenletTree.forest():
+        lines.extend(tree.format_lines(details=True))
+
+    del lines
+
+dump_stacks = format_run_info
+
+def _line(f):
+    @functools.wraps(f)
+    def w(self, *args, **kwargs):
+        r = f(self, *args, **kwargs)
+        self.lines.append(r)
+
+    return w
+
+class _TreeFormatter(object):
+    UP_AND_RIGHT = '+'
+    HORIZONTAL = '-'
+    VERTICAL = '|'
+    VERTICAL_AND_RIGHT = '+'
+    DATA = ':'
+
+    label_space = 1
+    horiz_width = 3
+    indent = 1
+
+    def __init__(self, details, depth=0):
+        self.lines = []
+        self.depth = depth
+        self.details = details
+        if not details:
+            self.child_data = lambda *args, **kwargs: None
+
+    def deeper(self):
+        return type(self)(self.details, self.depth + 1)
+
+    @_line
+    def node_label(self, text):
+        return text
+
+    @_line
+    def child_head(self, label, right=VERTICAL_AND_RIGHT):
+        return (
+            ' ' * self.indent
+            + right
+            + self.HORIZONTAL * self.horiz_width
+            + ' ' * self.label_space
+            + label
+        )
+
+    def last_child_head(self, label):
+        return self.child_head(label, self.UP_AND_RIGHT)
+
+    @_line
+    def child_tail(self, line, vertical=VERTICAL):
+        return (
+            ' ' * self.indent
+            + vertical
+            + ' ' * self.horiz_width
+            + line
+        )
+
+    def last_child_tail(self, line):
+        return self.child_tail(line, vertical=' ' * len(self.VERTICAL))
+
+    @_line
+    def child_data(self, data, data_marker=DATA): # pylint:disable=method-hidden
+        return ((
+            ' ' * self.indent
+            + (data_marker if not self.depth else ' ')
+            + ' ' * self.horiz_width
+            + ' ' * self.label_space
+            + data
+        ),)
+
+    def last_child_data(self, data):
+        return self.child_data(data, ' ')
+
+    def child_multidata(self, data):
+        # Remove embedded newlines
+        for l in data.splitlines():
+            self.child_data(l)
+
+
+class GreenletTree(object):
+    """
+    Represents a tree of greenlets.
+
+    In gevent, the *parent* of a greenlet is usually the hub, so this
+    tree is primarily arganized along the *spawning_greenlet* dimension.
+
+    This object has a small str form showing this hierarchy. The `format`
+    method can output more details. The exact output is unspecified but is
+    intended to be human readable.
+
+    Use the `forest` method to get the root greenlet trees for
+    all threads, and the `current_tree` to get the root greenlet tree for
+    the current thread.
+    """
+
+    #: The greenlet this tree represents.
+    greenlet = None
+
+
+    def __init__(self, greenlet):
+        self.greenlet = greenlet
+        self.child_trees = []
+
+    def add_child(self, tree):
+        if tree is self:
+            return
+        self.child_trees.append(tree)
+
+    @property
+    def root(self):
+        return self.greenlet.parent is None
+
+    def __getattr__(self, name):
+        return getattr(self.greenlet, name)
+
+    def format_lines(self, details=True):
+        """
+        Return a sequence of lines for the greenlet tree.
+
+        :keyword bool details: If true (the default),
+            then include more informative details in the output.
+        """
+        if not isinstance(details, dict):
+            if not details:
+                details = {}
+            else:
+                details = {'stacks': True, 'locals': True}
+        tree = _TreeFormatter(details, depth=0)
+        lines = [l[0] if isinstance(l, tuple) else l
+                 for l in self._render(tree)]
+        return lines
+
+    def format(self, details=True):
+        """
+        Like `format_lines` but returns a string.
+        """
+        lines = self.format_lines(details)
+        return '\n'.join(lines)
+
+    def __str__(self):
+        return self.format(False)
+
+    @staticmethod
+    def __render_tb(tree, label, frame):
+        tree.child_data(label)
+        tb = ''.join(traceback.format_stack(frame))
+        tree.child_multidata(tb)
+
+    def __render_locals(self, tree):
+        gr_locals = all_local_dicts_for_greenlet(self.greenlet)
         if gr_locals:
-            lines.append("Greenlet Locals:\n")
+            tree.child_data("Greenlet Locals:")
             for (kind, idl), vals in gr_locals:
-                lines.append("\tLocal %s at %s\n" % (kind, hex(idl)))
-                lines.append("\t" + pprint.pformat(vals))
+                tree.child_data("  Local %s at %s" % (kind, hex(idl)))
+                tree.child_multidata("    " + pprint.pformat(vals))
+
+    def _render(self, tree):
+        label = repr(self.greenlet)
+        if not self.greenlet: # Not running or dead
+            # raw greenlets do not have ready
+            if getattr(self.greenlet, 'ready', _ready)():
+                label += '; finished'
+                if self.greenlet.value is not None:
+                    label += ' with value ' + repr(self.greenlet.value)[:30]
+                elif getattr(self.greenlet, 'exception', None) is not None:
+                    label += ' with exception ' + repr(self.greenlet.exception)
+            else:
+                label += '; not running'
+        tree.node_label(label)
+
+        if self.greenlet.parent is not None:
+            tree.child_data('Parent: ' + repr(self.greenlet.parent))
+
+        if self.greenlet and tree.details and tree.details['stacks']:
+            self.__render_tb(tree, 'Running:', self.greenlet.gr_frame)
+
+
+        spawning_stack = getattr(self.greenlet, 'spawning_stack', None)
+        if spawning_stack and tree.details and tree.details['stacks']:
+            self.__render_tb(tree, 'Spawned at:', spawning_stack)
+
+        spawning_parent = getattr(self.greenlet, 'spawning_greenlet', _noop)()
+        tree_locals = getattr(self.greenlet, 'spawn_tree_locals', None)
+        if tree_locals and tree_locals is not getattr(spawning_parent, 'spawn_tree_locals', None):
+            tree.child_data('Spawn Tree Locals')
+            tree.child_multidata(pprint.pformat(tree_locals))
+
+        self.__render_locals(tree)
+        self.__render_children(tree)
+        return tree.lines
+
+    def __render_children(self, tree):
+        children = sorted(self.child_trees,
+                          key=lambda c: (
+                              # raw greenlets first
+                              getattr(c, 'minimal_ident', -1),
+                              # running greenlets first
+                              getattr(c, 'ready', _ready)(),
+                              id(c.parent)))
+        for n, child in enumerate(children):
+            child_tree = child._render(tree.deeper())
+
+            head = tree.child_head
+            tail = tree.child_tail
+            data = tree.child_data
+
+            if n == len(children) - 1:
+                # last child does not get the line drawn
+                head = tree.last_child_head
+                tail = tree.last_child_tail
+                data = tree.last_child_data
+
+            head(child_tree.pop(0))
+            for child_data in child_tree:
+                if isinstance(child_data, tuple):
+                    data(child_data[0])
+                else:
+                    tail(child_data)
+
+        return tree.lines
+
+
+    @staticmethod
+    def _root_greenlet(greenlet):
+        while greenlet.parent is not None:
+            greenlet = greenlet.parent
+        return greenlet
+
+    @classmethod
+    def _forest(cls):
+        main_greenlet = cls._root_greenlet(getcurrent())
+
+        trees = {}
+        roots = {}
+        current_tree = roots[main_greenlet] = trees[main_greenlet] = cls(main_greenlet)
+
+
+
+        for ob in gc.get_objects():
+            if not isinstance(ob, RawGreenlet):
+                continue
+
+            spawn_parent = getattr(ob, 'spawning_greenlet', _noop)()
+
+            if spawn_parent is None:
+                root = cls._root_greenlet(ob)
+                try:
+                    tree = roots[root]
+                except KeyError: # pragma: no cover
+                    tree = GreenletTree(root)
+                    roots[root] = trees[root] = tree
+            else:
+                try:
+                    tree = trees[spawn_parent]
+                except KeyError: # pragma: no cover
+                    tree = trees[spawn_parent] = cls(spawn_parent)
 
+            try:
+                child_tree = trees[ob]
+            except KeyError:
+                trees[ob] = child_tree = cls(ob)
+            tree.add_child(child_tree)
 
-    del lines
+        return roots, current_tree
 
-dump_stacks = format_run_info
+    @classmethod
+    def forest(cls):
+        """
+        forest() -> sequence
+
+        Return a sequence of `GreenletTree`, one for each running
+        native thread.
+        """
+
+        return list(cls._forest()[0].values())
+
+    @classmethod
+    def current_tree(cls):
+        """
+        current_tree() -> GreenletTree
+
+        Returns the `GreenletTree` for the current thread.
+        """
+        return cls._forest()[1]

src/greentest/test__util.py

@@ -6,6 +6,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import gc
+
 import greentest
 
 import gevent
@@ -36,6 +38,7 @@ class TestFormat(greentest.TestCase):
         rl.foo = 1
         def root():
             l = MyLocal(42)
+            assert l
             gevent.getcurrent().spawn_tree_locals['a value'] = 42
             g = gevent.spawn(util.format_run_info)
             g.join()
@@ -47,12 +50,100 @@ class TestFormat(greentest.TestCase):
         value = '\n'.join(g.value)
 
         self.assertIn("Spawned at", value)
-        self.assertIn("Parent greenlet", value)
+        self.assertIn("Parent:", value)
         self.assertIn("Spawn Tree Locals", value)
         self.assertIn("Greenlet Locals:", value)
         self.assertIn('MyLocal', value)
         self.assertIn("Printer", value) # The name is printed
 
+@greentest.skipOnPyPy("See TestFormat")
+class TestTree(greentest.TestCase):
+
+    @greentest.ignores_leakcheck
+    def test_tree(self):
+        # pylint:disable=too-many-locals
+        # Python 2.7 on Travis seems to show unexpected greenlet objects
+        # so perhaps we need a GC?
+        gc.collect()
+        gc.collect()
+
+        import re
+        glets = []
+        l = MyLocal(42)
+        assert l
+
+        def s(f):
+            g = gevent.spawn(f)
+            # Access this in spawning order for consistent sorting
+            # at print time in the test case.
+            getattr(g, 'minimal_ident')
+            return g
+
+        def t1():
+            raise greentest.ExpectedException()
+
+        def t2():
+            l = MyLocal(16)
+            assert l
+            return s(t1)
+
+        s1 = s(t2)
+        s1.join()
+
+        glets.append(s(t2))
+
+        def t3():
+            return s(t2)
+
+        s3 = s(t3)
+        s3.spawn_tree_locals['stl'] = 'STL'
+        s3.join()
+
+
+        s4 = s(util.GreenletTree.current_tree)
+        s4.join()
+
+        tree = s4.value
+        self.assertTrue(tree.root)
+
+        self.assertNotIn('Parent', str(tree)) # Simple output
+        value = tree.format(details={'stacks': False})
+        hexobj = re.compile('0x[0123456789abcdef]+L?', re.I)
+        value = hexobj.sub('X', value)
+        value = value.replace('epoll', 'select')
+        value = value.replace('select', 'default')
+        value = value.replace('test__util', '__main__')
+        value = re.compile(' fileno=.').sub('', value)
+        value = value.replace('ref=-1', 'ref=0')
+
+        self.maxDiff = None
+        expected = """\
+<greenlet.greenlet object at X>
+ :    Greenlet Locals:
+ :      Local <class '__main__.MyLocal'> at X
+ :        {'foo': 42}
+ +--- <QuietHub at X default default pending=0 ref=0>
+ :          Parent: <greenlet.greenlet object at X>
+ +--- <Greenlet "Greenlet-1" at X: _run>; finished with value <Greenlet "Greenlet-0" at X
+ :          Parent: <QuietHub at X default default pending=0 ref=0>
+ |    +--- <Greenlet "Greenlet-0" at X: _run>; finished with exception ExpectedException()
+ :                Parent: <QuietHub at X default default pending=0 ref=0>
+ +--- <Greenlet "Greenlet-2" at X: _run>; finished with value <Greenlet "Greenlet-4" at X
+ :          Parent: <QuietHub at X default default pending=0 ref=0>
+ |    +--- <Greenlet "Greenlet-4" at X: _run>; finished with exception ExpectedException()
+ :                Parent: <QuietHub at X default default pending=0 ref=0>
+ +--- <Greenlet "Greenlet-3" at X: _run>; finished with value <Greenlet "Greenlet-5" at X
+ :          Parent: <QuietHub at X default default pending=0 ref=0>
+ :          Spawn Tree Locals
+ :          {'stl': 'STL'}
+ |    +--- <Greenlet "Greenlet-5" at X: _run>; finished with value <Greenlet "Greenlet-6" at X
+ :                Parent: <QuietHub at X default default pending=0 ref=0>
+ |         +--- <Greenlet "Greenlet-6" at X: _run>; finished with exception ExpectedException()
+ :                      Parent: <QuietHub at X default default pending=0 ref=0>
+ +--- <Greenlet "Greenlet-7" at X: _run>; finished with value <gevent.util.GreenletTree obje
+            Parent: <QuietHub at X default default pending=0 ref=0>
+        """.strip()
+        self.assertEqual(value, expected)
 
 if __name__ == '__main__':
     greentest.main()