Add util.print_run_info and limit params for stack traces.

src/gevent/_threading.py

@@ -11,7 +11,7 @@ from collections import deque
 from itertools import islice as _islice
 
 from gevent import monkey
-from gevent._compat import PY3
+from gevent._compat import thread_mod_name
 
 
 __all__ = [
@@ -21,9 +21,8 @@ __all__ = [
 ]
 
 
-thread_name = '_thread' if PY3 else 'thread'
-start_new_thread, Lock, = monkey.get_original(thread_name, [
-    'start_new_thread', 'allocate_lock',
+start_new_thread, Lock, get_thread_ident, = monkey.get_original(thread_mod_name, [
+    'start_new_thread', 'allocate_lock', 'get_ident',
 ])
 
 

src/gevent/threadpool.py

@@ -2,6 +2,11 @@
 from __future__ import absolute_import
 import sys
 import os
+
+from weakref import ref as wref
+
+from greenlet import greenlet as RawGreenlet
+
 from gevent._compat import integer_types
 from gevent.hub import _get_hub_noargs as get_hub
 from gevent.hub import getcurrent
@@ -11,12 +16,39 @@ from gevent.event import AsyncResult
 from gevent.greenlet import Greenlet
 from gevent.pool import GroupMappingMixin
 from gevent.lock import Semaphore
-from gevent._threading import Lock, Queue, start_new_thread
 
+from gevent._threading import Lock
+from gevent._threading import Queue
+from gevent._threading import start_new_thread
+from gevent._threading import get_thread_ident
+
+
+__all__ = [
+    'ThreadPool',
+    'ThreadResult',
+]
 
-__all__ = ['ThreadPool',
-           'ThreadResult']
 
+class _WorkerGreenlet(RawGreenlet):
+    # Exists to produce a more useful repr for worker pool
+    # threads/greenlets.
+
+    def __init__(self, threadpool):
+        RawGreenlet.__init__(self, threadpool._worker)
+        self.thread_ident = get_thread_ident()
+        self._threadpool_wref = wref(threadpool)
+
+        # Inform the gevent.util.GreenletTree that this should be
+        # considered the root (for printing purposes) and to
+        # ignore the parent attribute. (We can't set parent to None.)
+        self.greenlet_tree_is_root = True
+        self.parent.greenlet_tree_is_ignored = True
+
+    def __repr__(self):
+        return "<ThreadPoolWorker at 0x%x thread_ident=0x%x %s>" % (
+            id(self),
+            self.thread_ident,
+            self._threadpool_wref())
 
 class ThreadPool(GroupMappingMixin):
     """
@@ -58,7 +90,11 @@ class ThreadPool(GroupMappingMixin):
     maxsize = property(_get_maxsize, _set_maxsize)
 
     def __repr__(self):
-        return '<%s at 0x%x %s/%s/%s>' % (self.__class__.__name__, id(self), len(self), self.size, self.maxsize)
+        return '<%s at 0x%x %s/%s/%s hub=<%s at 0x%x thread_ident=0x%s>>' % (
+            self.__class__.__name__,
+            id(self),
+            len(self), self.size, self.maxsize,
+            self.hub.__class__.__name__, id(self.hub), self.hub.thread_ident)
 
     def __len__(self):
         # XXX just do unfinished_tasks property
@@ -155,7 +191,7 @@ class ThreadPool(GroupMappingMixin):
         with self._lock:
             self._size += 1
         try:
-            start_new_thread(self._worker, ())
+            start_new_thread(self.__trampoline, ())
         except:
             with self._lock:
                 self._size -= 1
@@ -210,6 +246,13 @@ class ThreadPool(GroupMappingMixin):
         if hub is not None and hub.periodic_monitoring_thread is not None:
             hub.periodic_monitoring_thread.ignore_current_greenlet_blocking()
 
+    def __trampoline(self):
+        # The target that we create new threads with. It exists
+        # solely to create the _WorkerGreenlet and switch to it.
+        # (the __class__ of a raw greenlet cannot be changed.)
+        g = _WorkerGreenlet(self)
+        g.switch()
+
     def _worker(self):
         # pylint:disable=too-many-branches
         need_decrease = True

src/gevent/util.py

@@ -5,20 +5,24 @@ Low-level utilities.
 
 from __future__ import absolute_import, print_function, division
 
-import gc
 import functools
+import gc
 import pprint
+import sys
 import traceback
 
 from greenlet import getcurrent
 from greenlet import greenlet as RawGreenlet
 
 from gevent._compat import PYPY
+from gevent._compat import thread_mod_name
+from gevent._util import _NONE
 
 __all__ = [
-    'wrap_errors',
     'format_run_info',
+    'print_run_info',
     'GreenletTree',
+    'wrap_errors',
 ]
 
 # PyPy is very slow at formatting stacks
@@ -81,41 +85,77 @@ class wrap_errors(object):
     def __getattr__(self, name):
         return getattr(self.__func, name)
 
+
+def print_run_info(thread_stacks=True, greenlet_stacks=True, limit=_NONE, file=None):
+    """
+    Call `format_run_info` and print the results to *file*.
+
+    If *file* is not given, `sys.stderr` will be used.
+
+    .. versionadded:: 1.3b1
+    """
+    lines = format_run_info(thread_stacks=thread_stacks,
+                            greenlet_stacks=greenlet_stacks,
+                            limit=limit)
+    file = sys.stderr if file is None else file
+    for l in lines:
+        print(l, file=file)
+
+
 def format_run_info(thread_stacks=True,
                     greenlet_stacks=True,
+                    limit=_NONE,
                     current_thread_ident=None):
     """
-    format_run_info(thread_stacks=True, greenlet_stacks=True) -> [str]
+    format_run_info(thread_stacks=True, greenlet_stacks=True, limit=None) -> [str]
 
     Request information about the running threads of the current process.
 
     This is a debugging utility. Its output has no guarantees other than being
     intended for human consumption.
 
+    :keyword bool thread_stacks: If true, then include the stacks for
+       running threads.
+    :keyword bool greenlet_stacks: If true, then include the stacks for
+       running greenlets. (Spawning stacks will always be printed.)
+       Setting this to False can reduce the output volume considerably
+       without reducing the overall information if *thread_stacks* is true
+       and you can associate a greenlet to a thread (using ``thread_ident``
+       printed values).
+    :keyword int limit: If given, passed directly to `traceback.format_stack`.
+       If not given, this defaults to the whole stack under CPython, and a
+       smaller stack under PyPy.
+
     :return: A sequence of text lines detailing the stacks of running
             threads and greenlets. (One greenlet will duplicate one thread,
             the current thread and greenlet. If there are multiple running threads,
             the stack for the current greenlet may be incorrectly duplicated in multiple
             greenlets.)
             Extra information about
-            :class:`gevent.greenlet.Greenlet` object will also be returned.
+            :class:`gevent.Greenlet` object will also be returned.
 
     .. versionadded:: 1.3a1
     .. versionchanged:: 1.3a2
        Renamed from ``dump_stacks`` to reflect the fact that this
        prints additional information about greenlets, including their
        spawning stack, parent, locals, and any spawn tree locals.
+    .. versionchanged:: 1.3b1
+       Added the *thread_stacks*, *greenlet_stacks*, and *limit* params.
     """
+    if current_thread_ident is None:
+        from gevent import monkey
+        current_thread_ident = monkey.get_original(thread_mod_name, 'get_ident')()
 
     lines = []
 
-    _format_thread_info(lines, thread_stacks, current_thread_ident)
-    _format_greenlet_info(lines, greenlet_stacks)
+    limit = _STACK_LIMIT if limit is _NONE else limit
+    _format_thread_info(lines, thread_stacks, limit, current_thread_ident)
+    _format_greenlet_info(lines, greenlet_stacks, limit)
     return lines
 
-def _format_thread_info(lines, thread_stacks, current_thread_ident):
+
+def _format_thread_info(lines, thread_stacks, limit, current_thread_ident):
     import threading
-    import sys
 
     threads = {th.ident: th for th in threading.enumerate()}
 
@@ -134,7 +174,7 @@ def _format_thread_info(lines, thread_stacks, current_thread_ident):
             name = '%s) (CURRENT' % (name,)
         lines.append('Thread 0x%x (%s)\n' % (thread_ident, name))
         if thread_stacks:
-            lines.append(''.join(traceback.format_stack(frame, _STACK_LIMIT)))
+            lines.append(''.join(traceback.format_stack(frame, limit)))
         else:
             lines.append('\t...stack elided...')
 
@@ -145,14 +185,17 @@ def _format_thread_info(lines, thread_stacks, current_thread_ident):
     del lines
     del threads
 
-def _format_greenlet_info(lines, greenlet_stacks):
+def _format_greenlet_info(lines, greenlet_stacks, limit):
     # Use the gc module to inspect all objects to find the greenlets
     # since there isn't a global registry
     lines.append('*' * 80)
     lines.append('* Greenlets')
     lines.append('*' * 80)
     for tree in GreenletTree.forest():
-        lines.extend(tree.format_lines(details={'running_stacks': greenlet_stacks}))
+        lines.extend(tree.format_lines(details={
+            'running_stacks': greenlet_stacks,
+            'running_stack_limit': limit,
+        }))
 
     del lines
 
@@ -273,6 +316,7 @@ class GreenletTree(object):
 
     DEFAULT_DETAILS = {
         'running_stacks': True,
+        'running_stack_limit': _STACK_LIMIT,
         'spawning_stacks': True,
         'locals': True,
     }
@@ -309,9 +353,9 @@ class GreenletTree(object):
         return self.format(False)
 
     @staticmethod
-    def __render_tb(tree, label, frame):
+    def __render_tb(tree, label, frame, limit):
         tree.child_data(label)
-        tb = ''.join(traceback.format_stack(frame, _STACK_LIMIT))
+        tb = ''.join(traceback.format_stack(frame, limit))
         tree.child_multidata(tb)
 
     @staticmethod
@@ -349,12 +393,14 @@ class GreenletTree(object):
             tree.child_data('Monitoring Thread:' + repr(self.greenlet.gevent_monitoring_thread()))
 
         if self.greenlet and tree.details and tree.details['running_stacks']:
-            self.__render_tb(tree, 'Running:', self.greenlet.gr_frame)
+            self.__render_tb(tree, 'Running:', self.greenlet.gr_frame,
+                             tree.details['running_stack_limit'])
 
 
         spawning_stack = getattr(self.greenlet, 'spawning_stack', None)
         if spawning_stack and tree.details and tree.details['spawning_stacks']:
-            self.__render_tb(tree, 'Spawned at:', spawning_stack)
+            # We already placed a limit on the spawning stack when we captured it.
+            self.__render_tb(tree, 'Spawned at:', spawning_stack, None)
 
         spawning_parent = self.__spawning_parent(self.greenlet)
         tree_locals = getattr(self.greenlet, 'spawn_tree_locals', None)
@@ -399,7 +445,7 @@ class GreenletTree(object):
 
     @staticmethod
     def _root_greenlet(greenlet):
-        while greenlet.parent is not None:
+        while greenlet.parent is not None and not getattr(greenlet, 'greenlet_tree_is_root', False):
             greenlet = greenlet.parent
         return greenlet
 
@@ -416,6 +462,8 @@ class GreenletTree(object):
         for ob in gc.get_objects():
             if not isinstance(ob, RawGreenlet):
                 continue
+            if getattr(ob, 'greenlet_tree_is_ignored', False):
+                continue
 
             spawn_parent = cls.__spawning_parent(ob)
 

src/greentest/test__threadpool.py

@@ -10,10 +10,11 @@ from gevent.threadpool import ThreadPool
 import gevent
 from greentest import ExpectedException
 from greentest import six
+from greentest import PYPY
 import gc
 
 
-PYPY = hasattr(sys, 'pypy_version_info')
+# pylint:disable=too-many-ancestors
 
 
 @contextlib.contextmanager
@@ -151,6 +152,22 @@ if greentest.PYPY and (greentest.WIN or greentest.RUN_COVERAGE):
 
 class TestPool(_AbstractPoolTest):
 
+    def test_greenlet_class(self):
+        from greenlet import getcurrent
+        from gevent.threadpool import _WorkerGreenlet
+        worker_greenlet = self.pool.apply(getcurrent)
+
+        self.assertIsInstance(worker_greenlet, _WorkerGreenlet)
+        r = repr(worker_greenlet)
+        self.assertIn('ThreadPoolWorker', r)
+        self.assertIn('thread_ident', r)
+        self.assertIn('hub=', r)
+
+        from gevent.util import format_run_info
+
+        info = '\n'.join(format_run_info())
+        self.assertIn("<ThreadPoolWorker", info)
+
     def test_apply(self):
         papply = self.pool.apply
         self.assertEqual(papply(sqr, (5,)), sqr(5))

src/greentest/test__util.py

@@ -14,6 +14,8 @@ import gevent
 from gevent import util
 from gevent import local
 
+from gevent._compat import NativeStrIO
+
 class MyLocal(local.local):
     def __init__(self, foo):
         self.foo = foo
@@ -40,14 +42,15 @@ class TestFormat(greentest.TestCase):
             l = MyLocal(42)
             assert l
             gevent.getcurrent().spawn_tree_locals['a value'] = 42
-            g = gevent.spawn(util.format_run_info)
+            io = NativeStrIO()
+            g = gevent.spawn(util.print_run_info, file=io)
             g.join()
-            return g.value
+            return io.getvalue()
 
         g = gevent.spawn(root)
         g.name = 'Printer'
         g.join()
-        value = '\n'.join(g.value)
+        value = g.value
 
         self.assertIn("Spawned at", value)
         self.assertIn("Parent:", value)
@@ -63,6 +66,7 @@ class TestTree(greentest.TestCase):
         super(TestTree, self).setUp()
         self.track_greenlet_tree = gevent.config.track_greenlet_tree
         gevent.config.track_greenlet_tree = True
+        self.maxDiff = None
 
     def tearDown(self):
         gevent.config.track_greenlet_tree = self.track_greenlet_tree
@@ -92,7 +96,9 @@ class TestTree(greentest.TestCase):
         def t2():
             l = MyLocal(16)
             assert l
-            return s(t1)
+            g = s(t1)
+            g.name = 'CustomName-' + str(g.minimal_ident)
+            return g
 
         s1 = s(t2)
         s1.join()
@@ -108,7 +114,6 @@ class TestTree(greentest.TestCase):
             s3.spawn_tree_locals['stl'] = 'STL'
         s3.join()
 
-
         s4 = s(util.GreenletTree.current_tree)
         s4.join()
 
@@ -116,15 +121,8 @@ class TestTree(greentest.TestCase):
         return tree, str(tree), tree.format(details={'running_stacks': False,
                                                      'spawning_stacks': False})
 
-    @greentest.ignores_leakcheck
-    def test_tree(self):
+    def _normalize_tree_format(self, value):
         import re
-        tree, str_tree, tree_format = self._build_tree()
-
-        self.assertTrue(tree.root)
-
-        self.assertNotIn('Parent', str_tree) # Simple output
-        value = tree_format
         hexobj = re.compile('0x[0123456789abcdef]+L?', re.I)
         value = hexobj.sub('X', value)
         value = value.replace('epoll', 'select')
@@ -132,8 +130,17 @@ class TestTree(greentest.TestCase):
         value = value.replace('test__util', '__main__')
         value = re.compile(' fileno=.').sub('', value)
         value = value.replace('ref=-1', 'ref=0')
+        return value
+
+    @greentest.ignores_leakcheck
+    def test_tree(self):
+        tree, str_tree, tree_format = self._build_tree()
+
+        self.assertTrue(tree.root)
+
+        self.assertNotIn('Parent', str_tree) # Simple output
+        value = self._normalize_tree_format(tree_format)
 
-        self.maxDiff = None
         expected = """\
 <greenlet.greenlet object at X>
  :    Parent: None
@@ -142,21 +149,21 @@ class TestTree(greentest.TestCase):
  :        {'foo': 42}
  +--- <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  :          Parent: <greenlet.greenlet object at X>
- +--- <Greenlet "Greenlet-1" at X: _run>; finished with value <Greenlet "Greenlet-0" at X
+ +--- <Greenlet "Greenlet-1" at X: _run>; finished with value <Greenlet "CustomName-0" at 0x
  :          Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
- |    +--- <Greenlet "Greenlet-0" at X: _run>; finished with exception ExpectedException()
+ |    +--- <Greenlet "CustomName-0" at X: _run>; finished with exception ExpectedException()
  :                Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
- +--- <Greenlet "Greenlet-2" at X: _run>; finished with value <Greenlet "Greenlet-4" at X
+ +--- <Greenlet "Greenlet-2" at X: _run>; finished with value <Greenlet "CustomName-4" at 0x
  :          Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
- |    +--- <Greenlet "Greenlet-4" at X: _run>; finished with exception ExpectedException()
+ |    +--- <Greenlet "CustomName-4" at X: _run>; finished with exception ExpectedException()
  :                Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  +--- <Greenlet "Greenlet-3" at X: _run>; finished with value <Greenlet "Greenlet-5" at X
  :          Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  :          Spawn Tree Locals
  :          {'stl': 'STL'}
- |    +--- <Greenlet "Greenlet-5" at X: _run>; finished with value <Greenlet "Greenlet-6" at X
+ |    +--- <Greenlet "Greenlet-5" at X: _run>; finished with value <Greenlet "CustomName-6" at 0x
  :                Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
- |         +--- <Greenlet "Greenlet-6" at X: _run>; finished with exception ExpectedException()
+ |         +--- <Greenlet "CustomName-6" at X: _run>; finished with exception ExpectedException()
  :                      Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  +--- <Greenlet "Greenlet-7" at X: _run>; finished with value <gevent.util.GreenletTree obje
             Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
@@ -169,5 +176,32 @@ class TestTree(greentest.TestCase):
         self._build_tree()
 
 
+    @greentest.ignores_leakcheck
+    def test_forest_fake_parent(self):
+        from greenlet import greenlet as RawGreenlet
+
+        def t4():
+            # Ignore this one, make the child the parent,
+            # and don't be a child of the hub.
+            c = RawGreenlet(util.GreenletTree.current_tree)
+            c.parent.greenlet_tree_is_ignored = True
+            c.greenlet_tree_is_root = True
+            return c.switch()
+
+
+        g = RawGreenlet(t4)
+        tree = g.switch()
+
+        tree_format = tree.format(details={'running_stacks': False,
+                                           'spawning_stacks': False})
+        value = self._normalize_tree_format(tree_format)
+
+        expected = """\
+<greenlet.greenlet object at X>; not running
+ :    Parent: <greenlet.greenlet object at X>
+        """.strip()
+
+        self.assertEqual(expected, value)
+
 if __name__ == '__main__':
     greentest.main()