Add support for a background monitoring thread to be associated with each hub.

Right now, it is used to detect blocked event loops, and it is extensible by users. In the future there will be some more default monitoring options (e.g., memory).

Refs #1021.

CHANGES.rst

@@ -23,6 +23,21 @@
 - Add additional optimizations for spawning greenlets, making it
   faster than 1.3a2.
 
+- Add an optional monitoring thread for each hub. When enabled, this
+  thread (by default) looks for greenlets that block the event loop
+  for more than 0.1s. You can add your own periodic monitoring
+  functions to this thread.
+
+- When gevent prints a timestamp as part of an error message, it is
+  now in UTC format as specified by RFC3339.
+
+- Threadpool threads that exit now always destroy their hub (if one
+  was created). This prevents some forms of resource leaks (notably
+  visible as blocking functions reported by the new monitoring abilities).
+
+- Hub objects now include the value of their ``name`` attribute in
+  their repr.
+
 1.3a2 (2018-03-06)
 ==================
 

src/gevent/_compat.py

@@ -23,6 +23,7 @@ if PY3:
     integer_types = (int,)
     text_type = str
     native_path_types = (str, bytes)
+    thread_mod_name = '_thread'
 
 else:
     import __builtin__ # pylint:disable=import-error
@@ -30,6 +31,7 @@ else:
     text_type = __builtin__.unicode
     integer_types = (int, __builtin__.long)
     native_path_types = string_types
+    thread_mod_name = 'thread'
 
 
 ## Exceptions

src/gevent/_config.py

@@ -265,6 +265,30 @@ class ImportableSetting(object):
             return value
         return self._import([self.shortname_map.get(x, x) for x in value])
 
+class BoolSettingMixin(object):
+    validate = staticmethod(validate_bool)
+    # Don't do string-to-list conversion.
+    _convert = staticmethod(convert_str_value_as_is)
+
+class IntSettingMixin(object):
+    # Don't do string-to-list conversion.
+    def _convert(self, value):
+        if value:
+            return int(value)
+
+    validate = staticmethod(validate_anything)
+
+
+class FloatSettingMixin(object):
+    def _convert(self, value):
+        if value:
+            return float(value)
+
+    def validate(self, value):
+        if value is not None and value <= 0:
+            raise ValueError("Must be > 0")
+        return value
+
 
 class Resolver(ImportableSetting, Setting):
 
@@ -364,7 +388,7 @@ class FileObject(ImportableSetting, Setting):
     }
 
 
-class WatchChildren(Setting):
+class WatchChildren(BoolSettingMixin, Setting):
     desc = """\
     Should we *not* watch children with the event loop watchers?
 
@@ -376,14 +400,11 @@ class WatchChildren(Setting):
     environment_key = 'GEVENT_NOWAITPID'
     default = False
 
-    validate = staticmethod(validate_bool)
-
 
-class TraceMalloc(Setting):
+class TraceMalloc(IntSettingMixin, Setting):
     name = 'trace_malloc'
     environment_key = 'PYTHONTRACEMALLOC'
     default = False
-    validate = staticmethod(validate_bool)
 
     desc = """\
     Should FFI objects track their allocation?
@@ -399,15 +420,11 @@ class TraceMalloc(Setting):
     """
 
 
-class TrackGreenletTree(Setting):
+class TrackGreenletTree(BoolSettingMixin, Setting):
     name = 'track_greenlet_tree'
     environment_key = 'GEVENT_TRACK_GREENLET_TREE'
     default = True
 
-    validate = staticmethod(validate_bool)
-    # Don't do string-to-list conversion.
-    _convert = staticmethod(convert_str_value_as_is)
-
     desc = """\
     Should `Greenlet` objects track their spawning tree?
 
@@ -419,6 +436,57 @@ class TrackGreenletTree(Setting):
     .. versionadded:: 1.3b1
     """
 
+class MonitorThread(BoolSettingMixin, Setting):
+    name = 'monitor_thread'
+    environment_key = 'GEVENT_ENABLE_MONITOR_THREAD'
+    default = False
+
+    desc = """\
+    Should each hub start a native OS thread to monitor
+    for problems?
+
+    Such a thread will periodically check to see if the event loop
+    is blocked for longer than `max_blocking_time`, producing output on
+    the hub's exception stream (stderr by default) if it detects this condition.
+
+    If this setting is true, then this thread will be created
+    the first time the hub is switched to,
+    or you can call `gevent.hub.Hub.start_periodic_monitoring_thread` at any
+    time to create it (from the same thread that will run the hub). That function
+    will return an object with a method ``add_monitoring_function(function, period)``
+    that you can call to add your own periodic monitoring function. ``function``
+    will be called with one argument, the hub it is monitoring. It will be called
+    in a separate native thread than the one running the hub and **must not**
+    attempt to use the gevent asynchronous API.
+
+    .. seealso:: `max_blocking_time`
+
+    .. versionadded:: 1.3b1
+    """
+
+class MaxBlockingTime(FloatSettingMixin, Setting):
+    name = 'max_blocking_time'
+    environment_key = 'GEVENT_MAX_BLOCKING_TIME'
+    default = 0.1
+
+    desc = """\
+    If the `monitor_thread` is enabled, this is
+    approximately how long (in seconds)
+    the event loop will be allowed to block before a warning is issued.
+
+    This function depends on using `greenlet.settrace`, so installing
+    your own trace function after starting the monitoring thread will
+    cause this feature to misbehave unless you call the function
+    returned by `greenlet.settrace`. If you install a tracing function *before*
+    the monitoring thread is started, it will still be called.
+
+    .. note:: In the unlikely event of creating and using multiple different
+        gevent hubs in the same native thread in a short period of time,
+        especially without destroying the hubs, false positives may be reported.
+
+    .. versionadded:: 1.3b1
+    """
+
 # The ares settings are all interpreted by
 # gevent/resolver/ares.pyx, so we don't do
 # any validation here.
@@ -540,7 +608,7 @@ class ResolverNameservers(AresSettingMixin, Setting):
         return 'servers'
 
 # Generic timeout, works for dnspython and ares
-class ResolverTimeout(AresSettingMixin, Setting):
+class ResolverTimeout(FloatSettingMixin, AresSettingMixin, Setting):
     document = True
     name = 'resolver_timeout'
     environment_key = 'GEVENT_RESOLVER_TIMEOUT'
@@ -552,10 +620,6 @@ class ResolverTimeout(AresSettingMixin, Setting):
     .. versionadded:: 1.3a2
     """
 
-    def _convert(self, value):
-        if value:
-            return float(value)
-
     @property
     def kwarg_name(self):
         return 'timeout'

src/gevent/_threading.py

@@ -83,7 +83,7 @@ class Condition(object):
         self.__waiters.append(waiter)
         saved_state = self._release_save()
         try:    # restore state no matter what (e.g., KeyboardInterrupt)
-            waiter.acquire()
+            waiter.acquire() # Block on the native lock
         finally:
             self._acquire_restore(saved_state)
 

src/gevent/hub.py

@@ -11,7 +11,10 @@ import sys
 import traceback
 from weakref import ref as wref
 
-from greenlet import greenlet as RawGreenlet, getcurrent, GreenletExit
+from greenlet import greenlet as RawGreenlet
+from greenlet import getcurrent
+from greenlet import GreenletExit
+from greenlet import settrace
 
 
 __all__ = [
@@ -30,37 +33,34 @@ __all__ = [
 from gevent._config import config as GEVENT_CONFIG
 from gevent._compat import string_types
 from gevent._compat import xrange
+from gevent._compat import thread_mod_name
 from gevent._util import _NONE
 from gevent._util import readproperty
 from gevent._util import Lazy
 from gevent._ident import IdentRegistry
 
-if sys.version_info[0] <= 2:
-    import thread # pylint:disable=import-error
-else:
-    import _thread as thread # python 2 pylint:disable=import-error
+from gevent.monkey import get_original
+from gevent.util import format_run_info
+
 
 # These must be the "real" native thread versions,
 # not monkey-patched.
-threadlocal = thread._local
-
-
-class _threadlocal(threadlocal):
+class _Threadlocal(get_original(thread_mod_name, '_local')):
 
     def __init__(self):
         # Use a class with an initializer so that we can test
         # for 'is None' instead of catching AttributeError, making
         # the code cleaner and possibly solving some corner cases
         # (like #687)
-        threadlocal.__init__(self)
+        super(_Threadlocal, self).__init__()
         self.Hub = None
         self.loop = None
         self.hub = None
 
-_threadlocal = _threadlocal()
+_threadlocal = _Threadlocal()
 
-get_ident = thread.get_ident
-MAIN_THREAD = get_ident()
+get_thread_ident = get_original(thread_mod_name, 'get_ident')
+MAIN_THREAD = get_thread_ident() # XXX: Assuming import is done on the main thread.
 
 
 class LoopExit(Exception):
@@ -453,6 +453,159 @@ def _config(default, envvar):
 
 hub_ident_registry = IdentRegistry()
 
+def _gmctime():
+    import time
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+class PeriodicMonitoringThread(object):
+
+    # A counter, incremented by the greenlet trace function
+    # we install on every greenlet switch. This is reset when the
+    # periodic monitoring thread runs.
+    _greenlet_switch_counter = 0
+    # The greenlet being switched to.
+    _active_greenlet = None
+
+    # The trace function that was previously installed,
+    # if any.
+    previous_trace_function = staticmethod(lambda _event, _args: None)
+
+    # The absolute minimum we will sleep, regardless of
+    # what particular monitoring functions want to say.
+    min_sleep_time = 0.005
+
+    # A list of tuples: [(function(hub), period))]
+    _additional_monitoring_functions = ()
+
+    def __init__(self, hub):
+        self._hub_wref = wref(hub, self.kill)
+        self.should_run = True
+
+        # Must be installed in the thread that the hub is running in;
+        # the trace function is threadlocal
+        assert get_thread_ident() == hub.thread_ident
+        prev_trace = settrace(self.greenlet_trace)
+        self.previous_trace_function = prev_trace or self.previous_trace_function
+
+        # Create the actual monitoring thread. This is effectively a "daemon"
+        # thread.
+        self.monitor_thread_ident = get_original(thread_mod_name, 'start_new_thread')(self, ())
+
+    def greenlet_trace(self, event, args):
+        # This function runs in the thread we are monitoring.
+        self._greenlet_switch_counter += 1
+        if event in ('switch', 'throw'):
+            # args is (origin, target). This is the only defined
+            # case
+            self._active_greenlet = args[1]
+        else:
+            self._active_greenlet = None
+        self.previous_trace_function(event, args)
+
+    def monitoring_functions(self):
+        # Return a list of tuples: [(function, period)]
+
+        # Calculate our hardcoded list every time so that changes to max_blocking_time can
+        # happen at runtime.
+        monitoring_functions = [(self.monitor_blocking, GEVENT_CONFIG.max_blocking_time)]
+        monitoring_functions.extend(self._additional_monitoring_functions)
+        return monitoring_functions
+
+    def add_monitoring_function(self, function, period):
+        self._additional_monitoring_functions += ((function, period),)
+
+    def calculate_sleep_time(self, monitoring_functions):
+        min_sleep = min(x[1] or 0 for x in monitoring_functions)
+        return  max((min_sleep, self.min_sleep_time))
+
+    def kill(self):
+        # Stop this monitoring thread from running.
+        self.should_run = False
+
+    def __call__(self):
+        # The function that runs in the monitoring thread.
+        # We cannot use threading.current_thread because it would
+        # create an immortal DummyThread object.
+        getcurrent().gevent_monitoring_thread = wref(self)
+        thread_sleep = get_original('time', 'sleep')
+        try:
+            while self.should_run:
+                functions = self.monitoring_functions()
+                assert functions
+                sleep_time = self.calculate_sleep_time(functions)
+
+                thread_sleep(sleep_time)
+
+                # Make sure the hub is still around, and still active,
+                # and keep it around while we are here.
+                hub = self._hub_wref()
+                if hub:
+                    for f, _ in functions:
+                        f(hub)
+        except SystemExit:
+            pass
+        except: # pylint:disable=bare-except
+            # We're a daemon thread, so swallow any exceptions that get here
+            # during interpreter shutdown.
+            if not sys or not sys.stderr:
+                # Interpreter is shutting down
+                pass
+            else:
+                hub = self._hub_wref()
+                if hub is not None:
+                    hub.handle_error(self, *sys.exc_info())
+
+    def monitor_blocking(self, hub):
+        # Called periodically to see if the trace function has
+        # fired to switch greenlets. If not, we will print
+        # the greenlet tree.
+
+        # There is a race condition with this being incremented in the
+        # thread we're monitoring, but probably not often enough to lead
+        # to annoying false positives.
+
+        did_switch = self._greenlet_switch_counter != 0
+        self._greenlet_switch_counter = 0
+
+        if did_switch or self._active_greenlet is None or isinstance(self._active_greenlet, Hub):
+            # Either we switched, or nothing is running (we got a
+            # trace event we don't know about), or we spent the whole time in the hub,
+            # blocked for IO. Nothing to report.
+            return
+
+        if (self._active_greenlet is getcurrent()
+                or getattr(self._active_greenlet, 'gevent_monitoring_thread', None)):
+            # Ourself or another monitoring thread for the same hub thread.
+            # This happens if multiple hubs are used in quick succession (probably only in tests)
+            # Ignore it. (XXX: Maybe we should kill() ourself?)
+            return
+
+        report = ['\n%s : Greenlet %s appears to be blocked' %
+                  (_gmctime(), self._active_greenlet)]
+        report.append("    Reported by %s" % (self,))
+        try:
+            frame = sys._current_frames()[hub.thread_ident]
+        except KeyError:
+            # The thread holding the hub has died. Perhaps we shouldn't
+            # even report this?
+            stack = ["Unknown: No thread found for hub %r\n" % (hub,)]
+        else:
+            stack = traceback.format_stack(frame)
+        report.append('Blocked Stack (for thread id %s):' % (hex(hub.thread_ident),))
+        report.append(''.join(stack))
+        report.append("Info:")
+        report.extend(format_run_info(self.monitor_thread_ident))
+        hub.exception_stream.write('\n'.join(report))
+
+    def __repr__(self):
+        return '<%s at %s in thread %s greenlet %r for %r>' % (
+            self.__class__.__name__,
+            hex(id(self)),
+            hex(self.monitor_thread_ident),
+            getcurrent(),
+            self._hub_wref())
+
+
 class Hub(TrackedRawGreenlet):
     """
     A greenlet that runs the event loop.
@@ -480,9 +633,22 @@ class Hub(TrackedRawGreenlet):
 
     threadpool_size = 10
 
+    # An instance of PeriodicMonitoringThread, if started.
+    periodic_monitoring_thread = None
+
+    # The ident of the thread we were created in, which should be the
+    # thread that we run in.
+    thread_ident = None
+
+    #: A string giving the name of this hub. Useful for associating hubs
+    #: with particular threads. Printed as part of the default repr.
+    #:
+    #: .. versionadded:: 1.3b1
+    name = ''
 
     def __init__(self, loop=None, default=None):
         TrackedRawGreenlet.__init__(self, None, None)
+        self.thread_ident = get_thread_ident()
         if hasattr(loop, 'run'):
             if default is not None:
                 raise TypeError("Unexpected argument: default")
@@ -493,7 +659,7 @@ class Hub(TrackedRawGreenlet):
             # loop. See #237 and #238.
             self.loop = _threadlocal.loop
         else:
-            if default is None and get_ident() != MAIN_THREAD:
+            if default is None and self.thread_ident != MAIN_THREAD:
                 default = False
 
             if loop is None:
@@ -516,6 +682,16 @@ class Hub(TrackedRawGreenlet):
     def backend(self):
         return GEVENT_CONFIG.libev_backend
 
+    @property
+    def main_hub(self):
+        """
+        Is this the hub for the main thread?
+
+        .. versionadded:: 1.3b1
+        """
+        return self.thread_ident == MAIN_THREAD
+
+
     def __repr__(self):
         if self.loop is None:
             info = 'destroyed'
@@ -524,11 +700,16 @@ class Hub(TrackedRawGreenlet):
                 info = self.loop._format()
             except Exception as ex: # pylint:disable=broad-except
                 info = str(ex) or repr(ex) or 'error'
-        result = '<%s at 0x%x %s' % (self.__class__.__name__, id(self), info)
+        result = '<%s %r at 0x%x %s' % (
+            self.__class__.__name__,
+            self.name,
+            id(self),
+            info)
         if self._resolver is not None:
             result += ' resolver=%r' % self._resolver
         if self._threadpool is not None:
             result += ' threadpool=%r' % self._threadpool
+        result += ' thread_ident=%s' % (hex(self.thread_ident), )
         return result + '>'
 
     def handle_error(self, context, type, value, tb):
@@ -602,8 +783,7 @@ class Hub(TrackedRawGreenlet):
         del tb
 
         try:
-            import time
-            errstream.write(time.ctime())
+            errstream.write(_gmctime())
             errstream.write(' ' if context is not None else '\n')
         except: # pylint:disable=bare-except
             # Possible not safe to import under certain
@@ -695,7 +875,8 @@ class Hub(TrackedRawGreenlet):
            programming error.
         """
         assert self is getcurrent(), 'Do not call Hub.run() directly'
-        while True:
+        self.start_periodic_monitoring_thread()
+        while 1:
             loop = self.loop
             loop.error_handler = self
             try:
@@ -711,6 +892,19 @@ class Hub(TrackedRawGreenlet):
         # It is still possible to kill this greenlet with throw. However, in that case
         # switching to it is no longer safe, as switch will return immediately
 
+    def start_periodic_monitoring_thread(self):
+        if self.periodic_monitoring_thread is None and GEVENT_CONFIG.monitor_thread:
+            # TODO: If we're the main thread, then add the memory monitoring
+            # function.
+
+            # Note that it is possible for one real thread to
+            # (temporarily) wind up with multiple monitoring threads,
+            # if hubs are started and stopped within the thread. This shows up
+            # in the threadpool tests. The monitoring threads will eventually notice their
+            # hub object is gone.
+            self.periodic_monitoring_thread = PeriodicMonitoringThread(self)
+        return self.periodic_monitoring_thread
+
     def join(self, timeout=None):
         """Wait for the event loop to finish. Exits only when there are
         no more spawned greenlets, started servers, active timeouts or watchers.
@@ -742,6 +936,9 @@ class Hub(TrackedRawGreenlet):
         return False
 
     def destroy(self, destroy_loop=None):
+        if self.periodic_monitoring_thread is not None:
+            self.periodic_monitoring_thread.kill()
+            self.periodic_monitoring_thread = None
         if self._resolver is not None:
             self._resolver.close()
             del self._resolver
@@ -764,6 +961,8 @@ class Hub(TrackedRawGreenlet):
         if _threadlocal.hub is self:
             _threadlocal.hub = None
 
+    # XXX: We can probably simplify the resolver and threadpool properties.
+
     @property
     def resolver_class(self):
         return GEVENT_CONFIG.resolver

src/gevent/threadpool.py

@@ -197,13 +197,18 @@ class ThreadPool(GroupMappingMixin):
             with _lock:
                 self._size -= 1
 
-    _destroy_worker_hub = False
+    # XXX: This used to be false by default. It really seems like
+    # it should be true to avoid leaking resources.
+    _destroy_worker_hub = True
 
     def _worker(self):
         # pylint:disable=too-many-branches
         need_decrease = True
         try:
             while 1: # tiny bit faster than True on Py2
+                h = _get_hub()
+                if h is not None:
+                    h.name = 'ThreadPool Worker Hub'
                 task_queue = self.task_queue
                 task = task_queue.get()
                 try:

src/gevent/util.py

@@ -13,7 +13,6 @@ import traceback
 from greenlet import getcurrent
 from greenlet import greenlet as RawGreenlet
 
-from gevent.local import all_local_dicts_for_greenlet
 
 __all__ = [
     'wrap_errors',
@@ -76,8 +75,10 @@ class wrap_errors(object):
     def __getattr__(self, name):
         return getattr(self.__func, name)
 
-def format_run_info():
+def format_run_info(_current_thread_ident=None):
     """
+    format_run_info() -> []
+
     Request information about the running threads of the current process.
 
     This is a debugging utility. Its output has no guarantees other than being
@@ -85,7 +86,10 @@ def format_run_info():
 
     :return: A sequence of text lines detailing the stacks of running
             threads and greenlets. (One greenlet will duplicate one thread,
-            the current thread and greenlet.) Extra information about
+            the current thread and greenlet. If there are multiple running threads,
+            the stack for the current greenlet may be incorrectly duplicated in multiple
+            greenlets.)
+            Extra information about
             :class:`gevent.greenlet.Greenlet` object will also be returned.
 
     .. versionadded:: 1.3a1
@@ -97,24 +101,30 @@ def format_run_info():
 
     lines = []
 
-    _format_thread_info(lines)
+    _format_thread_info(lines, _current_thread_ident)
     _format_greenlet_info(lines)
     return lines
 
-def _format_thread_info(lines):
+def _format_thread_info(lines, current_thread_ident):
     import threading
     import sys
 
-    threads = {th.ident: th.name for th in threading.enumerate()}
+    threads = {th.ident: th for th in threading.enumerate()}
 
     lines.append('*' * 80)
     lines.append('* Threads')
 
     thread = None
     frame = None
-    for thread, frame in sys._current_frames().items():
+    for thread_ident, frame in sys._current_frames().items():
         lines.append("*" * 80)
-        lines.append('Thread 0x%x (%s)\n' % (thread, threads.get(thread)))
+        thread = threads.get(thread_ident)
+        name = thread.name if thread else None
+        if getattr(thread, 'gevent_monitoring_thread', None):
+            name = repr(thread.gevent_monitoring_thread())
+        if current_thread_ident == thread_ident:
+            name = '%s) (CURRENT' % (name,)
+        lines.append('Thread 0x%x (%s)\n' % (thread_ident, name))
         lines.append(''.join(traceback.format_stack(frame)))
 
     # We may have captured our own frame, creating a reference
@@ -288,6 +298,9 @@ class GreenletTree(object):
         return (getattr(greenlet, 'spawning_greenlet', None) or _noop)()
 
     def __render_locals(self, tree):
+        # Defer the import to avoid cycles
+        from gevent.local import all_local_dicts_for_greenlet
+
         gr_locals = all_local_dicts_for_greenlet(self.greenlet)
         if gr_locals:
             tree.child_data("Greenlet Locals:")
@@ -309,9 +322,11 @@ class GreenletTree(object):
                 label += '; not running'
         tree.node_label(label)
 
-        if self.greenlet.parent is not None:
         tree.child_data('Parent: ' + repr(self.greenlet.parent))
 
+        if getattr(self.greenlet, 'gevent_monitoring_thread', None) is not None:
+            tree.child_data('Monitoring Thread:' + repr(self.greenlet.gevent_monitoring_thread()))
+
         if self.greenlet and tree.details and tree.details['stacks']:
             self.__render_tb(tree, 'Running:', self.greenlet.gr_frame)
 

src/greentest/test__hub.py

@@ -26,6 +26,7 @@ import re
 import gevent
 from gevent import socket
 from gevent.hub import Waiter, get_hub
+from gevent._compat import PYPY
 
 DELAY = 0.1
 
@@ -115,5 +116,57 @@ class TestWaiter(greentest.TestCase):
         g.kill()
 
 
+class TestPeriodicMonitoringThread(greentest.TestCase):
+
+    def setUp(self):
+        super(TestPeriodicMonitoringThread, self).setUp()
+        self.monitor_thread = gevent.config.monitor_thread
+        gevent.config.monitor_thread = True
+        self.monitor_fired = 0
+
+    def tearDown(self):
+        if not self.monitor_thread and get_hub().periodic_monitoring_thread:
+            # If it was true, nothing to do. If it was false, tear things down.
+            get_hub().periodic_monitoring_thread.kill()
+            get_hub().periodic_monitoring_thread = None
+        gevent.config.monitor_thread = self.monitor_thread
+
+    def _monitor(self, _hub):
+        self.monitor_fired += 1
+
+    def test_config(self):
+        self.assertEqual(0.1, gevent.config.max_blocking_time)
+
+    def test_blocking(self):
+        import io
+        hub = get_hub()
+        monitor = hub.start_periodic_monitoring_thread()
+        self.assertIsNotNone(monitor)
+        before_funs = monitor._additional_monitoring_functions
+
+        monitor.add_monitoring_function(self._monitor, 0)
+        self.assertIn((self._monitor, 0), monitor.monitoring_functions())
+
+        # We must make sure we have switched greenlets at least once,
+        # otherwise we can't detect a failure.
+        gevent.sleep(0.01)
+        stream = hub.exception_stream = io.BytesIO() if str is bytes else io.StringIO()
+        assert hub.exception_stream is stream
+        try:
+            time.sleep(0.3) # Thrice the default; PyPy is very slow to format stacks
+            # XXX: This is racy even on CPython
+        finally:
+            monitor._additional_monitoring_functions = before_funs
+            assert hub.exception_stream is stream
+            del hub.exception_stream
+
+        if not PYPY:
+            # PyPy may still be formatting the stacks in the other thread.
+            self.assertGreaterEqual(self.monitor_fired, 1)
+            data = stream.getvalue()
+            self.assertIn('appears to be blocked', data)
+            self.assertIn('PeriodicMonitoringThread', data)
+
+
 if __name__ == '__main__':
     greentest.main()

src/greentest/test__util.py

@@ -135,31 +135,32 @@ class TestTree(greentest.TestCase):
         self.maxDiff = None
         expected = """\
 <greenlet.greenlet object at X>
+ :    Parent: None
  :    Greenlet Locals:
  :      Local <class '__main__.MyLocal'> at X
  :        {'foo': 42}
- +--- <QuietHub at X default default pending=0 ref=0>
+ +--- <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  :          Parent: <greenlet.greenlet object at X>
  +--- <Greenlet "Greenlet-1" at X: _run>; finished with value <Greenlet "Greenlet-0" at X
- :          Parent: <QuietHub at X default default pending=0 ref=0>
+ :          Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  |    +--- <Greenlet "Greenlet-0" at X: _run>; finished with exception ExpectedException()
- :                Parent: <QuietHub at X default default pending=0 ref=0>
+ :                Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  +--- <Greenlet "Greenlet-2" at X: _run>; finished with value <Greenlet "Greenlet-4" at X
- :          Parent: <QuietHub at X default default pending=0 ref=0>
+ :          Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  |    +--- <Greenlet "Greenlet-4" at X: _run>; finished with exception ExpectedException()
- :                Parent: <QuietHub at X default default pending=0 ref=0>
+ :                Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  +--- <Greenlet "Greenlet-3" at X: _run>; finished with value <Greenlet "Greenlet-5" at X
- :          Parent: <QuietHub at X default default pending=0 ref=0>
+ :          Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  :          Spawn Tree Locals
  :          {'stl': 'STL'}
  |    +--- <Greenlet "Greenlet-5" at X: _run>; finished with value <Greenlet "Greenlet-6" at X
- :                Parent: <QuietHub at X default default pending=0 ref=0>
+ :                Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  |         +--- <Greenlet "Greenlet-6" at X: _run>; finished with exception ExpectedException()
- :                      Parent: <QuietHub at X default default pending=0 ref=0>
+ :                      Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
  +--- <Greenlet "Greenlet-7" at X: _run>; finished with value <gevent.util.GreenletTree obje
-            Parent: <QuietHub at X default default pending=0 ref=0>
+            Parent: <QuietHub '' at X default default pending=0 ref=0 thread_ident=X>
         """.strip()
-        self.assertEqual(value, expected)
+        self.assertEqual(expected, value)
 
     @greentest.ignores_leakcheck
     def test_tree_no_track(self):