Merge pull request #1150 from gevent/mem-monitor

Add monitoring for memory usage, emitting events as it moves around the threshold.

CHANGES.rst

@@ -4,31 +4,33 @@
 
 .. currentmodule:: gevent
 
-1.3a3 (unreleased)
+1.3b1 (unreleased)
 ==================
 
-- Use strongly typed watcher callbacks in the libuv CFFI extensions.
-  This prevents dozens of compiler warnings.
+Dependencies
+------------
+
+- Cython 0.28.1 is now used to build gevent from a source checkout.
+
+Bug Fixes
+---------
 
 - On Python 2, when monkey-patching `threading.Event`, also
   monkey-patch the underlying class, ``threading._Event``. Some code
   may be type-checking for that. See :issue:`1136`.
 
-- Introduce the configuration variable
-  `gevent.config.track_greenlet_tree` (aka
-  ``GEVENT_TRACK_GREENLET_TREE``) to allow disabling the greenlet tree
-  features for applications where greenlet spawning is performance
-  critical. This restores spawning performance to 1.2 levels.
+- Fix libuv io watchers polling for events that only stopped watchers
+  are interested in, reducing CPU usage. Reported in :issue:`1144` by
+  wwqgtxx.
+
+Enhancements
+------------
 
 - Add additional optimizations for spawning greenlets, making it
   faster than 1.3a2.
 
-- Add an optional monitoring thread for each hub. When enabled, this
-  thread (by default) looks for greenlets that block the event loop
-  for more than 0.1s. You can add your own periodic monitoring
-  functions to this thread. Set ``GEVENT_MONITOR_THREAD_ENABLE`` to
-  use it, and ``GEVENT_MAX_BLOCKING_TIME`` to configure the blocking
-  interval.
+- Use strongly typed watcher callbacks in the libuv CFFI extensions.
+  This prevents dozens of compiler warnings.
 
 - When gevent prints a timestamp as part of an error message, it is
   now in UTC format as specified by RFC3339.
@@ -40,15 +42,33 @@
 - Hub objects now include the value of their ``name`` attribute in
   their repr.
 
-- Fix libuv io watchers polling for events that only stopped watchers
-  are interested in, reducing CPU usage. Reported in :issue:`1144` by
-  wwqgtxx.
+Monitoring and Debugging
+------------------------
+
+- Introduce the configuration variable
+  `gevent.config.track_greenlet_tree` (aka
+  ``GEVENT_TRACK_GREENLET_TREE``) to allow disabling the greenlet tree
+  features for applications where greenlet spawning is performance
+  critical. This restores spawning performance to 1.2 levels.
+
+- Add an optional monitoring thread for each hub. When enabled, this
+  thread (by default) looks for greenlets that block the event loop
+  for more than 0.1s. You can add your own periodic monitoring
+  functions to this thread. Set ``GEVENT_MONITOR_THREAD_ENABLE`` to
+  use it, and ``GEVENT_MAX_BLOCKING_TIME`` to configure the blocking
+  interval.
 
 - Add a simple event framework for decoupled communication. It uses
   :mod:`zope.event` if that is installed. The monitoring thread emits
   events when it detects certain conditions, like loop blocked or
   memory limits exceeded.
 
+- Add settings for monitoring memory usage and emitting events when a
+  threshold is exceeded and then corrected. gevent currently supplies
+  no policy for what to do when memory exceeds the configured limit.
+  ``psutil`` must be installed to use this. See :pr:`1150`.
+
+
 1.3a2 (2018-03-06)
 ==================
 

ci-requirements.txt

@@ -5,7 +5,7 @@ wheel
 # 0.28 is faster, and (important!) lets us specify the target module
 # name to be created so that we can have both foo.py and _foo.so
 # at the same time.
-Cython >= 0.28
+Cython >= 0.28.1
 
 # Python 3.7b1 requires this.
 greenlet>=0.4.13 ; platform_python_implementation == "CPython"

rtd-requirements.txt

-cython >= 0.28
+cython >= 0.28.1
 repoze.sphinx.autointerface

src/gevent/__init__.py

@@ -18,11 +18,16 @@ _version_info = namedtuple('version_info',
 
 #: The programatic version identifier. The fields have (roughly) the
 #: same meaning as :data:`sys.version_info`
-#: Deprecated in 1.2.
+#: .. deprecated:: 1.2
+#:  Use ``pkg_resources.parse_version(__version__)`` (or the equivalent
+#:  ``packaging.version.Version(__version__)``).
 version_info = _version_info(1, 3, 0, 'dev', 0)
 
-#: The human-readable PEP 440 version identifier
-__version__ = '1.3a3.dev0'
+#: The human-readable PEP 440 version identifier.
+#: Use ``pkg_resources.parse_version(__version__)`` or
+#: ``packaging.version.Version(__version__)`` to get a machine-usable
+#: value.
+__version__ = '1.3b1.dev0'
 
 
 __all__ = [

src/gevent/_compat.py

@@ -140,3 +140,17 @@ except ImportError:
             # exist. Probably on Python 2 with an astral character.
             # Not sure how to handle this.
             raise UnicodeEncodeError("Can't encode path to filesystem encoding")
+
+
+## Clocks
+try:
+    # Python 3.3+ (PEP 418)
+    from time import perf_counter
+    perf_counter = perf_counter
+except ImportError:
+    import time
+
+    if sys.platform == "win32":
+        perf_counter = time.clock
+    else:
+        perf_counter = time.time

src/gevent/_config.py

@@ -279,17 +279,41 @@ class IntSettingMixin(object):
     validate = staticmethod(validate_anything)
 
 
-class FloatSettingMixin(object):
-    def _convert(self, value):
-        if value:
-            return float(value)
+class _PositiveValueMixin(object):
 
     def validate(self, value):
         if value is not None and value <= 0:
-            raise ValueError("Must be > 0")
+            raise ValueError("Must be positive")
         return value
 
 
+class FloatSettingMixin(_PositiveValueMixin):
+    def _convert(self, value):
+        if value:
+            return float(value)
+
+
+class ByteCountSettingMixin(_PositiveValueMixin):
+
+    _MULTIPLES = {
+        # All keys must be the same size.
+        'kb': 1024,
+        'mb': 1024 * 1024,
+        'gb': 1024 * 1024 * 1024,
+    }
+
+    _SUFFIX_SIZE = 2
+
+    def _convert(self, value):
+        if not value or not isinstance(value, str):
+            return value
+        value = value.lower()
+        for s, m in self._MULTIPLES.items():
+            if value[-self._SUFFIX_SIZE:] == s:
+                return int(value[:-self._SUFFIX_SIZE]) * m
+        return int(value)
+
+
 class Resolver(ImportableSetting, Setting):
 
     desc = """\
@@ -493,6 +517,36 @@ class MaxBlockingTime(FloatSettingMixin, Setting):
     .. versionadded:: 1.3b1
     """
 
+class MonitorMemoryPeriod(FloatSettingMixin, Setting):
+    name = 'memory_monitor_period'
+
+    environment_key = 'GEVENT_MONITOR_MEMORY_PERIOD'
+    default = 5
+
+    desc = """\
+    If `monitor_thread` is enabled, this is approximately how long
+    (in seconds) we will go between checking the processes memory usage.
+
+    Checking the memory usage is relatively expensive on some operating
+    systems, so this should not be too low. gevent will place a floor
+    value on it.
+    """
+
+class MonitorMemoryMaxUsage(ByteCountSettingMixin, Setting):
+    name = 'max_memory_usage'
+
+    environment_key = 'GEVENT_MONITOR_MEMORY_MAX'
+    default = None
+
+    desc = """\
+    If `monitor_thread` is enabled,
+    then if memory usage exceeds this amount (in bytes), events will
+    be emitted. See `gevent.events`.
+
+    There is no default value for this setting. If you wish to
+    cap memory usage, you must choose a value.
+    """
+
 # The ares settings are all interpreted by
 # gevent/resolver/ares.pyx, so we don't do
 # any validation here.

src/gevent/_monitor.py

@@ -14,23 +14,14 @@ from gevent.monkey import get_original
 from gevent.util import format_run_info
 from gevent.events import notify
 from gevent.events import EventLoopBlocked
+from gevent.events import MemoryUsageThresholdExceeded
+from gevent.events import MemoryUsageUnderThreshold
 
 from gevent._compat import thread_mod_name
+from gevent._compat import perf_counter
 from gevent._util import gmctime
 
 
-# Clocks
-try:
-    # Python 3.3+ (PEP 418)
-    from time import perf_counter
-except ImportError:
-    import time
-
-    if sys.platform == "win32":
-        perf_counter = time.clock
-    else:
-        perf_counter = time.time
-
 __all__ = [
     'PeriodicMonitoringThread',
 ]
@@ -39,6 +30,24 @@ get_thread_ident = get_original(thread_mod_name, 'get_ident')
 start_new_thread = get_original(thread_mod_name, 'start_new_thread')
 thread_sleep = get_original('time', 'sleep')
 
+try:
+    # The standard library 'resource' module doesn't provide
+    # a standard way to get the RSS measure, only the maximum.
+    # You might be tempted to try to compute something by adding
+    # together text and data sizes, but on many systems those come back
+    # zero. So our only option is psutil.
+    from psutil import Process, AccessDenied
+    # Make sure it works (why would we be denied access to our own process?)
+    try:
+        Process().memory_full_info()
+    except AccessDenied: # pragma: no cover
+        Process = None
+except ImportError:
+    pass
+
+class MonitorWarning(RuntimeWarning):
+    """The type of warnings we emit."""
+
 class _MonitorEntry(object):
 
     __slots__ = ('function', 'period', 'last_run_time')
@@ -76,6 +85,10 @@ class PeriodicMonitoringThread(object):
     # what particular monitoring functions want to say.
     min_sleep_time = 0.005
 
+    # The minimum period in seconds at which we will check memory usage.
+    # Getting memory usage is fairly expensive.
+    min_memory_monitor_period = 2
+
     # A list of _MonitorEntry objects: [(function(hub), period, last_run_time))]
     # The first entry is always our entry for self.monitor_blocking
     _monitoring_functions = None
@@ -83,6 +96,11 @@ class PeriodicMonitoringThread(object):
     # The calculated min sleep time for the monitoring functions list.
     _calculated_sleep_time = None
 
+    # A boolean value that also happens to capture the
+    # memory usage at the time we exceeded the threshold. Reset
+    # to 0 when we go back below.
+    _memory_exceeded = 0
+
     def __init__(self, hub):
         self._hub_wref = wref(hub, self._on_hub_gc)
         self.should_run = True
@@ -288,6 +306,52 @@ class PeriodicMonitoringThread(object):
     def monitor_current_greenlet_blocking(self):
         self._active_greenlet = getcurrent()
 
+    def can_monitor_memory_usage(self):
+        return Process is not None
+
+    def install_monitor_memory_usage(self):
+        # Start monitoring memory usage, if possible.
+        # If not possible, emit a warning.
+        if not self.can_monitor_memory_usage():
+            import warnings
+            warnings.warn("Unable to monitor memory usage. Install psutil.",
+                          MonitorWarning)
+            return
+
+        self.add_monitoring_function(self.monitor_memory_usage,
+                                     max(GEVENT_CONFIG.memory_monitor_period,
+                                         self.min_memory_monitor_period))
+
+    def monitor_memory_usage(self, _hub):
+        max_allowed = GEVENT_CONFIG.max_memory_usage
+        if not max_allowed:
+            # They disabled it.
+            return -1 # value for tests
+
+        rusage = Process().memory_full_info()
+        # uss only documented available on Windows, Linux, and OS X.
+        # If not available, fall back to rss as an aproximation.
+        mem_usage = getattr(rusage, 'uss', 0) or rusage.rss
+
+        event = None # Return value for tests
+
+        if mem_usage > max_allowed:
+            if mem_usage > self._memory_exceeded:
+                # We're still growing
+                event = MemoryUsageThresholdExceeded(
+                    mem_usage, max_allowed, rusage)
+                notify(event)
+            self._memory_exceeded = mem_usage
+        else:
+            # we're below. Were we above it last time?
+            if self._memory_exceeded:
+                event = MemoryUsageUnderThreshold(
+                    mem_usage, max_allowed, rusage, self._memory_exceeded)
+                notify(event)
+            self._memory_exceeded = 0
+
+        return event
+
     def __repr__(self):
         return '<%s at %s in thread %s greenlet %r for %r>' % (
             self.__class__.__name__,

src/gevent/events.py

@@ -30,6 +30,10 @@ __all__ = [
     'subscribers',
     'IEventLoopBlocked',
     'EventLoopBlocked',
+    'IMemoryUsageThresholdExceeded',
+    'MemoryUsageThresholdExceeded',
+    'IMemoryUsageUnderThreshold',
+    'MemoryUsageUnderThreshold',
 ]
 
 try:
@@ -94,3 +98,70 @@ class EventLoopBlocked(object):
         self.greenlet = greenlet
         self.blocking_time = blocking_time
         self.info = info
+
+class IMemoryUsageThresholdExceeded(Interface):
+    """
+    The event emitted when the memory usage threshold is exceeded.
+
+    This event is emitted only while memory continues to grow
+    above the threshold. Only if the condition or stabilized is corrected (memory
+    usage drops) will the event be emitted in the future.
+
+    This event is emitted in the monitor thread.
+    """
+
+    mem_usage = Attribute("The current process memory usage, in bytes.")
+    max_allowed = Attribute("The maximum allowed memory usage, in bytes.")
+    memory_info = Attribute("The tuple of memory usage stats return by psutil.")
+
+class _AbstractMemoryEvent(object):
+
+    def __init__(self, mem_usage, max_allowed, memory_info):
+        self.mem_usage = mem_usage
+        self.max_allowed = max_allowed
+        self.memory_info = memory_info
+
+    def __repr__(self):
+        return "<%s used=%d max=%d details=%r>" % (
+            self.__class__.__name__,
+            self.mem_usage,
+            self.max_allowed,
+            self.memory_info,
+        )
+
+@implementer(IMemoryUsageThresholdExceeded)
+class MemoryUsageThresholdExceeded(_AbstractMemoryEvent):
+    """
+    Implementation of `IMemoryUsageThresholdExceeded`.
+    """
+
+
+class IMemoryUsageUnderThreshold(Interface):
+    """
+    The event emitted when the memory usage drops below the
+    threshold after having previously been above it.
+
+    This event is emitted only the first time memory usage is detected
+    to be below the threshold after having previously been above it.
+    If memory usage climbs again, a `IMemoryUsageThresholdExceeded`
+    event will be broadcast, and then this event could be broadcast again.
+
+    This event is emitted in the monitor thread.
+    """
+
+    mem_usage = Attribute("The current process memory usage, in bytes.")
+    max_allowed = Attribute("The maximum allowed memory usage, in bytes.")
+    max_memory_usage = Attribute("The memory usage that caused the previous "
+                                 "IMemoryUsageThresholdExceeded event.")
+    memory_info = Attribute("The tuple of memory usage stats return by psutil.")
+
+
+@implementer(IMemoryUsageUnderThreshold)
+class MemoryUsageUnderThreshold(_AbstractMemoryEvent):
+    """
+    Implementation of `IMemoryUsageUnderThreshold`.
+    """
+
+    def __init__(self, mem_usage, max_allowed, memory_info, max_usage):
+        super(MemoryUsageUnderThreshold, self).__init__(mem_usage, max_allowed, memory_info)
+        self.max_memory_usage = max_usage

src/gevent/hub.py

@@ -61,7 +61,7 @@ class _Threadlocal(get_original(thread_mod_name, '_local')):
 _threadlocal = _Threadlocal()
 
 get_thread_ident = get_original(thread_mod_name, 'get_ident')
-MAIN_THREAD = get_thread_ident() # XXX: Assuming import is done on the main thread.
+MAIN_THREAD_IDENT = get_thread_ident() # XXX: Assuming import is done on the main thread.
 
 
 class LoopExit(Exception):
@@ -507,7 +507,7 @@ class Hub(TrackedRawGreenlet):
             # loop. See #237 and #238.
             self.loop = _threadlocal.loop
         else:
-            if default is None and self.thread_ident != MAIN_THREAD:
+            if default is None and self.thread_ident != MAIN_THREAD_IDENT:
                 default = False
 
             if loop is None:
@@ -537,7 +537,7 @@ class Hub(TrackedRawGreenlet):
 
         .. versionadded:: 1.3b1
         """
-        return self.thread_ident == MAIN_THREAD
+        return self.thread_ident == MAIN_THREAD_IDENT
 
 
     def __repr__(self):
@@ -742,9 +742,6 @@ class Hub(TrackedRawGreenlet):
 
     def start_periodic_monitoring_thread(self):
         if self.periodic_monitoring_thread is None and GEVENT_CONFIG.monitor_thread:
-            # TODO: If we're the main thread, then add the memory monitoring
-            # function.
-
             # Note that it is possible for one real thread to
             # (temporarily) wind up with multiple monitoring threads,
             # if hubs are started and stopped within the thread. This shows up
@@ -752,6 +749,10 @@ class Hub(TrackedRawGreenlet):
             # hub object is gone.
             from gevent._monitor import PeriodicMonitoringThread
             self.periodic_monitoring_thread = PeriodicMonitoringThread(self)
+
+            if self.main_hub:
+                self.periodic_monitoring_thread.install_monitor_memory_usage()
+
         return self.periodic_monitoring_thread
 
     def join(self, timeout=None):

src/greentest/greentest/skipping.py

@@ -39,6 +39,7 @@ skipOnPyPy = _do_not_skip
 skipOnPyPyOnCI = _do_not_skip
 skipOnPyPy3OnCI = _do_not_skip
 skipOnPyPy3 = _do_not_skip
+skipOnPyPyOnWindows = _do_not_skip
 
 skipOnPurePython = unittest.skip if sysinfo.PURE_PYTHON else _do_not_skip
 skipWithCExtensions = unittest.skip if not sysinfo.PURE_PYTHON else _do_not_skip
@@ -75,6 +76,9 @@ if sysinfo.PYPY:
     if sysinfo.RUNNING_ON_CI:
         skipOnPyPyOnCI = unittest.skip
 
+    if sysinfo.WIN:
+        skipOnPyPyOnWindows = unittest.skip
+
     if sysinfo.PYPY3:
         skipOnPyPy3 = unittest.skip
         if sysinfo.RUNNING_ON_CI:

src/greentest/test___monitor.py

@@ -11,7 +11,10 @@ from gevent.monkey import get_original
 from gevent._compat import thread_mod_name
 from gevent._compat import NativeStrIO
 
+from greentest.skipping import skipOnPyPyOnWindows
+
 from gevent import _monitor as monitor
+from gevent import config as GEVENT_CONFIG
 
 get_ident = get_original(thread_mod_name, 'get_ident')
 
@@ -29,15 +32,20 @@ class MockHub(object):
     def handle_error(self, *args): # pylint:disable=unused-argument
         raise # pylint:disable=misplaced-bare-raise
 
-class TestPeriodicMonitoringThread(unittest.TestCase):
+class _AbstractTestPeriodicMonitoringThread(object):
+
+    # pylint:disable=no-member
 
     def setUp(self):
+        super(_AbstractTestPeriodicMonitoringThread, self).setUp()
         self._orig_start_new_thread = monitor.start_new_thread
         self._orig_thread_sleep = monitor.thread_sleep
         monitor.thread_sleep = lambda _s: gc.collect() # For PyPy
         monitor.start_new_thread = lambda _f, _a: 0xDEADBEEF
         self.hub = MockHub()
         self.pmt = monitor.PeriodicMonitoringThread(self.hub)
+        self.pmt_default_funcs = self.pmt.monitoring_functions()[:]
+        self.len_pmt_default_funcs = len(self.pmt_default_funcs)
 
     def tearDown(self):
         monitor.start_new_thread = self._orig_start_new_thread
@@ -46,6 +54,11 @@ class TestPeriodicMonitoringThread(unittest.TestCase):
         self.pmt.kill()
         assert gettrace() is prev, (gettrace(), prev)
         settrace(None)
+        super(_AbstractTestPeriodicMonitoringThread, self).tearDown()
+
+
+class TestPeriodicMonitoringThread(_AbstractTestPeriodicMonitoringThread,
+                                   unittest.TestCase):
 
     def test_constructor(self):
         self.assertEqual(0xDEADBEEF, self.pmt.monitor_thread_ident)
@@ -62,42 +75,6 @@ class TestPeriodicMonitoringThread(unittest.TestCase):
         self.assertFalse(self.pmt.should_run)
         self.assertIsNone(gettrace())
 
-    def test_previous_trace(self):
-        self.pmt.kill()
-        self.assertIsNone(gettrace())
-
-        called = []
-        def f(*args):
-            called.append(args)
-
-        settrace(f)
-
-        self.pmt = monitor.PeriodicMonitoringThread(self.hub)
-        self.assertEqual(gettrace(), self.pmt.greenlet_trace)
-        self.assertIs(self.pmt.previous_trace_function, f)
-
-        self.pmt.greenlet_trace('event', 'args')
-
-        self.assertEqual([('event', 'args')], called)
-
-    def test_greenlet_trace(self):
-        self.assertEqual(0, self.pmt._greenlet_switch_counter)
-        # Unknown event still counts as a switch (should it?)
-        self.pmt.greenlet_trace('unknown', None)
-        self.assertEqual(1, self.pmt._greenlet_switch_counter)
-        self.assertIsNone(self.pmt._active_greenlet)
-
-        origin = object()
-        target = object()
-
-        self.pmt.greenlet_trace('switch', (origin, target))
-        self.assertEqual(2, self.pmt._greenlet_switch_counter)
-        self.assertIs(target, self.pmt._active_greenlet)
-
-        # Unknown event removes active greenlet
-        self.pmt.greenlet_trace('unknown', self)
-        self.assertEqual(3, self.pmt._greenlet_switch_counter)
-        self.assertIsNone(self.pmt._active_greenlet)
 
     def test_add_monitoring_function(self):
 
@@ -109,17 +86,17 @@ class TestPeriodicMonitoringThread(unittest.TestCase):
 
         # Add
         self.pmt.add_monitoring_function(f, 1)
-        self.assertEqual(2, len(self.pmt.monitoring_functions()))
+        self.assertEqual(self.len_pmt_default_funcs + 1, len(self.pmt.monitoring_functions()))
         self.assertEqual(1, self.pmt.monitoring_functions()[1].period)
 
         # Update
         self.pmt.add_monitoring_function(f, 2)
-        self.assertEqual(2, len(self.pmt.monitoring_functions()))
+        self.assertEqual(self.len_pmt_default_funcs + 1, len(self.pmt.monitoring_functions()))
         self.assertEqual(2, self.pmt.monitoring_functions()[1].period)
 
         # Remove
         self.pmt.add_monitoring_function(f, None)
-        self.assertEqual(1, len(self.pmt.monitoring_functions()))
+        self.assertEqual(self.len_pmt_default_funcs, len(self.pmt.monitoring_functions()))
 
     def test_calculate_sleep_time(self):
         self.assertEqual(
@@ -146,6 +123,86 @@ class TestPeriodicMonitoringThread(unittest.TestCase):
             self.pmt.monitoring_functions()[0].period,
             self.pmt._calculated_sleep_time)
 
+    def test_call_destroyed_hub(self):
+        # Add a function that destroys the hub so we break out (eventually)
+        # This clears the wref, which eventually calls kill()
+        def f(_hub):
+            _hub = None
+            self.hub = None
+            gc.collect()
+
+        self.pmt.add_monitoring_function(f, 0.1)
+        self.pmt()
+        self.assertFalse(self.pmt.should_run)
+
+    def test_call_dead_hub(self):
+        # Add a function that makes the hub go false (e.g., it quit)
+        # This causes the function to kill itself.
+        def f(hub):
+            hub.dead = True
+        self.pmt.add_monitoring_function(f, 0.1)
+        self.pmt()
+        self.assertFalse(self.pmt.should_run)
+
+    def test_call_SystemExit(self):
+        # breaks the loop
+        def f(_hub):
+            raise SystemExit()
+
+        self.pmt.add_monitoring_function(f, 0.1)
+        self.pmt()
+
+    def test_call_other_error(self):
+        class MyException(Exception):
+            pass
+
+        def f(_hub):
+            raise MyException()
+
+        self.pmt.add_monitoring_function(f, 0.1)
+        with self.assertRaises(MyException):
+            self.pmt()
+
+
+class TestPeriodicMonitorBlocking(_AbstractTestPeriodicMonitoringThread,
+                                  unittest.TestCase):
+
+    def test_previous_trace(self):
+        self.pmt.kill()
+        self.assertIsNone(gettrace())
+
+        called = []
+        def f(*args):
+            called.append(args)
+
+        settrace(f)
+
+        self.pmt = monitor.PeriodicMonitoringThread(self.hub)
+        self.assertEqual(gettrace(), self.pmt.greenlet_trace)
+        self.assertIs(self.pmt.previous_trace_function, f)
+
+        self.pmt.greenlet_trace('event', 'args')
+
+        self.assertEqual([('event', 'args')], called)
+
+    def test_greenlet_trace(self):
+        self.assertEqual(0, self.pmt._greenlet_switch_counter)
+        # Unknown event still counts as a switch (should it?)
+        self.pmt.greenlet_trace('unknown', None)
+        self.assertEqual(1, self.pmt._greenlet_switch_counter)
+        self.assertIsNone(self.pmt._active_greenlet)
+
+        origin = object()
+        target = object()
+
+        self.pmt.greenlet_trace('switch', (origin, target))
+        self.assertEqual(2, self.pmt._greenlet_switch_counter)
+        self.assertIs(target, self.pmt._active_greenlet)
+
+        # Unknown event removes active greenlet
+        self.pmt.greenlet_trace('unknown', self)
+        self.assertEqual(3, self.pmt._greenlet_switch_counter)
+        self.assertIsNone(self.pmt._active_greenlet)
 
     def test_monitor_blocking(self):
         # Initially there's no active greenlet and no switches,
@@ -186,45 +243,110 @@ class TestPeriodicMonitoringThread(unittest.TestCase):
         self.hub.thread_ident = -1
         self.assertTrue(self.pmt.monitor_blocking(self.hub))
 
-    def test_call_destroyed_hub(self):
-        # Add a function that destroys the hub so we break out (eventually)
-        # This clears the wref, which eventually calls kill()
-        def f(_hub):
-            _hub = None
-            self.hub = None
-            gc.collect()
 
-        self.pmt.add_monitoring_function(f, 0.1)
-        self.pmt()
-        self.assertFalse(self.pmt.should_run)
+class MockProcess(object):
 
-    def test_call_dead_hub(self):
-        # Add a function that makes the hub go false (e.g., it quit)
-        # This causes the function to kill itself.
-        def f(hub):
-            hub.dead = True
-        self.pmt.add_monitoring_function(f, 0.1)
-        self.pmt()
-        self.assertFalse(self.pmt.should_run)
+    def __init__(self, rss):
+        self.rss = rss
 
-    def test_call_SystemExit(self):
-        # breaks the loop
-        def f(_hub):
-            raise SystemExit()
+    def memory_full_info(self):
+        return self
 
-        self.pmt.add_monitoring_function(f, 0.1)
-        self.pmt()
 
-    def test_call_other_error(self):
-        class MyException(Exception):
-            pass
+@skipOnPyPyOnWindows("psutil doesn't install on PyPy on Win")
+class TestPeriodicMonitorMemory(_AbstractTestPeriodicMonitoringThread,
+                                unittest.TestCase):
 
-        def f(_hub):
-            raise MyException()
+    rss = 0
 
-        self.pmt.add_monitoring_function(f, 0.1)
-        with self.assertRaises(MyException):
-            self.pmt()
+    def setUp(self):
+        super(TestPeriodicMonitorMemory, self).setUp()
+        self._old_max = GEVENT_CONFIG.max_memory_usage
+        GEVENT_CONFIG.max_memory_usage = None
+
+        self._old_process = monitor.Process
+        monitor.Process = lambda: MockProcess(self.rss)
+
+    def tearDown(self):
+        GEVENT_CONFIG.max_memory_usage = self._old_max
+        monitor.Process = self._old_process
+        super(TestPeriodicMonitorMemory, self).tearDown()
+
+    def test_can_monitor_and_install(self):
+
+        # We run tests with psutil installed, and we have access to our
+        # process.
+        self.assertTrue(self.pmt.can_monitor_memory_usage())
+        # No warning, adds a function
+
+        self.pmt.install_monitor_memory_usage()
+        self.assertEqual(self.len_pmt_default_funcs + 1, len(self.pmt.monitoring_functions()))
+
+    def test_cannot_monitor_and_install(self):
+        import warnings
+        monitor.Process = None
+        self.assertFalse(self.pmt.can_monitor_memory_usage())
+
+        # This emits a warning, visible by default
+        with warnings.catch_warnings(record=True) as ws:
+            self.pmt.install_monitor_memory_usage()
+
+        self.assertEqual(1, len(ws))
+        self.assertIs(monitor.MonitorWarning, ws[0].category)
+
+    def test_monitor_no_allowed(self):
+        self.assertEqual(-1, self.pmt.monitor_memory_usage(None))
+
+    def test_monitor_greater(self):
+        from gevent import events
+
+        self.rss = 2
+        GEVENT_CONFIG.max_memory_usage = 1
+
+        # Initial event
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsInstance(event, events.MemoryUsageThresholdExceeded)
+        self.assertEqual(2, event.mem_usage)
+        self.assertEqual(1, event.max_allowed)
+        self.assertIsInstance(event.memory_info, MockProcess)
+
+        # No growth, no event
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsNone(event)
+
+        # Growth, event
+        self.rss = 3
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsInstance(event, events.MemoryUsageThresholdExceeded)
+        self.assertEqual(3, event.mem_usage)
+
+        # Shrinking below gets us back
+        self.rss = 1
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsInstance(event, events.MemoryUsageUnderThreshold)
+        self.assertEqual(1, event.mem_usage)
+
+        # coverage
+        repr(event)
+
+        # No change, no event
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsNone(event)
+
+        # Growth, event
+        self.rss = 3
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsInstance(event, events.MemoryUsageThresholdExceeded)
+        self.assertEqual(3, event.mem_usage)
+
+
+    def test_monitor_initial_below(self):
+        self.rss = 1
+        GEVENT_CONFIG.max_memory_usage = 10
+
+
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsNone(event)
 
 if __name__ == '__main__':
     unittest.main()

src/greentest/test__event.py

@@ -4,12 +4,14 @@ import gevent
 from gevent.event import Event, AsyncResult
 
 import greentest
-from greentest.skipping import skipUnderCoverage
+
 from greentest.six import xrange
 from greentest.timing import AbstractGenericGetTestCase
 from greentest.timing import AbstractGenericWaitTestCase
+from greentest.timing import SMALL_TICK
+from greentest.timing import SMALL_TICK_MAX_ADJ
 
-DELAY = 0.01
+DELAY = SMALL_TICK + SMALL_TICK_MAX_ADJ
 
 
 class TestEventWait(AbstractGenericWaitTestCase):
@@ -105,21 +107,20 @@ class TestAsyncResult(greentest.TestCase):
         gevent.sleep(0)
         self.assertEqual(log, [('caught', obj)])
 
-    @skipUnderCoverage("This test is racy and sometimes fails")
     def test_set(self):
         event1 = AsyncResult()
         timer_exc = MyException('interrupted')
 
-        # Notice that this test is racy
+        # Notice that this test is racy:
+        # After DELAY, we set the event. We also try to immediately
+        # raise the exception with a timer of 0 --- but that depends
+        # on cycling the loop. Hence the fairly large value for DELAY.
         g = gevent.spawn_later(DELAY, event1.set, 'hello event1')
-        t = gevent.Timeout.start_new(0, timer_exc)
-        try:
+        self._close_on_teardown(g.kill)
+        with gevent.Timeout.start_new(0, timer_exc):
             with self.assertRaises(MyException) as exc:
                 event1.get()
-            self.assertEqual(timer_exc, exc.exception)
-        finally:
-            t.close()
-            g.kill()
+            self.assertIs(timer_exc, exc.exception)
 
     def test_set_with_timeout(self):
         event2 = AsyncResult()

src/greentest/test__events.py

@@ -15,6 +15,19 @@ class TestImplements(unittest.TestCase):
     def test_event_loop_blocked(self):
         verify.verifyClass(events.IEventLoopBlocked, events.EventLoopBlocked)
 
+    def test_mem_threshold(self):
+        verify.verifyClass(events.IMemoryUsageThresholdExceeded,
+                           events.MemoryUsageThresholdExceeded)
+        verify.verifyObject(events.IMemoryUsageThresholdExceeded,
+                            events.MemoryUsageThresholdExceeded(0, 0, 0))
+
+    def test_mem_decreased(self):
+        verify.verifyClass(events.IMemoryUsageUnderThreshold,
+                           events.MemoryUsageUnderThreshold)
+        verify.verifyObject(events.IMemoryUsageUnderThreshold,
+                            events.MemoryUsageUnderThreshold(0, 0, 0, 0))
+
+
 class TestEvents(unittest.TestCase):
 
     def test_is_zope(self):

src/greentest/test__hub.py

@@ -199,11 +199,11 @@ class TestPeriodicMonitoringThread(greentest.TestCase):
         monitor = hub.start_periodic_monitoring_thread()
         self.assertIsNotNone(monitor)
 
-        self.assertEqual(1, len(monitor.monitoring_functions()))
-        monitor.add_monitoring_function(self._monitor, 0.1)
         self.assertEqual(2, len(monitor.monitoring_functions()))
-        self.assertEqual(self._monitor, monitor.monitoring_functions()[1].function)
-        self.assertEqual(0.1, monitor.monitoring_functions()[1].period)
+        monitor.add_monitoring_function(self._monitor, 0.1)
+        self.assertEqual(3, len(monitor.monitoring_functions()))
+        self.assertEqual(self._monitor, monitor.monitoring_functions()[-1].function)
+        self.assertEqual(0.1, monitor.monitoring_functions()[-1].period)
 
         # We must make sure we have switched greenlets at least once,
         # otherwise we can't detect a failure.
@@ -214,7 +214,7 @@ class TestPeriodicMonitoringThread(greentest.TestCase):
             self._run_monitoring_threads(monitor)
         finally:
             monitor.add_monitoring_function(self._monitor, None)
-            self.assertEqual(1, len(monitor._monitoring_functions))
+            self.assertEqual(2, len(monitor._monitoring_functions))
             assert hub.exception_stream is stream
             monitor.kill()
             del hub.exception_stream