Add tests for memory monitor.

CHANGES.rst

@@ -4,31 +4,33 @@
 
 .. currentmodule:: gevent
 
-1.3a3 (unreleased)
+1.3b1 (unreleased)
 ==================
 
-- Use strongly typed watcher callbacks in the libuv CFFI extensions.
-  This prevents dozens of compiler warnings.
+Dependencies
+------------
+
+- Cython 0.28.1 is now used to build gevent from a source checkout.
+
+Bug Fixes
+---------
 
 - On Python 2, when monkey-patching `threading.Event`, also
   monkey-patch the underlying class, ``threading._Event``. Some code
   may be type-checking for that. See :issue:`1136`.
 
-- Introduce the configuration variable
-  `gevent.config.track_greenlet_tree` (aka
-  ``GEVENT_TRACK_GREENLET_TREE``) to allow disabling the greenlet tree
-  features for applications where greenlet spawning is performance
-  critical. This restores spawning performance to 1.2 levels.
+- Fix libuv io watchers polling for events that only stopped watchers
+  are interested in, reducing CPU usage. Reported in :issue:`1144` by
+  wwqgtxx.
+
+Enhancements
+------------
 
 - Add additional optimizations for spawning greenlets, making it
   faster than 1.3a2.
 
-- Add an optional monitoring thread for each hub. When enabled, this
-  thread (by default) looks for greenlets that block the event loop
-  for more than 0.1s. You can add your own periodic monitoring
-  functions to this thread. Set ``GEVENT_MONITOR_THREAD_ENABLE`` to
-  use it, and ``GEVENT_MAX_BLOCKING_TIME`` to configure the blocking
-  interval.
+- Use strongly typed watcher callbacks in the libuv CFFI extensions.
+  This prevents dozens of compiler warnings.
 
 - When gevent prints a timestamp as part of an error message, it is
   now in UTC format as specified by RFC3339.
@@ -40,9 +42,21 @@
 - Hub objects now include the value of their ``name`` attribute in
   their repr.
 
-- Fix libuv io watchers polling for events that only stopped watchers
-  are interested in, reducing CPU usage. Reported in :issue:`1144` by
-  wwqgtxx.
+Monitoring and Debugging
+------------------------
+
+- Introduce the configuration variable
+  `gevent.config.track_greenlet_tree` (aka
+  ``GEVENT_TRACK_GREENLET_TREE``) to allow disabling the greenlet tree
+  features for applications where greenlet spawning is performance
+  critical. This restores spawning performance to 1.2 levels.
+
+- Add an optional monitoring thread for each hub. When enabled, this
+  thread (by default) looks for greenlets that block the event loop
+  for more than 0.1s. You can add your own periodic monitoring
+  functions to this thread. Set ``GEVENT_MONITOR_THREAD_ENABLE`` to
+  use it, and ``GEVENT_MAX_BLOCKING_TIME`` to configure the blocking
+  interval.
 
 - Add a simple event framework for decoupled communication. It uses
   :mod:`zope.event` if that is installed. The monitoring thread emits
@@ -52,6 +66,8 @@
 - Add settings for monitoring memory usage and emitting events when a
   threshold is exceeded and then corrected. gevent currently supplies
   no policy for what to do when memory exceeds the configured limit.
+  ``psutil`` must be installed to use this. See :pr:`1150`.
+
 
 1.3a2 (2018-03-06)
 ==================

ci-requirements.txt

@@ -5,7 +5,7 @@ wheel
 # 0.28 is faster, and (important!) lets us specify the target module
 # name to be created so that we can have both foo.py and _foo.so
 # at the same time.
-Cython >= 0.28
+Cython >= 0.28.1
 
 # Python 3.7b1 requires this.
 greenlet>=0.4.13 ; platform_python_implementation == "CPython"

rtd-requirements.txt

-cython >= 0.28
+cython >= 0.28.1
 repoze.sphinx.autointerface

src/gevent/__init__.py

@@ -18,11 +18,16 @@ _version_info = namedtuple('version_info',
 
 #: The programatic version identifier. The fields have (roughly) the
 #: same meaning as :data:`sys.version_info`
-#: Deprecated in 1.2.
+#: .. deprecated:: 1.2
+#:  Use ``pkg_resources.parse_version(__version__)`` (or the equivalent
+#:  ``packaging.version.Version(__version__)``).
 version_info = _version_info(1, 3, 0, 'dev', 0)
 
-#: The human-readable PEP 440 version identifier
-__version__ = '1.3a3.dev0'
+#: The human-readable PEP 440 version identifier.
+#: Use ``pkg_resources.parse_version(__version__)`` or
+#: ``packaging.version.Version(__version__)`` to get a machine-usable
+#: value.
+__version__ = '1.3b1.dev0'
 
 
 __all__ = [

src/gevent/_compat.py

@@ -140,3 +140,17 @@ except ImportError:
             # exist. Probably on Python 2 with an astral character.
             # Not sure how to handle this.
             raise UnicodeEncodeError("Can't encode path to filesystem encoding")
+
+
+## Clocks
+try:
+    # Python 3.3+ (PEP 418)
+    from time import perf_counter
+    perf_counter = perf_counter
+except ImportError:
+    import time
+
+    if sys.platform == "win32":
+        perf_counter = time.clock
+    else:
+        perf_counter = time.time

src/gevent/_config.py

@@ -305,8 +305,8 @@ class ByteCountSettingMixin(_PositiveValueMixin):
     _SUFFIX_SIZE = 2
 
     def _convert(self, value):
-        if not value:
-            return
+        if not value or not isinstance(value, str):
+            return value
         value = value.lower()
         for s, m in self._MULTIPLES.items():
             if value[-self._SUFFIX_SIZE:] == s:

src/gevent/_monitor.py

@@ -18,21 +18,10 @@ from gevent.events import MemoryUsageThresholdExceeded
 from gevent.events import MemoryUsageUnderThreshold
 
 from gevent._compat import thread_mod_name
+from gevent._compat import perf_counter
 from gevent._util import gmctime
 
 
-# Clocks
-try:
-    # Python 3.3+ (PEP 418)
-    from time import perf_counter
-except ImportError:
-    import time
-
-    if sys.platform == "win32":
-        perf_counter = time.clock
-    else:
-        perf_counter = time.time
-
 __all__ = [
     'PeriodicMonitoringThread',
 ]
@@ -51,7 +40,7 @@ try:
     # Make sure it works (why would we be denied access to our own process?)
     try:
         Process().memory_full_info()
-    except AccessDenied:
+    except AccessDenied: # pragma: no cover
         Process = None
 except ImportError:
     pass
@@ -323,7 +312,7 @@ class PeriodicMonitoringThread(object):
     def install_monitor_memory_usage(self):
         # Start monitoring memory usage, if possible.
         # If not possible, emit a warning.
-        if not self.can_monitor_memory_usage:
+        if not self.can_monitor_memory_usage():
             import warnings
             warnings.warn("Unable to monitor memory usage. Install psutil.",
                           MonitorWarning)
@@ -337,26 +326,32 @@ class PeriodicMonitoringThread(object):
         max_allowed = GEVENT_CONFIG.max_memory_usage
         if not max_allowed:
             # They disabled it.
-            return
+            return -1 # value for tests
 
         rusage = Process().memory_full_info()
         # uss only documented available on Windows, Linux, and OS X.
         # If not available, fall back to rss as an aproximation.
         mem_usage = getattr(rusage, 'uss', 0) or rusage.rss
 
+        event = None # Return value for tests
+
         if mem_usage > max_allowed:
             if mem_usage > self._memory_exceeded:
                 # We're still growing
-                notify(MemoryUsageThresholdExceeded(
-                    mem_usage, max_allowed, rusage))
+                event = MemoryUsageThresholdExceeded(
+                    mem_usage, max_allowed, rusage)
+                notify(event)
             self._memory_exceeded = mem_usage
         else:
             # we're below. Were we above it last time?
             if self._memory_exceeded:
-                notify(MemoryUsageUnderThreshold(
-                    mem_usage, max_allowed, rusage, self._memory_exceeded))
+                event = MemoryUsageUnderThreshold(
+                    mem_usage, max_allowed, rusage, self._memory_exceeded)
+                notify(event)
             self._memory_exceeded = 0
 
+        return event
+
     def __repr__(self):
         return '<%s at %s in thread %s greenlet %r for %r>' % (
             self.__class__.__name__,

src/greentest/greentest/skipping.py

@@ -39,6 +39,7 @@ skipOnPyPy = _do_not_skip
 skipOnPyPyOnCI = _do_not_skip
 skipOnPyPy3OnCI = _do_not_skip
 skipOnPyPy3 = _do_not_skip
+skipOnPyPyOnWindows = _do_not_skip
 
 skipOnPurePython = unittest.skip if sysinfo.PURE_PYTHON else _do_not_skip
 skipWithCExtensions = unittest.skip if not sysinfo.PURE_PYTHON else _do_not_skip
@@ -75,6 +76,9 @@ if sysinfo.PYPY:
     if sysinfo.RUNNING_ON_CI:
         skipOnPyPyOnCI = unittest.skip
 
+    if sysinfo.WIN:
+        skipOnPyPyOnWindows = unittest.skip
+
     if sysinfo.PYPY3:
         skipOnPyPy3 = unittest.skip
         if sysinfo.RUNNING_ON_CI:

src/greentest/test___monitor.py

@@ -11,7 +11,10 @@ from gevent.monkey import get_original
 from gevent._compat import thread_mod_name
 from gevent._compat import NativeStrIO
 
+from greentest.skipping import skipOnPyPyOnWindows
+
 from gevent import _monitor as monitor
+from gevent import config as GEVENT_CONFIG
 
 get_ident = get_original(thread_mod_name, 'get_ident')
 
@@ -29,15 +32,20 @@ class MockHub(object):
     def handle_error(self, *args): # pylint:disable=unused-argument
         raise # pylint:disable=misplaced-bare-raise
 
-class TestPeriodicMonitoringThread(unittest.TestCase):
+class _AbstractTestPeriodicMonitoringThread(object):
+
+    # pylint:disable=no-member
 
     def setUp(self):
+        super(_AbstractTestPeriodicMonitoringThread, self).setUp()
         self._orig_start_new_thread = monitor.start_new_thread
         self._orig_thread_sleep = monitor.thread_sleep
         monitor.thread_sleep = lambda _s: gc.collect() # For PyPy
         monitor.start_new_thread = lambda _f, _a: 0xDEADBEEF
         self.hub = MockHub()
         self.pmt = monitor.PeriodicMonitoringThread(self.hub)
+        self.pmt_default_funcs = self.pmt.monitoring_functions()[:]
+        self.len_pmt_default_funcs = len(self.pmt_default_funcs)
 
     def tearDown(self):
         monitor.start_new_thread = self._orig_start_new_thread
@@ -46,6 +54,11 @@ class TestPeriodicMonitoringThread(unittest.TestCase):
         self.pmt.kill()
         assert gettrace() is prev, (gettrace(), prev)
         settrace(None)
+        super(_AbstractTestPeriodicMonitoringThread, self).tearDown()
+
+
+class TestPeriodicMonitoringThread(_AbstractTestPeriodicMonitoringThread,
+                                   unittest.TestCase):
 
     def test_constructor(self):
         self.assertEqual(0xDEADBEEF, self.pmt.monitor_thread_ident)
@@ -62,42 +75,6 @@ class TestPeriodicMonitoringThread(unittest.TestCase):
         self.assertFalse(self.pmt.should_run)
         self.assertIsNone(gettrace())
 
-    def test_previous_trace(self):
-        self.pmt.kill()
-        self.assertIsNone(gettrace())
-
-        called = []
-        def f(*args):
-            called.append(args)
-
-        settrace(f)
-
-        self.pmt = monitor.PeriodicMonitoringThread(self.hub)
-        self.assertEqual(gettrace(), self.pmt.greenlet_trace)
-        self.assertIs(self.pmt.previous_trace_function, f)
-
-        self.pmt.greenlet_trace('event', 'args')
-
-        self.assertEqual([('event', 'args')], called)
-
-    def test_greenlet_trace(self):
-        self.assertEqual(0, self.pmt._greenlet_switch_counter)
-        # Unknown event still counts as a switch (should it?)
-        self.pmt.greenlet_trace('unknown', None)
-        self.assertEqual(1, self.pmt._greenlet_switch_counter)
-        self.assertIsNone(self.pmt._active_greenlet)
-
-        origin = object()
-        target = object()
-
-        self.pmt.greenlet_trace('switch', (origin, target))
-        self.assertEqual(2, self.pmt._greenlet_switch_counter)
-        self.assertIs(target, self.pmt._active_greenlet)
-
-        # Unknown event removes active greenlet
-        self.pmt.greenlet_trace('unknown', self)
-        self.assertEqual(3, self.pmt._greenlet_switch_counter)
-        self.assertIsNone(self.pmt._active_greenlet)
 
     def test_add_monitoring_function(self):
 
@@ -109,17 +86,17 @@ class TestPeriodicMonitoringThread(unittest.TestCase):
 
         # Add
         self.pmt.add_monitoring_function(f, 1)
-        self.assertEqual(2, len(self.pmt.monitoring_functions()))
+        self.assertEqual(self.len_pmt_default_funcs + 1, len(self.pmt.monitoring_functions()))
         self.assertEqual(1, self.pmt.monitoring_functions()[1].period)
 
         # Update
         self.pmt.add_monitoring_function(f, 2)
-        self.assertEqual(2, len(self.pmt.monitoring_functions()))
+        self.assertEqual(self.len_pmt_default_funcs + 1, len(self.pmt.monitoring_functions()))
         self.assertEqual(2, self.pmt.monitoring_functions()[1].period)
 
         # Remove
         self.pmt.add_monitoring_function(f, None)
-        self.assertEqual(1, len(self.pmt.monitoring_functions()))
+        self.assertEqual(self.len_pmt_default_funcs, len(self.pmt.monitoring_functions()))
 
     def test_calculate_sleep_time(self):
         self.assertEqual(
@@ -146,6 +123,86 @@ class TestPeriodicMonitoringThread(unittest.TestCase):
             self.pmt.monitoring_functions()[0].period,
             self.pmt._calculated_sleep_time)
 
+    def test_call_destroyed_hub(self):
+        # Add a function that destroys the hub so we break out (eventually)
+        # This clears the wref, which eventually calls kill()
+        def f(_hub):
+            _hub = None
+            self.hub = None
+            gc.collect()
+
+        self.pmt.add_monitoring_function(f, 0.1)
+        self.pmt()
+        self.assertFalse(self.pmt.should_run)
+
+    def test_call_dead_hub(self):
+        # Add a function that makes the hub go false (e.g., it quit)
+        # This causes the function to kill itself.
+        def f(hub):
+            hub.dead = True
+        self.pmt.add_monitoring_function(f, 0.1)
+        self.pmt()
+        self.assertFalse(self.pmt.should_run)
+
+    def test_call_SystemExit(self):
+        # breaks the loop
+        def f(_hub):
+            raise SystemExit()
+
+        self.pmt.add_monitoring_function(f, 0.1)
+        self.pmt()
+
+    def test_call_other_error(self):
+        class MyException(Exception):
+            pass
+
+        def f(_hub):
+            raise MyException()
+
+        self.pmt.add_monitoring_function(f, 0.1)
+        with self.assertRaises(MyException):
+            self.pmt()
+
+
+class TestPeriodicMonitorBlocking(_AbstractTestPeriodicMonitoringThread,
+                                  unittest.TestCase):
+
+    def test_previous_trace(self):
+        self.pmt.kill()
+        self.assertIsNone(gettrace())
+
+        called = []
+        def f(*args):
+            called.append(args)
+
+        settrace(f)
+
+        self.pmt = monitor.PeriodicMonitoringThread(self.hub)
+        self.assertEqual(gettrace(), self.pmt.greenlet_trace)
+        self.assertIs(self.pmt.previous_trace_function, f)
+
+        self.pmt.greenlet_trace('event', 'args')
+
+        self.assertEqual([('event', 'args')], called)
+
+    def test_greenlet_trace(self):
+        self.assertEqual(0, self.pmt._greenlet_switch_counter)
+        # Unknown event still counts as a switch (should it?)
+        self.pmt.greenlet_trace('unknown', None)
+        self.assertEqual(1, self.pmt._greenlet_switch_counter)
+        self.assertIsNone(self.pmt._active_greenlet)
+
+        origin = object()
+        target = object()
+
+        self.pmt.greenlet_trace('switch', (origin, target))
+        self.assertEqual(2, self.pmt._greenlet_switch_counter)
+        self.assertIs(target, self.pmt._active_greenlet)
+
+        # Unknown event removes active greenlet
+        self.pmt.greenlet_trace('unknown', self)
+        self.assertEqual(3, self.pmt._greenlet_switch_counter)
+        self.assertIsNone(self.pmt._active_greenlet)
 
     def test_monitor_blocking(self):
         # Initially there's no active greenlet and no switches,
@@ -186,45 +243,110 @@ class TestPeriodicMonitoringThread(unittest.TestCase):
         self.hub.thread_ident = -1
         self.assertTrue(self.pmt.monitor_blocking(self.hub))
 
-    def test_call_destroyed_hub(self):
-        # Add a function that destroys the hub so we break out (eventually)
-        # This clears the wref, which eventually calls kill()
-        def f(_hub):
-            _hub = None
-            self.hub = None
-            gc.collect()
 
-        self.pmt.add_monitoring_function(f, 0.1)
-        self.pmt()
-        self.assertFalse(self.pmt.should_run)
+class MockProcess(object):
 
-    def test_call_dead_hub(self):
-        # Add a function that makes the hub go false (e.g., it quit)
-        # This causes the function to kill itself.
-        def f(hub):
-            hub.dead = True
-        self.pmt.add_monitoring_function(f, 0.1)
-        self.pmt()
-        self.assertFalse(self.pmt.should_run)
+    def __init__(self, rss):
+        self.rss = rss
 
-    def test_call_SystemExit(self):
-        # breaks the loop
-        def f(_hub):
-            raise SystemExit()
+    def memory_full_info(self):
+        return self
 
-        self.pmt.add_monitoring_function(f, 0.1)
-        self.pmt()
 
-    def test_call_other_error(self):
-        class MyException(Exception):
-            pass
+@skipOnPyPyOnWindows("psutil doesn't install on PyPy on Win")
+class TestPeriodicMonitorMemory(_AbstractTestPeriodicMonitoringThread,
+                                unittest.TestCase):
 
-        def f(_hub):
-            raise MyException()
+    rss = 0
 
-        self.pmt.add_monitoring_function(f, 0.1)
-        with self.assertRaises(MyException):
-            self.pmt()
+    def setUp(self):
+        super(TestPeriodicMonitorMemory, self).setUp()
+        self._old_max = GEVENT_CONFIG.max_memory_usage
+        GEVENT_CONFIG.max_memory_usage = None
+
+        self._old_process = monitor.Process
+        monitor.Process = lambda: MockProcess(self.rss)
+
+    def tearDown(self):
+        GEVENT_CONFIG.max_memory_usage = self._old_max
+        monitor.Process = self._old_process
+        super(TestPeriodicMonitorMemory, self).tearDown()
+
+    def test_can_monitor_and_install(self):
+
+        # We run tests with psutil installed, and we have access to our
+        # process.
+        self.assertTrue(self.pmt.can_monitor_memory_usage())
+        # No warning, adds a function
+
+        self.pmt.install_monitor_memory_usage()
+        self.assertEqual(self.len_pmt_default_funcs + 1, len(self.pmt.monitoring_functions()))
+
+    def test_cannot_monitor_and_install(self):
+        import warnings
+        monitor.Process = None
+        self.assertFalse(self.pmt.can_monitor_memory_usage())
+
+        # This emits a warning, visible by default
+        with warnings.catch_warnings(record=True) as ws:
+            self.pmt.install_monitor_memory_usage()
+
+        self.assertEqual(1, len(ws))
+        self.assertIs(monitor.MonitorWarning, ws[0].category)
+
+    def test_monitor_no_allowed(self):
+        self.assertEqual(-1, self.pmt.monitor_memory_usage(None))
+
+    def test_monitor_greater(self):
+        from gevent import events
+
+        self.rss = 2
+        GEVENT_CONFIG.max_memory_usage = 1
+
+        # Initial event
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsInstance(event, events.MemoryUsageThresholdExceeded)
+        self.assertEqual(2, event.mem_usage)
+        self.assertEqual(1, event.max_allowed)
+        self.assertIsInstance(event.memory_info, MockProcess)
+
+        # No growth, no event
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsNone(event)
+
+        # Growth, event
+        self.rss = 3
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsInstance(event, events.MemoryUsageThresholdExceeded)
+        self.assertEqual(3, event.mem_usage)
+
+        # Shrinking below gets us back
+        self.rss = 1
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsInstance(event, events.MemoryUsageUnderThreshold)
+        self.assertEqual(1, event.mem_usage)
+
+        # coverage
+        repr(event)
+
+        # No change, no event
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsNone(event)
+
+        # Growth, event
+        self.rss = 3
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsInstance(event, events.MemoryUsageThresholdExceeded)
+        self.assertEqual(3, event.mem_usage)
+
+
+    def test_monitor_initial_below(self):
+        self.rss = 1
+        GEVENT_CONFIG.max_memory_usage = 10
+
+
+        event = self.pmt.monitor_memory_usage(None)
+        self.assertIsNone(event)
 
 if __name__ == '__main__':
     unittest.main()