Commit 0a56f56f authored by Jason Madden's avatar Jason Madden

Add tests for memory monitor.

parent 7cef8181
......@@ -4,31 +4,33 @@
.. currentmodule:: gevent
1.3a3 (unreleased)
1.3b1 (unreleased)
==================
- Use strongly typed watcher callbacks in the libuv CFFI extensions.
This prevents dozens of compiler warnings.
Dependencies
------------
- Cython 0.28.1 is now used to build gevent from a source checkout.
Bug Fixes
---------
- On Python 2, when monkey-patching `threading.Event`, also
monkey-patch the underlying class, ``threading._Event``. Some code
may be type-checking for that. See :issue:`1136`.
- Introduce the configuration variable
`gevent.config.track_greenlet_tree` (aka
``GEVENT_TRACK_GREENLET_TREE``) to allow disabling the greenlet tree
features for applications where greenlet spawning is performance
critical. This restores spawning performance to 1.2 levels.
- Fix libuv io watchers polling for events that only stopped watchers
are interested in, reducing CPU usage. Reported in :issue:`1144` by
wwqgtxx.
Enhancements
------------
- Add additional optimizations for spawning greenlets, making it
faster than 1.3a2.
- Add an optional monitoring thread for each hub. When enabled, this
thread (by default) looks for greenlets that block the event loop
for more than 0.1s. You can add your own periodic monitoring
functions to this thread. Set ``GEVENT_MONITOR_THREAD_ENABLE`` to
use it, and ``GEVENT_MAX_BLOCKING_TIME`` to configure the blocking
interval.
- Use strongly typed watcher callbacks in the libuv CFFI extensions.
This prevents dozens of compiler warnings.
- When gevent prints a timestamp as part of an error message, it is
now in UTC format as specified by RFC3339.
......@@ -40,9 +42,21 @@
- Hub objects now include the value of their ``name`` attribute in
their repr.
- Fix libuv io watchers polling for events that only stopped watchers
are interested in, reducing CPU usage. Reported in :issue:`1144` by
wwqgtxx.
Monitoring and Debugging
------------------------
- Introduce the configuration variable
`gevent.config.track_greenlet_tree` (aka
``GEVENT_TRACK_GREENLET_TREE``) to allow disabling the greenlet tree
features for applications where greenlet spawning is performance
critical. This restores spawning performance to 1.2 levels.
- Add an optional monitoring thread for each hub. When enabled, this
thread (by default) looks for greenlets that block the event loop
for more than 0.1s. You can add your own periodic monitoring
functions to this thread. Set ``GEVENT_MONITOR_THREAD_ENABLE`` to
use it, and ``GEVENT_MAX_BLOCKING_TIME`` to configure the blocking
interval.
- Add a simple event framework for decoupled communication. It uses
:mod:`zope.event` if that is installed. The monitoring thread emits
......@@ -52,6 +66,8 @@
- Add settings for monitoring memory usage and emitting events when a
threshold is exceeded and then corrected. gevent currently supplies
no policy for what to do when memory exceeds the configured limit.
``psutil`` must be installed to use this. See :pr:`1150`.
1.3a2 (2018-03-06)
==================
......
......@@ -5,7 +5,7 @@ wheel
# 0.28 is faster, and (important!) lets us specify the target module
# name to be created so that we can have both foo.py and _foo.so
# at the same time.
Cython >= 0.28
Cython >= 0.28.1
# Python 3.7b1 requires this.
greenlet>=0.4.13 ; platform_python_implementation == "CPython"
......
......@@ -18,11 +18,16 @@ _version_info = namedtuple('version_info',
#: The programatic version identifier. The fields have (roughly) the
#: same meaning as :data:`sys.version_info`
#: Deprecated in 1.2.
#: .. deprecated:: 1.2
#: Use ``pkg_resources.parse_version(__version__)`` (or the equivalent
#: ``packaging.version.Version(__version__)``).
version_info = _version_info(1, 3, 0, 'dev', 0)
#: The human-readable PEP 440 version identifier
__version__ = '1.3a3.dev0'
#: The human-readable PEP 440 version identifier.
#: Use ``pkg_resources.parse_version(__version__)`` or
#: ``packaging.version.Version(__version__)`` to get a machine-usable
#: value.
__version__ = '1.3b1.dev0'
__all__ = [
......
......@@ -140,3 +140,17 @@ except ImportError:
# exist. Probably on Python 2 with an astral character.
# Not sure how to handle this.
raise UnicodeEncodeError("Can't encode path to filesystem encoding")
## Clocks
try:
# Python 3.3+ (PEP 418)
from time import perf_counter
perf_counter = perf_counter
except ImportError:
import time
if sys.platform == "win32":
perf_counter = time.clock
else:
perf_counter = time.time
......@@ -305,8 +305,8 @@ class ByteCountSettingMixin(_PositiveValueMixin):
_SUFFIX_SIZE = 2
def _convert(self, value):
if not value:
return
if not value or not isinstance(value, str):
return value
value = value.lower()
for s, m in self._MULTIPLES.items():
if value[-self._SUFFIX_SIZE:] == s:
......
......@@ -18,21 +18,10 @@ from gevent.events import MemoryUsageThresholdExceeded
from gevent.events import MemoryUsageUnderThreshold
from gevent._compat import thread_mod_name
from gevent._compat import perf_counter
from gevent._util import gmctime
# Clocks
try:
# Python 3.3+ (PEP 418)
from time import perf_counter
except ImportError:
import time
if sys.platform == "win32":
perf_counter = time.clock
else:
perf_counter = time.time
__all__ = [
'PeriodicMonitoringThread',
]
......@@ -51,7 +40,7 @@ try:
# Make sure it works (why would we be denied access to our own process?)
try:
Process().memory_full_info()
except AccessDenied:
except AccessDenied: # pragma: no cover
Process = None
except ImportError:
pass
......@@ -323,7 +312,7 @@ class PeriodicMonitoringThread(object):
def install_monitor_memory_usage(self):
# Start monitoring memory usage, if possible.
# If not possible, emit a warning.
if not self.can_monitor_memory_usage:
if not self.can_monitor_memory_usage():
import warnings
warnings.warn("Unable to monitor memory usage. Install psutil.",
MonitorWarning)
......@@ -337,26 +326,32 @@ class PeriodicMonitoringThread(object):
max_allowed = GEVENT_CONFIG.max_memory_usage
if not max_allowed:
# They disabled it.
return
return -1 # value for tests
rusage = Process().memory_full_info()
# uss only documented available on Windows, Linux, and OS X.
# If not available, fall back to rss as an aproximation.
mem_usage = getattr(rusage, 'uss', 0) or rusage.rss
event = None # Return value for tests
if mem_usage > max_allowed:
if mem_usage > self._memory_exceeded:
# We're still growing
notify(MemoryUsageThresholdExceeded(
mem_usage, max_allowed, rusage))
event = MemoryUsageThresholdExceeded(
mem_usage, max_allowed, rusage)
notify(event)
self._memory_exceeded = mem_usage
else:
# we're below. Were we above it last time?
if self._memory_exceeded:
notify(MemoryUsageUnderThreshold(
mem_usage, max_allowed, rusage, self._memory_exceeded))
event = MemoryUsageUnderThreshold(
mem_usage, max_allowed, rusage, self._memory_exceeded)
notify(event)
self._memory_exceeded = 0
return event
def __repr__(self):
return '<%s at %s in thread %s greenlet %r for %r>' % (
self.__class__.__name__,
......
......@@ -39,6 +39,7 @@ skipOnPyPy = _do_not_skip
skipOnPyPyOnCI = _do_not_skip
skipOnPyPy3OnCI = _do_not_skip
skipOnPyPy3 = _do_not_skip
skipOnPyPyOnWindows = _do_not_skip
skipOnPurePython = unittest.skip if sysinfo.PURE_PYTHON else _do_not_skip
skipWithCExtensions = unittest.skip if not sysinfo.PURE_PYTHON else _do_not_skip
......@@ -75,6 +76,9 @@ if sysinfo.PYPY:
if sysinfo.RUNNING_ON_CI:
skipOnPyPyOnCI = unittest.skip
if sysinfo.WIN:
skipOnPyPyOnWindows = unittest.skip
if sysinfo.PYPY3:
skipOnPyPy3 = unittest.skip
if sysinfo.RUNNING_ON_CI:
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment