Merge pull request #1388 from gevent/issue1365

Tests do reasonable things when some of the test deps are missing

setup.py

@@ -363,6 +363,10 @@ def run_setup(ext_modules, run_make):
                 'repoze.sphinx.autointerface',
             ],
             'test': [
+                # To the extent possible, we should work to make sure
+                # our tests run, at least a basic set, without any of
+                # these extra dependencies (i.e., skip things when they are
+                # missing). This helps serve as a smoketest for users.
                 'zope.interface',
                 'zope.event',
 

src/gevent/_compat.py

@@ -165,3 +165,22 @@ except ImportError:
         perf_counter = time.clock
     else:
         perf_counter = time.time
+
+
+## Monitoring
+def get_this_psutil_process():
+    # Depends on psutil. Defer the import until needed, who knows what
+    # it imports (psutil imports subprocess which on Python 3 imports
+    # selectors. This can expose issues with monkey-patching.)
+    # Returns a freshly queried object each time.
+    try:
+        from psutil import Process, AccessDenied
+        # Make sure it works (why would we be denied access to our own process?)
+        try:
+            proc = Process()
+            proc.memory_full_info()
+        except AccessDenied: # pragma: no cover
+            proc = None
+    except ImportError:
+        proc = None
+    return proc

src/gevent/_monitor.py

@@ -20,6 +20,7 @@ from gevent.events import implementer
 from gevent._tracer import GreenletTracer
 from gevent._compat import thread_mod_name
 from gevent._compat import perf_counter
+from gevent._compat import get_this_psutil_process
 
 
 
@@ -251,22 +252,7 @@ class PeriodicMonitoringThread(object):
         self._greenlet_tracer.monitor_current_greenlet_blocking()
 
     def _get_process(self): # pylint:disable=method-hidden
-        try:
-            # The standard library 'resource' module doesn't provide
-            # a standard way to get the RSS measure, only the maximum.
-            # You might be tempted to try to compute something by adding
-            # together text and data sizes, but on many systems those come back
-            # zero. So our only option is psutil.
-            from psutil import Process, AccessDenied
-            # Make sure it works (why would we be denied access to our own process?)
-            try:
-                proc = Process()
-                proc.memory_full_info()
-            except AccessDenied: # pragma: no cover
-                proc = None
-        except ImportError:
-            proc = None
-
+        proc = get_this_psutil_process()
         self._get_process = lambda: proc
         return proc
 

src/gevent/testing/__init__.py

@@ -128,9 +128,35 @@ def gc_collect_if_needed():
     if PYPY: # pragma: no cover
         gc.collect()
 
+# Our usage of mock should be limited to '@mock.patch()'
+# and other things that are easily...mocked...here on Python 2
+# when mock is not installed.
 try:
     from unittest import mock
 except ImportError: # Python 2
-    import mock
+    try:
+        import mock
+    except ImportError: # pragma: no cover
+        # Backport not installed
+        class mock(object):
+
+            @staticmethod
+            def patch(reason):
+                return unittest.skip(reason)
 
 mock = mock
+
+
+# zope.interface
+try:
+    from zope.interface import verify
+except ImportError:
+    class verify(object):
+
+        @staticmethod
+        def verifyObject(*_):
+            import warnings
+            warnings.warn("zope.interface is not installed; not verifying")
+            return
+
+verify = verify

src/gevent/testing/leakcheck.py

@@ -25,7 +25,11 @@ import types
 from functools import wraps
 import unittest
 
-import objgraph
+try:
+    import objgraph
+except ImportError: # pragma: no cover
+    # Optional test dependency
+    objgraph = None
 
 import gevent
 import gevent.core
@@ -193,6 +197,11 @@ class _RefCountChecker(object):
 
 
 def wrap_refcount(method):
+    if objgraph is None:
+        import warnings
+        warnings.warn("objgraph not available, leakchecks disabled")
+        return method
+
     if getattr(method, 'ignore_leakcheck', False):
         return method
 

src/gevent/testing/skipping.py

@@ -19,6 +19,7 @@
 # THE SOFTWARE.
 from __future__ import absolute_import, print_function, division
 
+import functools
 import unittest
 
 from . import sysinfo
@@ -94,7 +95,60 @@ skipUnderCoverage = unittest.skip if sysinfo.RUN_COVERAGE else _do_not_skip
 skipIf = unittest.skipIf
 skipUnless = unittest.skipUnless
 
-
+_has_psutil_process = None
+def _check_psutil():
+    global _has_psutil_process
+    if _has_psutil_process is None:
+        _has_psutil_process = sysinfo.get_this_psutil_process() is not None
+    return _has_psutil_process
+
+
+def skipWithoutPSUtil(reason):
+    # Important: If you use this on classes, you must not use the
+    # two-argument form of super()
+    reason = "psutil not available: " + reason
+    def decorator(test_item):
+        # Defer the check until runtime to avoid imports
+        if not isinstance(test_item, type):
+            f = test_item
+            @functools.wraps(test_item)
+            def skip_wrapper(*args):
+                if not _check_psutil():
+                    raise unittest.SkipTest(reason)
+                return f(*args)
+            test_item = skip_wrapper
+        else:
+            # given a class, subclass its setUp method to do the same.
+
+            # The trouble with this is that the decorator automatically
+            # rebinds to the same name, and if there are two-argument calls
+            # like `super(MyClass, self).thing` in the class, we get infinite
+            # recursion (because MyClass has been rebound to the object we return.)
+            # This is easy to fix on Python 3: use the zero argument `super()`, because
+            # the lookup relies not on names but implicit slots.
+            #
+            # I didn't find a good workaround for this on Python 2, so
+            # I'm just forbidding using the two argument super.
+            base = test_item
+            class SkipWrapper(base):
+                def setUp(self):
+                    if not _check_psutil():
+                        raise unittest.SkipTest(reason)
+                    base.setUp(self)
+
+                def _super(self):
+                    return super(base, self)
+            SkipWrapper.__name__ = test_item.__name__
+            SkipWrapper.__module__ = test_item.__module__
+            try:
+                SkipWrapper.__qualname__ = test_item.__qualname__
+            except AttributeError:
+                # Python 2
+                pass
+            test_item = SkipWrapper
+
+        return test_item
+    return decorator
 
 if sysinfo.LIBUV:
     skipOnLibuv = unittest.skip

src/gevent/testing/sysinfo.py

@@ -33,6 +33,8 @@ OSX = gsysinfo.OSX
 
 PURE_PYTHON = gsysinfo.PURE_PYTHON
 
+get_this_psutil_process = gsysinfo.get_this_psutil_process
+
 # XXX: Formalize this better
 LIBUV = 'libuv' in gevent.core.loop.__module__ # pylint:disable=no-member
 CFFI_BACKEND = PYPY or LIBUV or 'cffi' in os.getenv('GEVENT_LOOP', '')

src/gevent/tests/test___monitor.py

@@ -11,7 +11,8 @@ from gevent.monkey import get_original
 from gevent._compat import thread_mod_name
 from gevent._compat import NativeStrIO
 
-from gevent.testing.skipping import skipOnPyPyOnWindows
+from gevent.testing import verify
+from gevent.testing.skipping import skipWithoutPSUtil
 
 from gevent import _monitor as monitor
 from gevent import config as GEVENT_CONFIG
@@ -82,10 +83,11 @@ class TestPeriodicMonitoringThread(_AbstractTestPeriodicMonitoringThread,
         self.assertEqual(0xDEADBEEF, self.pmt.monitor_thread_ident)
         self.assertEqual(gettrace(), self.pmt._greenlet_tracer)
 
-    @skipOnPyPyOnWindows("psutil doesn't install on PyPy on Win")
+    @skipWithoutPSUtil("Verifies the process")
     def test_get_process(self):
         proc = self.pmt._get_process()
         self.assertIsNotNone(proc)
+        # Same object is returned each time.
         self.assertIs(proc, self.pmt._get_process())
 
     def test_hub_wref(self):
@@ -245,7 +247,6 @@ class TestPeriodicMonitorBlocking(_AbstractTestPeriodicMonitoringThread,
         # so nothing is considered blocked
         from gevent.events import subscribers
         from gevent.events import IEventLoopBlocked
-        from zope.interface.verify import verifyObject
         events = []
         subscribers.append(events.append)
 
@@ -263,7 +264,7 @@ class TestPeriodicMonitorBlocking(_AbstractTestPeriodicMonitoringThread,
         # Again without switching is a problem.
         self.assertTrue(self.pmt.monitor_blocking(self.hub))
         self.assertTrue(events)
-        verifyObject(IEventLoopBlocked, events[0])
+        verify.verifyObject(IEventLoopBlocked, events[0])
         del events[:]
 
         # But we can order it not to be a problem
@@ -289,14 +290,14 @@ class MockProcess(object):
         return self
 
 
-@skipOnPyPyOnWindows("psutil doesn't install on PyPy on Win")
+@skipWithoutPSUtil("Accessess memory info")
 class TestPeriodicMonitorMemory(_AbstractTestPeriodicMonitoringThread,
                                 unittest.TestCase):
 
     rss = 0
 
     def setUp(self):
-        super(TestPeriodicMonitorMemory, self).setUp()
+        _AbstractTestPeriodicMonitoringThread.setUp(self)
         self._old_max = GEVENT_CONFIG.max_memory_usage
         GEVENT_CONFIG.max_memory_usage = None
 
@@ -304,10 +305,9 @@ class TestPeriodicMonitorMemory(_AbstractTestPeriodicMonitoringThread,
 
     def tearDown(self):
         GEVENT_CONFIG.max_memory_usage = self._old_max
-        super(TestPeriodicMonitorMemory, self).tearDown()
+        _AbstractTestPeriodicMonitoringThread.tearDown(self)
 
     def test_can_monitor_and_install(self):
-
         # We run tests with psutil installed, and we have access to our
         # process.
         self.assertTrue(self.pmt.can_monitor_memory_usage())

src/gevent/tests/test__events.py

@@ -8,8 +8,19 @@ from __future__ import print_function
 import unittest
 
 from gevent import events
-from zope.interface import verify
 
+try:
+    from zope.interface import verify
+except ImportError:
+    verify = None
+
+try:
+    from zope import event
+except ImportError:
+    event = None
+
+
+@unittest.skipIf(verify is None, "Needs zope.interface")
 class TestImplements(unittest.TestCase):
 
     def test_event_loop_blocked(self):
@@ -28,10 +39,10 @@ class TestImplements(unittest.TestCase):
                             events.MemoryUsageUnderThreshold(0, 0, 0, 0))
 
 
+@unittest.skipIf(event is None, "Needs zope.event")
 class TestEvents(unittest.TestCase):
 
     def test_is_zope(self):
-        from zope import event
         self.assertIs(events.subscribers, event.subscribers)
         self.assertIs(events.notify, event.notify)
 

src/gevent/tests/test__hub.py

@@ -30,6 +30,7 @@ import gevent
 from gevent import socket
 from gevent.hub import Waiter, get_hub
 from gevent._compat import NativeStrIO
+from gevent._compat import get_this_psutil_process
 
 DELAY = 0.1
 
@@ -204,9 +205,16 @@ class TestPeriodicMonitoringThread(greentest.TestCase):
         monitor = hub.start_periodic_monitoring_thread()
         self.assertIsNotNone(monitor)
 
-        self.assertEqual(2, len(monitor.monitoring_functions()))
+        basic_monitor_func_count = 1
+        if get_this_psutil_process() is not None:
+            # psutil is installed
+            basic_monitor_func_count += 1
+
+        self.assertEqual(basic_monitor_func_count,
+                         len(monitor.monitoring_functions()))
         monitor.add_monitoring_function(self._monitor, 0.1)
-        self.assertEqual(3, len(monitor.monitoring_functions()))
+        self.assertEqual(basic_monitor_func_count + 1,
+                         len(monitor.monitoring_functions()))
         self.assertEqual(self._monitor, monitor.monitoring_functions()[-1].function)
         self.assertEqual(0.1, monitor.monitoring_functions()[-1].period)
 
@@ -219,7 +227,8 @@ class TestPeriodicMonitoringThread(greentest.TestCase):
             self._run_monitoring_threads(monitor)
         finally:
             monitor.add_monitoring_function(self._monitor, None)
-            self.assertEqual(2, len(monitor._monitoring_functions))
+            self.assertEqual(basic_monitor_func_count,
+                             len(monitor._monitoring_functions))
             assert hub.exception_stream is stream
             monitor.kill()
             del hub.exception_stream
@@ -320,7 +329,7 @@ class TestPeriodicMonitoringThread(greentest.TestCase):
 class TestLoopInterface(unittest.TestCase):
 
     def test_implemensts_ILoop(self):
-        from zope.interface import verify
+        from gevent.testing import verify
         from gevent._interfaces import ILoop
 
         loop = get_hub().loop

src/gevent/tests/test__monkey.py

@@ -70,7 +70,7 @@ class TestMonkey(SubscriberCleanupMixin, unittest.TestCase):
 
     def test_patch_twice_warnings_events(self):
         import warnings
-        from zope.interface import verify
+        from gevent.testing import verify
 
         orig_saved = {}
         for k, v in monkey.saved.items():