More warning cleanup.

examples/processes.py

@@ -23,3 +23,6 @@ else:
         print('ls: %r' % p2.stdout.read())
     else:
         print('ls: job is still running')
+
+    p1.stdout.close()
+    p2.stdout.close()

src/gevent/_compat.py

@@ -46,6 +46,18 @@ else:
     reraise = reraise # export
     exc_clear = sys.exc_clear
 
+## import locks
+try:
+    # In Python 3.4 and newer in CPython and PyPy3,
+    # imp.acquire_lock and imp.release_lock are delegated to
+    # '_imp'. (Which is also used by importlib.) 'imp' itself is
+    # deprecated. Avoid that warning.
+    import _imp as imp
+except ImportError:
+    import imp
+imp_acquire_lock = imp.acquire_lock
+imp_release_lock = imp.release_lock
+
 ## Functions
 if PY3:
     iteritems = dict.items

src/gevent/_patcher.py

@@ -9,12 +9,14 @@
 
 from __future__ import absolute_import, print_function
 
-import imp
 import importlib
 import sys
 
 from gevent._compat import PY3
 from gevent._compat import iteritems
+from gevent._compat import imp_acquire_lock
+from gevent._compat import imp_release_lock
+
 
 from gevent.builtins import __import__ as _import
 
@@ -92,10 +94,10 @@ class _SysModulesPatcher(object):
         try:
             self._restore()
         finally:
-            imp.release_lock()
+            imp_release_lock()
 
     def __enter__(self):
-        imp.acquire_lock()
+        imp_acquire_lock()
         self._save()
         self._replace()
 

src/gevent/builtins.py

@@ -2,10 +2,13 @@
 """gevent friendly implementations of builtin functions."""
 from __future__ import absolute_import
 
-import imp # deprecated since 3.4; issues PendingDeprecationWarning in 3.5
 import sys
 import weakref
+
 from gevent.lock import RLock
+from gevent._compat import imp_acquire_lock
+from gevent._compat import imp_release_lock
+
 
 # Normally we'd have the "expected" case inside the try
 # (Python 3, because Python 3 is the way forward). But
@@ -86,7 +89,7 @@ def __import__(*args, **kwargs):
         return _import(*args, **kwargs)
 
     module_lock = __module_lock(args[0]) # Get a lock for the module name
-    imp.acquire_lock()
+    imp_acquire_lock()
     try:
         module_lock.acquire()
         try:
@@ -94,7 +97,7 @@ def __import__(*args, **kwargs):
         finally:
             module_lock.release()
     finally:
-        imp.release_lock()
+        imp_release_lock()
     return result
 
 

src/greentest/greentest/testrunner.py

@@ -15,6 +15,7 @@ from greentest.util import log
 from greentest.sysinfo import RUNNING_ON_CI
 from greentest.sysinfo import PYPY
 from greentest.sysinfo import PY3
+from greentest.sysinfo import PY2
 from greentest.sysinfo import RESOLVER_ARES
 from greentest.sysinfo import LIBUV
 from greentest import six
@@ -225,7 +226,11 @@ def discover(tests=None, ignore_files=None,
                     continue
                 to_process.append((cmd, options))
         else:
-            cmd = [sys.executable, '-u', filename]
+            cmd = [sys.executable, '-u']
+            if PYPY and PY2:
+                # Doesn't seem to be an env var for this
+                cmd.extend(('-X', 'track-resources'))
+            cmd.append(filename)
             options = DEFAULT_RUN_OPTIONS.copy()
             options.update(TEST_FILE_OPTIONS.get(filename, {}))
             to_process.append((cmd, options))
@@ -327,6 +332,42 @@ def print_list(lst):
     for name in lst:
         log(' - %s', name)
 
+def _setup_environ():
+    if 'PYTHONWARNINGS' not in os.environ and not sys.warnoptions:
+        # Enable default warnings such as ResourceWarning.
+        # On Python 3[.6], the system site.py module has
+        # "open(fullname, 'rU')" which produces the warning that
+        # 'U' is deprecated, so ignore warnings from site.py
+
+        # importlib/_bootstrap.py likes to spit out "ImportWarning:
+        # can't resolve package from __spec__ or __package__, falling
+        # back on __name__ and __path__". I have no idea what that means, but it seems harmless
+        # and is annoying.
+        os.environ['PYTHONWARNINGS'] = 'default,ignore:::site:,ignore:::importlib._bootstrap:,ignore:::importlib._bootstrap_external:'
+
+    if 'PYTHONFAULTHANDLER' not in os.environ:
+        os.environ['PYTHONFAULTHANDLER'] = 'true'
+
+    if 'GEVENT_DEBUG' not in os.environ:
+        os.environ['GEVENT_DEBUG'] = 'debug'
+
+    if 'PYTHONTRACEMALLOC' not in os.environ:
+        os.environ['PYTHONTRACEMALLOC'] = '10'
+
+    if 'PYTHONDEVMODE' not in os.environ:
+        # Python 3.7
+        os.environ['PYTHONDEVMODE'] = '1'
+
+    if 'PYTHONMALLOC' not in os.environ:
+        # Python 3.6
+        os.environ['PYTHONMALLOC'] = 'debug'
+
+    if (sys.version_info == (3, 7, 0, 'beta', 2)
+            and (os.environ.get("PYTHONDEVMODE") or os.environ.get('PYTHONMALLOC'))):
+        # See https://twitter.com/ossmkitty/status/970693025130311680
+        # https://bugs.python.org/issue33005
+        os.environ.pop('PYTHONDEVMODE', None)
+        os.environ.pop('PYTHONMALLOC', None)
 
 def main():
     import argparse
@@ -357,32 +398,7 @@ def main():
         os.environ['COVERAGE_FILE'] = os.path.abspath(".") + os.sep + ".coverage"
         print("Enabling coverage to", os.environ['COVERAGE_FILE'])
 
-
-    if 'PYTHONWARNINGS' not in os.environ and not sys.warnoptions:
-        # Enable default warnings such as ResourceWarning.
-        # On Python 3[.6], the system site.py module has
-        # "open(fullname, 'rU')" which produces the warning that
-        # 'U' is deprecated, so ignore warnings from site.py
-
-        # importlib/_bootstrap.py likes to spit out "ImportWarning:
-        # can't resolve package from __spec__ or __package__, falling
-        # back on __name__ and __path__". I have no idea what that means, but it seems harmless
-        # and is annoying.
-        os.environ['PYTHONWARNINGS'] = 'default,ignore:::site:,ignore:::importlib._bootstrap:,ignore:::importlib._bootstrap_external:'
-
-    if 'PYTHONFAULTHANDLER' not in os.environ:
-        os.environ['PYTHONFAULTHANDLER'] = 'true'
-
-    if 'GEVENT_DEBUG' not in os.environ:
-        os.environ['GEVENT_DEBUG'] = 'debug'
-
-    if 'PYTHONTRACEMALLOC' not in os.environ:
-        os.environ['PYTHONTRACEMALLOC'] = '10'
-
-    if 'PYTHONDEVMODE' not in os.environ:
-        # Python 3.7
-        os.environ['PYTHONDEVMODE'] = '1'
-
+    _setup_environ()
 
     if options.config:
         config = {}

src/greentest/greentest/util.py

@@ -170,7 +170,27 @@ class RunResult(object):
         return self.code
 
 
-lock = threading.Lock()
+def _should_show_warning_output(out):
+    if b'Warning' in out:
+        # Strip out some patterns we specifically do not
+        # care about.
+        # from test.support for monkey-patched tests
+        out = out.replace(b'Warning -- reap_children', b'NADA')
+        out = out.replace(b"Warning -- threading_cleanup", b'NADA')
+
+        # The below *could* be done with sophisticated enough warning
+        # filters passed to the children
+
+        # collections.abc is the new home; setuptools uses the old one,
+        # as does dnspython
+        out = out.replace(b"DeprecationWarning: Using or importing the ABCs", b'NADA')
+        # libuv poor timer resolution
+        out = out.replace(b'UserWarning: libuv only supports', b'NADA')
+        # Packages on Python 2
+        out = out.replace(b'ImportWarning: Not importing directory', b'NADA')
+    return b'Warning' in out
+
+output_lock = threading.Lock()
 
 
 def run(command, **kwargs):
@@ -195,9 +215,9 @@ def run(command, **kwargs):
     finally:
         kill(popen)
     assert not err
-    with lock: # pylint:disable=not-context-manager
+    with output_lock: # pylint:disable=not-context-manager
         failed = bool(result)
-        if out and (failed or verbose or b'ResourceWarning' in out):
+        if out and (failed or verbose or _should_show_warning_output(out)):
             out = out.strip().decode('utf-8', 'ignore')
             if out:
                 out = '  ' + out.replace('\n', '\n  ')

src/greentest/known_failures.py

@@ -241,14 +241,6 @@ if sys.version_info[:2] >= (3, 4) and APPVEYOR:
         'FLAKY test_selectors.py'
     ]
 
-if sys.version_info == (3, 7, 0, 'beta', 2) and os.environ.get("PYTHONDEVMODE"):
-    # These crash when in devmode.
-    # See https://twitter.com/ossmkitty/status/970693025130311680
-    # https://bugs.python.org/issue33005
-    FAILING_TESTS += [
-        'test__monkey_sigchld_2.py',
-        'test__monkey_sigchld_3.py'
-    ]
 
 if COVERAGE:
     # The gevent concurrency plugin tends to slow things

src/greentest/test__fileobject.py

@@ -12,6 +12,12 @@ from greentest.sysinfo import PY3
 from greentest.flaky import reraiseFlakyTestRaceConditionLibuv
 from greentest.skipping import skipOnLibuvOnCIOnPyPy
 
+try:
+    ResourceWarning
+except NameError:
+    class ResourceWarning(Warning):
+        "Python 2 fallback"
+
 
 class Test(greentest.TestCase):
 
@@ -38,8 +44,12 @@ class Test(greentest.TestCase):
             import traceback
             traceback.print_exc()
 
-        del s # Deliberately getting ResourceWarning with FileObject(Thread) under Py3
-        gc.collect() # PyPy
+        import warnings
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', ResourceWarning)
+            # Deliberately getting ResourceWarning with FileObject(Thread) under Py3
+            del s
+            gc.collect() # PyPy
 
         if kwargs.get("close", True):
             with self.assertRaises((OSError, IOError)):
@@ -71,11 +81,16 @@ class Test(greentest.TestCase):
                 self._test_del(close=False)
 
     def test_newlines(self):
+        import warnings
         r, w = os.pipe()
         lines = [b'line1\n', b'line2\r', b'line3\r\n', b'line4\r\nline5', b'\nline6']
         g = gevent.spawn(writer, FileObject(w, 'wb'), lines)
+
         try:
-            fobj = FileObject(r, 'rU')
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                # U is deprecated in Python 3, shows up on FileObjectThread
+                fobj = FileObject(r, 'rU')
             result = fobj.read()
             fobj.close()
             self.assertEqual('line1\nline2\nline3\nline4\nline5\nline6', result)

src/greentest/test__monkey_sigchld_3.py

@@ -45,7 +45,8 @@ if hasattr(signal, 'SIGCHLD'):
         popen.stderr.read()
         popen.stdout.read()
         popen.wait() # This hangs if it doesn't.
-
+        popen.stderr.close()
+        popen.stdout.close()
         sys.exit(0)
 else:
     print("No SIGCHLD, not testing")

src/greentest/test__subprocess.py

@@ -27,10 +27,7 @@ python_universal_newlines = hasattr(sys.stdout, 'newlines')
 # The stdlib of Python 3 on Windows doesn't properly handle universal newlines
 # (it produces broken results compared to Python 2)
 # See gevent.subprocess for more details.
-if PY3 and subprocess.mswindows:
-    python_universal_newlines_broken = True
-else:
-    python_universal_newlines_broken = False
+python_universal_newlines_broken = PY3 and subprocess.mswindows
 
 
 class Test(greentest.TestCase):
@@ -61,6 +58,7 @@ class Test(greentest.TestCase):
         p = subprocess.Popen([sys.executable, "-c", "print()"],
                              stdout=subprocess.PIPE)
         p.wait()
+        p.stdout.close()
         del p
         if PYPY:
             gc.collect()
@@ -206,7 +204,7 @@ class Test(greentest.TestCase):
                 os.close(w)
 
     def test_issue148(self):
-        for i in range(7):
+        for _ in range(7):
             try:
                 subprocess.Popen('this_name_must_not_exist')
             except OSError as ex:

src/greentest/test_threading_2.py

@@ -372,20 +372,24 @@ class ThreadTests(unittest.TestCase):
         # Try hard to trigger #1703448: a thread is still returned in
         # threading.enumerate() after it has been join()ed.
         enum = threading.enumerate
-        old_interval = sys.getcheckinterval()
-        try:
-            for i in xrange(1, 100):
-                # Try a couple times at each thread-switching interval
-                # to get more interleavings.
-                sys.setcheckinterval(i // 5)
-                t = threading.Thread(target=lambda: None)
-                t.start()
-                t.join()
-                l = enum()
-                self.assertFalse(t in l,
-                                 "#1703448 triggered after %d trials: %s" % (i, l))
-        finally:
-            sys.setcheckinterval(old_interval)
+        import warnings
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            # get/set checkinterval are deprecated in Python 3
+            old_interval = sys.getcheckinterval()
+            try:
+                for i in xrange(1, 100):
+                    # Try a couple times at each thread-switching interval
+                    # to get more interleavings.
+                    sys.setcheckinterval(i // 5)
+                    t = threading.Thread(target=lambda: None)
+                    t.start()
+                    t.join()
+                    l = enum()
+                    self.assertFalse(t in l,
+                                     "#1703448 triggered after %d trials: %s" % (i, l))
+            finally:
+                sys.setcheckinterval(old_interval)
 
     if not hasattr(sys, 'pypy_version_info'):
         def test_no_refcycle_through_target(self):
@@ -436,6 +440,7 @@ class ThreadJoinOnShutdown(unittest.TestCase):
         p = subprocess.Popen([sys.executable, "-W", "ignore", "-c", script], stdout=subprocess.PIPE)
         rc = p.wait()
         data = p.stdout.read().replace(b'\r', b'')
+        p.stdout.close()
         self.assertEqual(data, b"end of main\nend of thread\n")
         self.assertNotEqual(rc, 2, b"interpreter was blocked")
         self.assertEqual(rc, 0, b"Unexpected error")