Merge pull request #1101 from gevent/libev-regen

Simplify libev default loop handling and let it regenerate itself.

CHANGES.rst

@@ -64,6 +64,11 @@
   extension implementation only) deallocated (garbage collected). See
   :issue:`1098`.
 
+- Simplify handling of the libev default loop and the ``destroy()``
+  method. The default loop, when destroyed, can again be requested and
+  it will regenerate itself. The default loop is the only one that can
+  receive child events.
+
 
 1.3a1 (2018-01-27)
 ==================

Makefile

@@ -90,7 +90,7 @@ allbackendtest:
 
 cffibackendtest:
 	@${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
-	GEVENT_LOOP=libuv GEVENTTEST_COVERAGE=1 make alltest
+	GEVENT_LOOP=libuv make alltest
 	@${PYTHON} scripts/travis.py fold_end libuv
 	@${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
 	GEVENT_LOOP=libev-cffi make alltest
@@ -172,7 +172,7 @@ develop:
 # Then start installing our deps so they can be cached. Note that use of --build-options / --global-options / --install-options
 # disables the cache.
 # We need wheel>=0.26 on Python 3.5. See previous revisions.
-	${PYTHON} -m pip install -U -r ci-requirements.txt
+	time ${PYTHON} -m pip install -U -r ci-requirements.txt
 	GEVENTSETUP_EV_VERIFY=3 time ${PYTHON} -m pip install -U -e .
 	ccache -s
 	@${PYTHON} scripts/travis.py fold_end install
@@ -193,7 +193,7 @@ test-py37: $(PY37)
 	LD_LIBRARY_PATH=$(BUILD_RUNTIMES)/versions/python3.7.0b1/openssl/lib PYTHON=python3.7.0b1 PATH=$(BUILD_RUNTIMES)/versions/python3.7.0b1/bin:$(PATH) make develop allbackendtest
 
 test-pypy: $(PYPY)
-	PYTHON=$(PYPY) PATH=$(BUILD_RUNTIMES)/versions/pypy5100/bin:$(PATH) make develop cffibackendtest coverage_combine
+	PYTHON=$(PYPY) PATH=$(BUILD_RUNTIMES)/versions/pypy5100/bin:$(PATH) make develop cffibackendtest
 
 test-pypy3: $(PYPY3)
 	PYTHON=$(PYPY3) PATH=$(BUILD_RUNTIMES)/versions/pypy3.5_5101/bin:$(PATH) make develop basictest

ci-requirements.txt

 setuptools
 wheel
-cython>=0.27.3 ; python_version < '3.7'
-# Thread state changes, especially in Pythran seem to need this,
-# which in turn requires CPython 3.7.0a3 or above
-git+https://github.com/cython/cython#egg=cython ; python_version >= '3.7'
-greenlet>=0.4.10 ; python_version < '3.7'
-# Thread state changes again; requires 0.4.13, as yet unreleased
-git+https://github.com/python-greenlet/greenlet#egg=greenlet ; python_version >= '3.7'
+# Python 3.7b1 requires at least this version.
+# 0.28 is preferred due to 3.7 specific changes.
+cython>=0.27.3
+# Python 3.7b1 requires this.
+greenlet>=0.4.13
 pylint>=1.8.0
 # pyyaml is included here and doesn't install on travis with 3.7a3
 prospector[with_pyroma] ; python_version < '3.7'
 coverage>=4.0
 coveralls>=1.0
 # See version requirements in setup.py
-cffi
+cffi ; platform_python_implementation == "CPython"
 futures
 dnspython
 idna

examples/udp_client.py

@@ -3,7 +3,7 @@
 
 Usage: python udp_client.py MESSAGE
 
-Make sure you're running a UDP server on port 9000 (see udp_server.py).
+Make sure you're running a UDP server on port 9001 (see udp_server.py).
 
 There's nothing gevent-specific here.
 """
@@ -11,7 +11,7 @@ from __future__ import print_function
 import sys
 from gevent import socket
 
-address = ('localhost', 9000)
+address = ('localhost', 9001)
 message = ' '.join(sys.argv[1:])
 sock = socket.socket(type=socket.SOCK_DGRAM)
 sock.connect(address)

src/gevent/_ffi/loop.py

@@ -487,12 +487,11 @@ class AbstractLoop(object):
         """
         if self._ptr:
             try:
-                if self._default:
-                    if not self._can_destroy_default_loop():
-                        return False
-                    type(self)._default_loop_destroyed = True
-
+                if not self._can_destroy_loop(self._ptr):
+                    return False
+                self._destroyed_loop(self._ptr)
                 self._stop_aux_watchers()
+
             finally:
                 # not ffi.NULL, we don't want something that can be
                 # passed to C and crash later. This will create nice friendly
@@ -501,8 +500,11 @@ class AbstractLoop(object):
 
             return True
 
-    def _can_destroy_default_loop(self):
-        return not type(self)._default_loop_destroyed
+    def _can_destroy_loop(self, ptr):
+        raise NotImplementedError()
+
+    def _destroyed_loop(self, ptr):
+        raise NotImplementedError()
 
     @property
     def ptr(self):

src/gevent/libev/corecext.pyx

@@ -259,8 +259,6 @@ cdef bint _check_loop(loop loop) except -1:
     return 1
 
 
-cdef bint _default_loop_destroyed = False
-
 
 cdef public class callback [object PyGeventCallbackObject, type PyGeventCallback_Type]:
     cdef public object callback
@@ -419,8 +417,6 @@ cdef public class loop [object PyGeventLoopObject, type PyGeventLoop_Type]:
             c_flags |= libev.EVFLAG_FORKCHECK
             if default is None:
                 default = True
-                if _default_loop_destroyed:
-                    default = False
             if default:
                 self._default = 1
                 self._ptr = libev.gevent_ev_default_loop(c_flags)
@@ -436,6 +432,8 @@ cdef public class loop [object PyGeventLoopObject, type PyGeventLoop_Type]:
             if default or __SYSERR_CALLBACK is None:
                 set_syserr_cb(self._handle_syserr)
 
+        # Mark as not destroyed
+        libev.ev_set_userdata(self._ptr, self._ptr)
 
         libev.ev_prepare_start(self._ptr, &self._prepare)
         libev.ev_unref(self._ptr)
@@ -489,36 +487,35 @@ cdef public class loop [object PyGeventLoopObject, type PyGeventLoop_Type]:
             libev.ev_timer_stop(ptr, &self._periodic_signal_checker)
 
     def destroy(self):
-        global _default_loop_destroyed
-
         cdef libev.ev_loop* ptr = self._ptr
         self._ptr = NULL
 
         if ptr:
-            if self._default and _default_loop_destroyed:
-                # Whoops! Program error. They destroyed the default loop,
+            if not libev.ev_userdata(ptr):
+                # Whoops! Program error. They destroyed the loop,
                 # using a different loop object. Our _ptr is still
                 # valid, but the libev loop is gone. Doing anything
                 # else with it will likely cause a crash.
                 return
+            # Mark as destroyed
+            libev.ev_set_userdata(ptr, NULL)
             self._stop_watchers(ptr)
             if __SYSERR_CALLBACK == self._handle_syserr:
                 set_syserr_cb(None)
-            if self._default:
-                _default_loop_destroyed = True
             libev.ev_loop_destroy(ptr)
 
     def __dealloc__(self):
         cdef libev.ev_loop* ptr = self._ptr
         self._ptr = NULL
         if ptr != NULL:
-            if self._default and _default_loop_destroyed:
+            if not libev.ev_userdata(ptr):
                 # See destroy(). This is a bug in the caller.
                 return
-
             self._stop_watchers(ptr)
             if not self._default:
                 libev.ev_loop_destroy(ptr)
+                # Mark as destroyed
+                libev.ev_set_userdata(ptr, NULL)
 
     @property
     def ptr(self):

src/gevent/libev/corecffi.py

@@ -223,9 +223,6 @@ class loop(AbstractLoop):
         c_flags |= libev.EVFLAG_FORKCHECK
         if default is None:
             default = True
-            if loop._default_loop_destroyed:
-                default = False
-
         if default:
             ptr = libev.gevent_ev_default_loop(c_flags)
             if not ptr:
@@ -237,6 +234,8 @@ class loop(AbstractLoop):
         if default or globals()["__SYSERR_CALLBACK"] is None:
             set_syserr_cb(self._handle_syserr)
 
+        # Mark this loop as being used.
+        libev.ev_set_userdata(ptr, ptr)
         return ptr
 
     def _init_and_start_check(self):
@@ -282,6 +281,14 @@ class loop(AbstractLoop):
             if should_destroy_loop:
                 libev.ev_loop_destroy(ptr)
 
+    def _can_destroy_loop(self, ptr):
+        # Is it marked as destroyed?
+        return libev.ev_userdata(ptr)
+
+    def _destroyed_loop(self, ptr):
+        # Mark as destroyed.
+        libev.ev_set_userdata(ptr, ffi.NULL)
+
     @property
     def MAXPRI(self):
         return libev.EV_MAXPRI

src/gevent/libev/libev.pxd

@@ -202,6 +202,8 @@ cdef extern from "libev.h" nogil:
 
     ev_loop* ev_default_loop(unsigned int flags)
     ev_loop* ev_loop_new(unsigned int flags)
+    void* ev_userdata(ev_loop*)
+    void ev_set_userdata(ev_loop*, void*)
     void ev_loop_destroy(ev_loop*)
     void ev_loop_fork(ev_loop*)
     int ev_is_default_loop(ev_loop*)

src/gevent/libuv/loop.py

@@ -68,8 +68,6 @@ class loop(AbstractLoop):
     # know this in general on libev
     min_sleep_time = 0.001 # 1ms
 
-    DEFAULT_LOOP_REGENERATES = True
-
     error_handler = None
 
     _CHECK_POINTER = 'uv_check_t *'
@@ -296,14 +294,16 @@ class loop(AbstractLoop):
                 closed_failed = libuv.uv_loop_close(ptr)
                 assert closed_failed == 0, closed_failed
 
-    def _can_destroy_default_loop(self):
+    def _can_destroy_loop(self, ptr):
         # We're being asked to destroy a loop that's,
         # at the time it was constructed, was the default loop.
         # If loop objects were constructed more than once,
         # it may have already been destroyed, though.
         # We track this in the data member.
-        return self._ptr.data
+        return ptr.data
 
+    def _destroyed_loop(self, ptr):
+        ptr.data = ffi.NULL
 
     def debug(self):
         """

src/greentest/greentest/__init__.py

@@ -61,6 +61,7 @@ from greentest.skipping import skipOnAppVeyor
 from greentest.skipping import skipOnCI
 from greentest.skipping import skipOnPyPy3OnCI
 from greentest.skipping import skipOnPyPy
+from greentest.skipping import skipOnPyPyOnCI
 from greentest.skipping import skipOnPyPy3
 from greentest.skipping import skipIf
 from greentest.skipping import skipOnLibuv

src/greentest/greentest/flaky.py

@@ -20,6 +20,7 @@
 from __future__ import absolute_import, print_function, division
 
 import sys
+import functools
 import unittest
 
 from gevent.util import dump_stacks
@@ -93,3 +94,17 @@ if sysinfo.RUNNING_ON_CI or (sysinfo.PYPY and sysinfo.WIN):
     if sysinfo.LIBUV:
         reraiseFlakyTestRaceConditionLibuv = reraiseFlakyTestRaceCondition
         reraiseFlakyTestTimeoutLibuv = reraiseFlakyTestTimeout
+
+
+def reraises_flaky_timeout(exc_kind):
+
+    def wrapper(f):
+        @functools.wraps(f)
+        def m(*args):
+            try:
+                f(*args)
+            except exc_kind:
+                reraiseFlakyTestTimeout()
+        return m
+
+    return wrapper

src/greentest/greentest/params.py

@@ -26,6 +26,7 @@ from greentest.sysinfo import LIBUV
 from greentest.sysinfo import OSX
 
 from greentest.sysinfo import RUNNING_ON_TRAVIS
+from greentest.sysinfo import RUNNING_ON_APPVEYOR
 from greentest.sysinfo import EXPECT_POOR_TIMER_RESOLUTION
 from greentest.sysinfo import RESOLVER_ARES
 
@@ -62,11 +63,19 @@ if RUNNING_ON_TRAVIS:
     # connected to with the same error.
     DEFAULT_BIND_ADDR = '127.0.0.1'
 
+if RUNNING_ON_APPVEYOR:
+    DEFAULT_BIND_ADDR = '127.0.0.1'
+    DEFAULT_LOCAL_HOST_ADDR = '127.0.0.1'
+
+
 if RESOLVER_ARES and OSX:
     # Ares likes to raise "malformed domain name" on '', at least
     # on OS X
     DEFAULT_BIND_ADDR = '127.0.0.1'
 
+DEFAULT_CONNECT = DEFAULT_LOCAL_HOST_ADDR
+DEFAULT_BIND_ADDR_TUPLE = (DEFAULT_BIND_ADDR, 0)
+
 # For in-process sockets
 DEFAULT_SOCKET_TIMEOUT = 0.1 if not EXPECT_POOR_TIMER_RESOLUTION else 2.0
 

src/greentest/greentest/patched_tests_setup.py

@@ -518,11 +518,14 @@ if WIN:
 
 if PYPY:
     disabled_tests += [
-        'test_subprocess.ProcessTestCase.test_failed_child_execute_fd_leak',
         # Does not exist in the CPython test suite, tests for a specific bug
         # in PyPy's forking. Only runs on linux and is specific to the PyPy
         # implementation of subprocess (possibly explains the extra parameter to
         # _execut_child)
+        'test_subprocess.ProcessTestCase.test_failed_child_execute_fd_leak',
+        # On some platforms, this returns "zlib_compression", but the test is looking for
+        # "ZLIB"
+        'test_ssl.ThreadedTests.test_compression',
     ]
 
 # Generic Python 3

src/greentest/greentest/skipping.py

@@ -31,10 +31,24 @@ def _do_not_skip(reason):
     return _identity
 
 
+skipOnWindows = _do_not_skip
+skipOnAppVeyor = _do_not_skip
+skipOnCI = _do_not_skip
+
+skipOnPyPy = _do_not_skip
+skipOnPyPyOnCI = _do_not_skip
+skipOnPyPy3OnCI = _do_not_skip
+skipOnPyPy3 = _do_not_skip
+
+skipOnLibuv = _do_not_skip
+skipOnLibuvOnCI = _do_not_skip
+skipOnLibuvOnCIOnPyPy = _do_not_skip
+skipOnLibuvOnPyPyOnWin = _do_not_skip
+
+
 if sysinfo.WIN:
     skipOnWindows = unittest.skip
-else:
-    skipOnWindows = _do_not_skip
+
 
 if sysinfo.RUNNING_ON_APPVEYOR:
     # See comments scattered around about timeouts and the timer
@@ -46,39 +60,28 @@ if sysinfo.RUNNING_ON_APPVEYOR:
     # separately on windows in a more stable environment.
     skipOnAppVeyor = unittest.skip
 
-else:
-    skipOnAppVeyor = _do_not_skip
 
 if sysinfo.RUNNING_ON_CI:
     skipOnCI = unittest.skip
-else:
-    skipOnCI = _do_not_skip
 
-if sysinfo.PYPY3 and sysinfo.RUNNING_ON_CI:
-    # Same as above, for PyPy3.3-5.5-alpha and 3.5-5.7.1-beta and 3.5-5.8
-    skipOnPyPy3OnCI = unittest.skip
-else:
-    skipOnPyPy3OnCI = _do_not_skip
 
 if sysinfo.PYPY:
     skipOnPyPy = unittest.skip
-else:
-    skipOnPyPy = _do_not_skip
+    if sysinfo.RUNNING_ON_CI:
+        skipOnPyPyOnCI = unittest.skip
+
+    if sysinfo.PYPY3:
+        skipOnPyPy3 = unittest.skip
+        if sysinfo.RUNNING_ON_CI:
+            # Same as above, for PyPy3.3-5.5-alpha and 3.5-5.7.1-beta and 3.5-5.8
+            skipOnPyPy3OnCI = unittest.skip
 
-if sysinfo.PYPY3:
-    skipOnPyPy3 = unittest.skip
-else:
-    skipOnPyPy3 = _do_not_skip
 
 skipUnderCoverage = unittest.skip if sysinfo.RUN_COVERAGE else _do_not_skip
 
 skipIf = unittest.skipIf
 
 
-skipOnLibuv = _do_not_skip
-skipOnLibuvOnCI = _do_not_skip
-skipOnLibuvOnCIOnPyPy = _do_not_skip
-skipOnLibuvOnPyPyOnWin = _do_not_skip
 
 if sysinfo.LIBUV:
     skipOnLibuv = unittest.skip

src/greentest/greentest/sockets.py

@@ -19,8 +19,9 @@
 # THE SOFTWARE.
 from __future__ import absolute_import, print_function, division
 
+from greentest.params import DEFAULT_BIND_ADDR_TUPLE
 
-def bind_and_listen(sock, address=('', 0), backlog=50, reuse_addr=True):
+def bind_and_listen(sock, address=DEFAULT_BIND_ADDR_TUPLE, backlog=50, reuse_addr=True):
     from socket import SOL_SOCKET, SO_REUSEADDR, error
     if reuse_addr:
         try:
@@ -32,7 +33,7 @@ def bind_and_listen(sock, address=('', 0), backlog=50, reuse_addr=True):
     sock.listen(backlog)
 
 
-def tcp_listener(address, backlog=50, reuse_addr=True):
+def tcp_listener(address=DEFAULT_BIND_ADDR_TUPLE, backlog=50, reuse_addr=True):
     """A shortcut to create a TCP socket, bind it and put it into listening state."""
     from gevent import socket
     sock = socket.socket()

src/greentest/greentest/testrunner.py

@@ -55,7 +55,7 @@ if PYPY:
     ]
 
 
-def run_many(tests, expected=(), failfast=False, quiet=False):
+def run_many(tests, configured_failing_tests=(), failfast=False, quiet=False):
     # pylint:disable=too-many-locals
     global NWORKERS
     start = time.time()
@@ -127,7 +127,8 @@ def run_many(tests, expected=(), failfast=False, quiet=False):
                 reap_all()
             except KeyboardInterrupt:
                 pool.terminate()
-                report(total, failed, passed, exit=False, took=time.time() - start, expected=expected)
+                report(total, failed, passed, exit=False, took=time.time() - start,
+                       configured_failing_tests=configured_failing_tests)
                 log('(partial results)\n')
                 raise
     except:
@@ -136,17 +137,18 @@ def run_many(tests, expected=(), failfast=False, quiet=False):
         raise
 
     reap_all()
-    report(total, failed, passed, took=time.time() - start, expected=expected)
+    report(total, failed, passed, took=time.time() - start,
+           configured_failing_tests=configured_failing_tests)
 
 
-def discover(tests=None, ignore=(), coverage=False):
-    if isinstance(ignore, six.string_types):
-        ignore_files = ignore.split(',')
-        ignore = set()
+def discover(tests=None, ignore_files=None,
+             ignored=(), coverage=False):
+    ignore = set(ignored or ())
+    if ignore_files:
+        ignore_files = ignore_files.split(',')
         for f in ignore_files:
             ignore.update(set(load_list_from_file(f)))
 
-    ignore = set(ignore or ())
     if coverage:
         ignore.update(IGNORE_COVERAGE)
 
@@ -200,10 +202,10 @@ def load_list_from_file(filename):
     return result
 
 
-def matches(expected, command, include_flaky=True):
+def matches(possibilities, command, include_flaky=True):
     if isinstance(command, list):
         command = ' '.join(command)
-    for line in expected:
+    for line in possibilities:
         if not include_flaky and line.startswith('FLAKY '):
             continue
         if command.endswith(' ' + line.replace('FLAKY ', '')):
@@ -220,7 +222,8 @@ def format_seconds(seconds):
     return seconds
 
 
-def report(total, failed, passed, exit=True, took=None, expected=None):
+def report(total, failed, passed, exit=True, took=None,
+           configured_failing_tests=()):
     # pylint:disable=redefined-builtin,too-many-branches
     runtimelog = util.runtimelog
     if runtimelog:
@@ -240,7 +243,7 @@ def report(total, failed, passed, exit=True, took=None, expected=None):
     passed_unexpected = []
 
     for name in passed:
-        if matches(expected, name, include_flaky=False):
+        if matches(configured_failing_tests, name, include_flaky=False):
             passed_unexpected.append(name)
 
     if passed_unexpected:
@@ -249,9 +252,9 @@ def report(total, failed, passed, exit=True, took=None, expected=None):
 
     if failed:
         log('\n%s/%s tests failed%s', len(failed), total, took)
-        expected = set(expected or [])
+
         for name in failed:
-            if matches(expected, name, include_flaky=True):
+            if matches(configured_failing_tests, name, include_flaky=True):
                 failed_expected.append(name)
             else:
                 failed_unexpected.append(name)
@@ -293,6 +296,7 @@ def main():
     parser.add_argument('tests', nargs='*')
     options = parser.parse_args()
     FAILING_TESTS = []
+    IGNORED_TESTS = []
     coverage = False
     if options.coverage or os.environ.get("GEVENTTEST_COVERAGE"):
         coverage = True
@@ -312,6 +316,7 @@ def main():
             config_data = f.read()
         six.exec_(config_data, config)
         FAILING_TESTS = config['FAILING_TESTS']
+        IGNORED_TESTS = config['IGNORED_TESTS']
 
     if 'PYTHONWARNINGS' not in os.environ and not sys.warnoptions:
         # Enable default warnings such as ResourceWarning.
@@ -330,13 +335,16 @@ def main():
     if 'GEVENT_DEBUG' not in os.environ:
         os.environ['GEVENT_DEBUG'] = 'debug'
 
-    tests = discover(options.tests, options.ignore, coverage)
+    tests = discover(options.tests,
+                     ignore_files=options.ignore,
+                     ignored=IGNORED_TESTS,
+                     coverage=coverage)
     if options.discover:
         for cmd, options in tests:
             print(util.getname(cmd, env=options.get('env'), setenv=options.get('setenv')))
         print('%s tests found.' % len(tests))
     else:
-        run_many(tests, expected=FAILING_TESTS, failfast=options.failfast, quiet=options.quiet)
+        run_many(tests, configured_failing_tests=FAILING_TESTS, failfast=options.failfast, quiet=options.quiet)
 
 
 if __name__ == '__main__':

src/greentest/known_failures.py

@@ -18,6 +18,7 @@ from greentest.sysinfo import PY35
 
 from greentest.sysinfo import LIBUV
 
+IGNORED_TESTS = []
 
 FAILING_TESTS = [
 
@@ -33,11 +34,14 @@ FAILING_TESTS = [
 
 
 if sys.platform == 'win32':
+    IGNORED_TESTS = [
+        # fork watchers don't get called on windows
+        # because fork is not a concept windows has.
+        # See this file for a detailed explanation.
+        'test__core_fork.py',
+    ]
     # other Windows-related issues (need investigating)
     FAILING_TESTS += [
-        # fork watchers don't get called in multithreaded programs on windows
-        # No idea why.
-        'test__core_fork.py',
         'FLAKY test__greenletset.py',
         # This has been seen to fail on Py3 and Py2 due to socket reuse
         # errors, probably timing related again.
@@ -70,14 +74,15 @@ if sys.platform == 'win32':
         ]
 
         if PYPY and LIBUV:
-            FAILING_TESTS += [
-                # This one sometimes seems to just stop right after
+            IGNORED_TESTS += [
+                # This one seems to just stop right after
                 # patching is done. It passes on a local win 10 vm, and the main
                 # test_threading_2.py does as well.
                 # Based on the printouts we added, it appears to not even
                 # finish importing:
                 # https://ci.appveyor.com/project/denik/gevent/build/1.0.1277/job/tpvhesij5gldjxqw#L1190
-                'FLAKY test_threading.py',
+                # Ignored because it takes two minutes to time out.
+                'test_threading.py',
             ]
 
         if PY3:
@@ -93,8 +98,8 @@ if sys.platform == 'win32':
 
     if not PY35:
         # Py35 added socket.socketpair, all other releases
-        # are missing it
-        FAILING_TESTS += [
+        # are missing it. No reason to even test it.
+        IGNORED_TESTS += [
             'test__socketpair.py',
         ]
 

src/greentest/test___example_servers.py

@@ -14,6 +14,7 @@ from greentest import DEFAULT_XPC_SOCKET_TIMEOUT
 from greentest import util
 from greentest import params
 
+@greentest.skipOnLibuvOnCIOnPyPy("Timing issues sometimes lead to a connection refused")
 class Test_wsgiserver(util.TestServer):
     server = 'wsgiserver.py'
     URL = 'http://%s:8088' % (params.DEFAULT_LOCAL_HOST_ADDR,)

src/greentest/test__core.py

 # pylint:disable=no-member
 import sys
 import unittest
-from greentest import main, skipOnLibuv
+import greentest
+
 from gevent import core
 
 
@@ -19,6 +20,9 @@ class TestCore(unittest.TestCase):
 
 class TestWatchers(unittest.TestCase):
 
+    def makeOne(self):
+        return core.loop()
+
     def test_io(self):
         if sys.platform == 'win32':
             # libev raises IOError, libuv raises ValueError
@@ -28,17 +32,17 @@ class TestWatchers(unittest.TestCase):
             Error = ValueError
             win32 = False
         with self.assertRaises(Error):
-            core.loop().io(-1, 1)
+            self.makeOne().io(-1, 1)
         if hasattr(core, 'TIMER'):
             # libev
             with self.assertRaises(ValueError):
-                core.loop().io(1, core.TIMER)
+                self.makeOne().io(1, core.TIMER)
 
         # Test we can set events and io before it's started
         if not win32:
             # We can't do this with arbitrary FDs on windows;
             # see libev_vfd.h
-            io = core.loop().io(1, core.READ)
+            io = self.makeOne().io(1, core.READ)
             io.fd = 2
             self.assertEqual(io.fd, 2)
             io.events = core.WRITE
@@ -47,17 +51,33 @@ class TestWatchers(unittest.TestCase):
                 self.assertEqual(core._events_to_str(io.events), 'WRITE|_IOFDSET')
             else:
                 self.assertEqual(core._events_to_str(io.events), 'WRITE')
+            io.start(lambda: None)
             io.close()
 
     def test_timer_constructor(self):
         with self.assertRaises(ValueError):
-            core.loop().timer(1, -1)
+            self.makeOne().timer(1, -1)
 
     def test_signal_constructor(self):
         with self.assertRaises(ValueError):
-            core.loop().signal(1000)
+            self.makeOne().signal(1000)
+
+class TestWatchersDefault(TestWatchers):
+
+    def makeOne(self):
+        return core.loop(default=True)
+
+@greentest.skipOnLibuvOnPyPyOnWin("This crashes with PyPy 5.10.0, only on Windows. "
+                                  "See https://ci.appveyor.com/project/denik/gevent/build/1.0.1380/job/lrlvid6mkjtyrhn5#L1103")
+class TestWatchersDefaultDestroyed(TestWatchers):
+
+    def makeOne(self):
+        l = core.loop(default=True)
+        l.destroy()
+        del l
+        return core.loop(default=True)
 
-@skipOnLibuv("Tests for libev-only functions")
+@greentest.skipOnLibuv("Tests for libev-only functions")
 class TestLibev(unittest.TestCase):
 
     def test_flags_conversion(self):
@@ -81,4 +101,4 @@ class TestEvents(unittest.TestCase):
         self.assertEqual(repr(core.EVENTS), 'gevent.core.EVENTS')
 
 if __name__ == '__main__':
-    main()
+    greentest.main()

src/greentest/test__core_fork.py

@@ -2,7 +2,7 @@ from __future__ import print_function
 import gevent.monkey; gevent.monkey.patch_all()
 import gevent
 import os
-import sys
+
 import multiprocessing
 
 hub = gevent.get_hub()
@@ -48,4 +48,10 @@ if __name__ == '__main__':
     # fork watchers weren't firing in multi-threading processes.
     # This test is designed to prove that they are.
     # However, it fails on Windows: The fork watcher never runs!
+    # This makes perfect sense: on Windows, our patches to os.fork()
+    # that call gevent.hub.reinit() don't get used; os.fork doesn't
+    # exist and multiprocessing.Process uses the windows-specific _subprocess.CreateProcess()
+    # to create a whole new process that has no relation to the current process;
+    # that process then calls multiprocessing.forking.main() to do its work.
+    # Since no state is shared, a fork watcher cannot exist in that process.
     test()

src/greentest/test__destroy.py

@@ -40,13 +40,10 @@ class TestDestroyHub(unittest.TestCase):
         self.assertIsNot(hub.loop.ptr, initloop.ptr)
         self.assertNotEqual(hub.loop.ptr, initloop.ptr)
 
-        # Destroy hub including default loop, create new hub with non-default loop.
+        # Destroy hub including default loop. The default loop regenerates.
         hub.destroy(destroy_loop=True)
         hub = gevent.get_hub()
-        if not getattr(hub.loop, 'DEFAULT_LOOP_REGENERATES', False):
-            self.assertFalse(hub.loop.default)
-        else:
-            self.assertTrue(hub.loop.default)
+        self.assertTrue(hub.loop.default)
 
         hub.destroy()
 

src/greentest/test__example_portforwarder.py

-from __future__ import print_function
+from __future__ import print_function, absolute_import
 from gevent import monkey; monkey.patch_all(subprocess=True)
 import signal
 import sys
@@ -8,9 +8,10 @@ from time import sleep
 import gevent
 from gevent.server import StreamServer
 
+import greentest
 from greentest import util
 
-
+@greentest.skipOnLibuvOnCIOnPyPy("Timing issues sometimes lead to connection refused")
 class Test(util.TestServer):
     server = 'portforwarder.py'
     args = ['127.0.0.1:10011', '127.0.0.1:10012']
@@ -62,5 +63,4 @@ class Test(util.TestServer):
 
 
 if __name__ == '__main__':
-    from greentest import main
-    main()
+    greentest.main()

src/greentest/test__example_udp_client.py

@@ -14,7 +14,7 @@ class Test_udp_client(TestCase):
             log.append(message)
             server.sendto(b'reply-from-server', address)
 
-        server = DatagramServer('127.0.0.1:9000', handle)
+        server = DatagramServer('127.0.0.1:9001', handle)
         server.start()
         try:
             run([sys.executable, '-W', 'ignore', '-u', 'udp_client.py', 'Test_udp_client'],

src/greentest/test__greenio.py

@@ -16,12 +16,15 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
+import sys
 
-from greentest import TestCase, main, tcp_listener
-from greentest import skipOnPyPy
 import gevent
 from gevent import socket
-import sys
+
+from greentest import TestCase, main, tcp_listener
+from greentest import skipOnPyPy
+from greentest import params
+
 
 
 PYPY = hasattr(sys, 'pypy_version_info')
@@ -45,7 +48,7 @@ class TestGreenIo(TestCase):
             # verify that the makefile and the socket are truly independent
             # by closing the socket prior to using the made file
             try:
-                conn, addr = listener.accept()
+                conn, _ = listener.accept()
                 fd = conn.makefile(mode='wb')
                 conn.close()
                 fd.write(b'hello\n')
@@ -59,7 +62,7 @@ class TestGreenIo(TestCase):
             # verify that the makefile and the socket are truly independent
             # by closing the made file and then sending a character
             try:
-                conn, addr = listener.accept()
+                conn, _ = listener.accept()
                 fd = conn.makefile(mode='wb')
                 fd.write(b'hello')
                 fd.close()
@@ -71,27 +74,25 @@ class TestGreenIo(TestCase):
                 listener.close()
 
         def did_it_work(server):
-            client = socket.create_connection(('127.0.0.1', server.getsockname()[1]))
+            client = socket.create_connection((params.DEFAULT_CONNECT, server.getsockname()[1]))
             fd = client.makefile(mode='rb')
             client.close()
-            assert fd.readline() == b'hello\n'
-            assert fd.read() == b''
+            self.assertEqual(fd.readline(), b'hello\n')
+            self.assertFalse(fd.read())
             fd.close()
 
-        server = tcp_listener(('0.0.0.0', 0))
+        server = tcp_listener()
         server_greenlet = gevent.spawn(accept_close_early, server)
         did_it_work(server)
         server_greenlet.kill()
 
-        server = tcp_listener(('0.0.0.0', 0))
+        server = tcp_listener()
         server_greenlet = gevent.spawn(accept_close_late, server)
         did_it_work(server)
         server_greenlet.kill()
 
     @skipOnPyPy("GC is different")
     def test_del_closes_socket(self):
-        timer = gevent.Timeout.start_new(0.5)
-
         def accept_once(listener):
             # delete/overwrite the original conn
             # object, only keeping the file object around
@@ -113,15 +114,15 @@ class TestGreenIo(TestCase):
                 if oconn is not None:
                     oconn.close()
 
-        server = tcp_listener(('0.0.0.0', 0))
+        server = tcp_listener()
         gevent.spawn(accept_once, server)
-        client = socket.create_connection(('127.0.0.1', server.getsockname()[1]))
-        fd = client.makefile()
-        client.close()
-        assert fd.read() == 'hello\n'
-        assert fd.read() == ''
+        client = socket.create_connection((params.DEFAULT_CONNECT, server.getsockname()[1]))
+        with gevent.Timeout.start_new(0.5):
+            fd = client.makefile()
+            client.close()
+            self.assertEqual(fd.read(), 'hello\n')
+            self.assertEqual(fd.read(), '')
 
-        timer.close()
 
 if __name__ == '__main__':
     main()

src/greentest/test__greenness.py

@@ -23,9 +23,11 @@
 To do that spawn a green server and then access it using a green socket.
 If either operation blocked the whole script would block and timeout.
 """
-import greentest
 from gevent import monkey
 monkey.patch_all()
+
+import greentest
+
 try:
     import urllib2
 except ImportError:
@@ -34,33 +36,37 @@ try:
     import BaseHTTPServer
 except ImportError:
     from http import server as BaseHTTPServer
+
 import gevent
+from greentest import params
 
 
 class TestGreenness(greentest.TestCase):
     check_totalrefcount = False
 
+    def setUp(self):
+        server_address = params.DEFAULT_BIND_ADDR_TUPLE
+        BaseHTTPServer.BaseHTTPRequestHandler.protocol_version = "HTTP/1.0"
+        self.httpd = BaseHTTPServer.HTTPServer(server_address, BaseHTTPServer.BaseHTTPRequestHandler)
+        self.httpd.request_count = 0
+
+    def tearDown(self):
+        self.httpd.server_close()
+        self.httpd = None
+
     def serve(self):
         self.httpd.handle_request()
         self.httpd.request_count += 1
 
     def test_urllib2(self):
-        server_address = ('', 0)
-        BaseHTTPServer.BaseHTTPRequestHandler.protocol_version = "HTTP/1.0"
-        self.httpd = BaseHTTPServer.HTTPServer(server_address, BaseHTTPServer.BaseHTTPRequestHandler)
-        self.httpd.request_count = 0
         server = gevent.spawn(self.serve)
 
         port = self.httpd.socket.getsockname()[1]
-        try:
+        with self.assertRaises(urllib2.HTTPError) as exc:
             urllib2.urlopen('http://127.0.0.1:%s' % port)
-            assert False, 'should not get there'
-        except urllib2.HTTPError as ex:
-            assert ex.code == 501, repr(ex)
+        self.assertEqual(exc.exception.code, 501)
         server.get(0.01)
         self.assertEqual(self.httpd.request_count, 1)
-        self.httpd.server_close()
-        self.httpd = None
 
 
 if __name__ == '__main__':

src/greentest/test__refcount.py

@@ -42,6 +42,7 @@ _socket.socket = Socket
 
 import greentest
 from gevent import monkey; monkey.patch_all()
+from greentest import flaky
 
 from pprint import pformat
 try:
@@ -137,16 +138,17 @@ def run_and_check(run_client):
         raise AssertionError('server should be dead by now')
 
 
-@greentest.skipOnAppVeyor("Often fail with timeouts or force closed connections; not sure why.")
-@greentest.skipOnPyPy3OnCI("Often fails with timeouts; not sure why.")
+@greentest.skipOnCI("Often fail with timeouts or force closed connections; not sure why.")
 class Test(greentest.TestCase):
 
-    __timeout__ = 10
+    __timeout__ = greentest.LARGE_TIMEOUT
 
+    @flaky.reraises_flaky_timeout(socket.timeout)
     def test_clean_exit(self):
         run_and_check(True)
         run_and_check(True)
 
+    @flaky.reraises_flaky_timeout(socket.timeout)
     def test_timeout_exit(self):
         run_and_check(False)
         run_and_check(False)