Address some more failing tests. Some of the UDP tests were fighting over ports!

Makefile

@@ -90,7 +90,7 @@ allbackendtest:
 
 cffibackendtest:
 	@${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
-	GEVENT_LOOP=libuv GEVENTTEST_COVERAGE=1 make alltest
+	GEVENT_LOOP=libuv make alltest
 	@${PYTHON} scripts/travis.py fold_end libuv
 	@${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
 	GEVENT_LOOP=libev-cffi make alltest
@@ -193,7 +193,7 @@ test-py37: $(PY37)
 	LD_LIBRARY_PATH=$(BUILD_RUNTIMES)/versions/python3.7.0b1/openssl/lib PYTHON=python3.7.0b1 PATH=$(BUILD_RUNTIMES)/versions/python3.7.0b1/bin:$(PATH) make develop allbackendtest
 
 test-pypy: $(PYPY)
-	PYTHON=$(PYPY) PATH=$(BUILD_RUNTIMES)/versions/pypy5100/bin:$(PATH) make develop cffibackendtest coverage_combine
+	PYTHON=$(PYPY) PATH=$(BUILD_RUNTIMES)/versions/pypy5100/bin:$(PATH) make develop cffibackendtest
 
 test-pypy3: $(PYPY3)
 	PYTHON=$(PYPY3) PATH=$(BUILD_RUNTIMES)/versions/pypy3.5_5101/bin:$(PATH) make develop basictest

examples/udp_client.py

@@ -3,7 +3,7 @@
 
 Usage: python udp_client.py MESSAGE
 
-Make sure you're running a UDP server on port 9000 (see udp_server.py).
+Make sure you're running a UDP server on port 9001 (see udp_server.py).
 
 There's nothing gevent-specific here.
 """
@@ -11,7 +11,7 @@ from __future__ import print_function
 import sys
 from gevent import socket
 
-address = ('localhost', 9000)
+address = ('localhost', 9001)
 message = ' '.join(sys.argv[1:])
 sock = socket.socket(type=socket.SOCK_DGRAM)
 sock.connect(address)

src/greentest/greentest/__init__.py

@@ -61,6 +61,7 @@ from greentest.skipping import skipOnAppVeyor
 from greentest.skipping import skipOnCI
 from greentest.skipping import skipOnPyPy3OnCI
 from greentest.skipping import skipOnPyPy
+from greentest.skipping import skipOnPyPyOnCI
 from greentest.skipping import skipOnPyPy3
 from greentest.skipping import skipIf
 from greentest.skipping import skipOnLibuv

src/greentest/greentest/params.py

@@ -26,6 +26,7 @@ from greentest.sysinfo import LIBUV
 from greentest.sysinfo import OSX
 
 from greentest.sysinfo import RUNNING_ON_TRAVIS
+from greentest.sysinfo import RUNNING_ON_APPVEYOR
 from greentest.sysinfo import EXPECT_POOR_TIMER_RESOLUTION
 from greentest.sysinfo import RESOLVER_ARES
 
@@ -62,11 +63,19 @@ if RUNNING_ON_TRAVIS:
     # connected to with the same error.
     DEFAULT_BIND_ADDR = '127.0.0.1'
 
+if RUNNING_ON_APPVEYOR:
+    DEFAULT_BIND_ADDR = '127.0.0.1'
+    DEFAULT_LOCAL_HOST_ADDR = '127.0.0.1'
+
+
 if RESOLVER_ARES and OSX:
     # Ares likes to raise "malformed domain name" on '', at least
     # on OS X
     DEFAULT_BIND_ADDR = '127.0.0.1'
 
+DEFAULT_CONNECT = DEFAULT_LOCAL_HOST_ADDR
+DEFAULT_BIND_ADDR_TUPLE = (DEFAULT_BIND_ADDR, 0)
+
 # For in-process sockets
 DEFAULT_SOCKET_TIMEOUT = 0.1 if not EXPECT_POOR_TIMER_RESOLUTION else 2.0
 

src/greentest/greentest/patched_tests_setup.py

@@ -518,11 +518,14 @@ if WIN:
 
 if PYPY:
     disabled_tests += [
-        'test_subprocess.ProcessTestCase.test_failed_child_execute_fd_leak',
         # Does not exist in the CPython test suite, tests for a specific bug
         # in PyPy's forking. Only runs on linux and is specific to the PyPy
         # implementation of subprocess (possibly explains the extra parameter to
         # _execut_child)
+        'test_subprocess.ProcessTestCase.test_failed_child_execute_fd_leak',
+        # On some platforms, this returns "zlib_compression", but the test is looking for
+        # "ZLIB"
+        'test_ssl.ThreadedTests.test_compression',
     ]
 
 # Generic Python 3

src/greentest/greentest/skipping.py

@@ -31,10 +31,24 @@ def _do_not_skip(reason):
     return _identity
 
 
+skipOnWindows = _do_not_skip
+skipOnAppVeyor = _do_not_skip
+skipOnCI = _do_not_skip
+
+skipOnPyPy = _do_not_skip
+skipOnPyPyOnCI = _do_not_skip
+skipOnPyPy3OnCI = _do_not_skip
+skipOnPyPy3 = _do_not_skip
+
+skipOnLibuv = _do_not_skip
+skipOnLibuvOnCI = _do_not_skip
+skipOnLibuvOnCIOnPyPy = _do_not_skip
+skipOnLibuvOnPyPyOnWin = _do_not_skip
+
+
 if sysinfo.WIN:
     skipOnWindows = unittest.skip
-else:
-    skipOnWindows = _do_not_skip
+
 
 if sysinfo.RUNNING_ON_APPVEYOR:
     # See comments scattered around about timeouts and the timer
@@ -46,39 +60,28 @@ if sysinfo.RUNNING_ON_APPVEYOR:
     # separately on windows in a more stable environment.
     skipOnAppVeyor = unittest.skip
 
-else:
-    skipOnAppVeyor = _do_not_skip
 
 if sysinfo.RUNNING_ON_CI:
     skipOnCI = unittest.skip
-else:
-    skipOnCI = _do_not_skip
 
-if sysinfo.PYPY3 and sysinfo.RUNNING_ON_CI:
-    # Same as above, for PyPy3.3-5.5-alpha and 3.5-5.7.1-beta and 3.5-5.8
-    skipOnPyPy3OnCI = unittest.skip
-else:
-    skipOnPyPy3OnCI = _do_not_skip
 
 if sysinfo.PYPY:
     skipOnPyPy = unittest.skip
-else:
-    skipOnPyPy = _do_not_skip
+    if sysinfo.RUNNING_ON_CI:
+        skipOnPyPyOnCI = unittest.skip
+
+    if sysinfo.PYPY3:
+        skipOnPyPy3 = unittest.skip
+        if sysinfo.RUNNING_ON_CI:
+            # Same as above, for PyPy3.3-5.5-alpha and 3.5-5.7.1-beta and 3.5-5.8
+            skipOnPyPy3OnCI = unittest.skip
 
-if sysinfo.PYPY3:
-    skipOnPyPy3 = unittest.skip
-else:
-    skipOnPyPy3 = _do_not_skip
 
 skipUnderCoverage = unittest.skip if sysinfo.RUN_COVERAGE else _do_not_skip
 
 skipIf = unittest.skipIf
 
 
-skipOnLibuv = _do_not_skip
-skipOnLibuvOnCI = _do_not_skip
-skipOnLibuvOnCIOnPyPy = _do_not_skip
-skipOnLibuvOnPyPyOnWin = _do_not_skip
 
 if sysinfo.LIBUV:
     skipOnLibuv = unittest.skip

src/greentest/greentest/sockets.py

@@ -19,8 +19,9 @@
 # THE SOFTWARE.
 from __future__ import absolute_import, print_function, division
 
+from greentest.params import DEFAULT_BIND_ADDR_TUPLE
 
-def bind_and_listen(sock, address=('', 0), backlog=50, reuse_addr=True):
+def bind_and_listen(sock, address=DEFAULT_BIND_ADDR_TUPLE, backlog=50, reuse_addr=True):
     from socket import SOL_SOCKET, SO_REUSEADDR, error
     if reuse_addr:
         try:
@@ -32,7 +33,7 @@ def bind_and_listen(sock, address=('', 0), backlog=50, reuse_addr=True):
     sock.listen(backlog)
 
 
-def tcp_listener(address, backlog=50, reuse_addr=True):
+def tcp_listener(address=DEFAULT_BIND_ADDR_TUPLE, backlog=50, reuse_addr=True):
     """A shortcut to create a TCP socket, bind it and put it into listening state."""
     from gevent import socket
     sock = socket.socket()

src/greentest/greentest/testrunner.py

@@ -55,7 +55,7 @@ if PYPY:
     ]
 
 
-def run_many(tests, expected=(), failfast=False, quiet=False):
+def run_many(tests, configured_failing_tests=(), failfast=False, quiet=False):
     # pylint:disable=too-many-locals
     global NWORKERS
     start = time.time()
@@ -127,7 +127,8 @@ def run_many(tests, expected=(), failfast=False, quiet=False):
                 reap_all()
             except KeyboardInterrupt:
                 pool.terminate()
-                report(total, failed, passed, exit=False, took=time.time() - start, expected=expected)
+                report(total, failed, passed, exit=False, took=time.time() - start,
+                       configured_failing_tests=configured_failing_tests)
                 log('(partial results)\n')
                 raise
     except:
@@ -136,17 +137,18 @@ def run_many(tests, expected=(), failfast=False, quiet=False):
         raise
 
     reap_all()
-    report(total, failed, passed, took=time.time() - start, expected=expected)
+    report(total, failed, passed, took=time.time() - start,
+           configured_failing_tests=configured_failing_tests)
 
 
-def discover(tests=None, ignore=(), coverage=False):
-    if isinstance(ignore, six.string_types):
-        ignore_files = ignore.split(',')
-        ignore = set()
+def discover(tests=None, ignore_files=None,
+             ignored=(), coverage=False):
+    ignore = set(ignored or ())
+    if ignore_files:
+        ignore_files = ignore_files.split(',')
         for f in ignore_files:
             ignore.update(set(load_list_from_file(f)))
 
-    ignore = set(ignore or ())
     if coverage:
         ignore.update(IGNORE_COVERAGE)
 
@@ -200,10 +202,10 @@ def load_list_from_file(filename):
     return result
 
 
-def matches(expected, command, include_flaky=True):
+def matches(possibilities, command, include_flaky=True):
     if isinstance(command, list):
         command = ' '.join(command)
-    for line in expected:
+    for line in possibilities:
         if not include_flaky and line.startswith('FLAKY '):
             continue
         if command.endswith(' ' + line.replace('FLAKY ', '')):
@@ -220,7 +222,8 @@ def format_seconds(seconds):
     return seconds
 
 
-def report(total, failed, passed, exit=True, took=None, expected=None):
+def report(total, failed, passed, exit=True, took=None,
+           configured_failing_tests=()):
     # pylint:disable=redefined-builtin,too-many-branches
     runtimelog = util.runtimelog
     if runtimelog:
@@ -240,7 +243,7 @@ def report(total, failed, passed, exit=True, took=None, expected=None):
     passed_unexpected = []
 
     for name in passed:
-        if matches(expected, name, include_flaky=False):
+        if matches(configured_failing_tests, name, include_flaky=False):
             passed_unexpected.append(name)
 
     if passed_unexpected:
@@ -249,9 +252,9 @@ def report(total, failed, passed, exit=True, took=None, expected=None):
 
     if failed:
         log('\n%s/%s tests failed%s', len(failed), total, took)
-        expected = set(expected or [])
+
         for name in failed:
-            if matches(expected, name, include_flaky=True):
+            if matches(configured_failing_tests, name, include_flaky=True):
                 failed_expected.append(name)
             else:
                 failed_unexpected.append(name)
@@ -293,6 +296,7 @@ def main():
     parser.add_argument('tests', nargs='*')
     options = parser.parse_args()
     FAILING_TESTS = []
+    IGNORED_TESTS = []
     coverage = False
     if options.coverage or os.environ.get("GEVENTTEST_COVERAGE"):
         coverage = True
@@ -312,6 +316,7 @@ def main():
             config_data = f.read()
         six.exec_(config_data, config)
         FAILING_TESTS = config['FAILING_TESTS']
+        IGNORED_TESTS = config['IGNORED_TESTS']
 
     if 'PYTHONWARNINGS' not in os.environ and not sys.warnoptions:
         # Enable default warnings such as ResourceWarning.
@@ -330,13 +335,16 @@ def main():
     if 'GEVENT_DEBUG' not in os.environ:
         os.environ['GEVENT_DEBUG'] = 'debug'
 
-    tests = discover(options.tests, options.ignore, coverage)
+    tests = discover(options.tests,
+                     ignore_files=options.ignore,
+                     ignored=IGNORED_TESTS,
+                     coverage=coverage)
     if options.discover:
         for cmd, options in tests:
             print(util.getname(cmd, env=options.get('env'), setenv=options.get('setenv')))
         print('%s tests found.' % len(tests))
     else:
-        run_many(tests, expected=FAILING_TESTS, failfast=options.failfast, quiet=options.quiet)
+        run_many(tests, configured_failing_tests=FAILING_TESTS, failfast=options.failfast, quiet=options.quiet)
 
 
 if __name__ == '__main__':

src/greentest/known_failures.py

@@ -18,6 +18,7 @@ from greentest.sysinfo import PY35
 
 from greentest.sysinfo import LIBUV
 
+IGNORED_TESTS = []
 
 FAILING_TESTS = [
 
@@ -70,14 +71,15 @@ if sys.platform == 'win32':
         ]
 
         if PYPY and LIBUV:
-            FAILING_TESTS += [
-                # This one sometimes seems to just stop right after
+            IGNORED_TESTS += [
+                # This one seems to just stop right after
                 # patching is done. It passes on a local win 10 vm, and the main
                 # test_threading_2.py does as well.
                 # Based on the printouts we added, it appears to not even
                 # finish importing:
                 # https://ci.appveyor.com/project/denik/gevent/build/1.0.1277/job/tpvhesij5gldjxqw#L1190
-                'FLAKY test_threading.py',
+                # Ignored because it takes two minutes to time out.
+                'test_threading.py',
             ]
 
         if PY3:

src/greentest/test___example_servers.py

@@ -14,6 +14,7 @@ from greentest import DEFAULT_XPC_SOCKET_TIMEOUT
 from greentest import util
 from greentest import params
 
+@greentest.skipOnLibuvOnCIOnPyPy("Timing issues sometimes lead to a connection refused")
 class Test_wsgiserver(util.TestServer):
     server = 'wsgiserver.py'
     URL = 'http://%s:8088' % (params.DEFAULT_LOCAL_HOST_ADDR,)

src/greentest/test__core.py

 # pylint:disable=no-member
 import sys
 import unittest
-from greentest import main, skipOnLibuv
+import greentest
+
 from gevent import core
 
 
@@ -66,6 +67,8 @@ class TestWatchersDefault(TestWatchers):
     def makeOne(self):
         return core.loop(default=True)
 
+@greentest.skipOnLibuvOnPyPyOnWin("This crashes with PyPy 5.10.0, only on Windows. "
+                                  "See https://ci.appveyor.com/project/denik/gevent/build/1.0.1380/job/lrlvid6mkjtyrhn5#L1103")
 class TestWatchersDefaultDestroyed(TestWatchers):
 
     def makeOne(self):
@@ -74,7 +77,7 @@ class TestWatchersDefaultDestroyed(TestWatchers):
         del l
         return core.loop(default=True)
 
-@skipOnLibuv("Tests for libev-only functions")
+@greentest.skipOnLibuv("Tests for libev-only functions")
 class TestLibev(unittest.TestCase):
 
     def test_flags_conversion(self):
@@ -98,4 +101,4 @@ class TestEvents(unittest.TestCase):
         self.assertEqual(repr(core.EVENTS), 'gevent.core.EVENTS')
 
 if __name__ == '__main__':
-    main()
+    greentest.main()

src/greentest/test__example_portforwarder.py

-from __future__ import print_function
+from __future__ import print_function, absolute_import
 from gevent import monkey; monkey.patch_all(subprocess=True)
 import signal
 import sys
@@ -8,9 +8,10 @@ from time import sleep
 import gevent
 from gevent.server import StreamServer
 
+import greentest
 from greentest import util
 
-
+@greentest.skipOnLibuvOnCIOnPyPy("Timing issues sometimes lead to connection refused")
 class Test(util.TestServer):
     server = 'portforwarder.py'
     args = ['127.0.0.1:10011', '127.0.0.1:10012']
@@ -62,5 +63,4 @@ class Test(util.TestServer):
 
 
 if __name__ == '__main__':
-    from greentest import main
-    main()
+    greentest.main()

src/greentest/test__example_udp_client.py

@@ -14,7 +14,7 @@ class Test_udp_client(TestCase):
             log.append(message)
             server.sendto(b'reply-from-server', address)
 
-        server = DatagramServer('127.0.0.1:9000', handle)
+        server = DatagramServer('127.0.0.1:9001', handle)
         server.start()
         try:
             run([sys.executable, '-W', 'ignore', '-u', 'udp_client.py', 'Test_udp_client'],

src/greentest/test__greenio.py

@@ -16,12 +16,15 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
+import sys
 
-from greentest import TestCase, main, tcp_listener
-from greentest import skipOnPyPy
 import gevent
 from gevent import socket
-import sys
+
+from greentest import TestCase, main, tcp_listener
+from greentest import skipOnPyPy
+from greentest import params
+
 
 
 PYPY = hasattr(sys, 'pypy_version_info')
@@ -45,7 +48,7 @@ class TestGreenIo(TestCase):
             # verify that the makefile and the socket are truly independent
             # by closing the socket prior to using the made file
             try:
-                conn, addr = listener.accept()
+                conn, _ = listener.accept()
                 fd = conn.makefile(mode='wb')
                 conn.close()
                 fd.write(b'hello\n')
@@ -59,7 +62,7 @@ class TestGreenIo(TestCase):
             # verify that the makefile and the socket are truly independent
             # by closing the made file and then sending a character
             try:
-                conn, addr = listener.accept()
+                conn, _ = listener.accept()
                 fd = conn.makefile(mode='wb')
                 fd.write(b'hello')
                 fd.close()
@@ -71,27 +74,25 @@ class TestGreenIo(TestCase):
                 listener.close()
 
         def did_it_work(server):
-            client = socket.create_connection(('127.0.0.1', server.getsockname()[1]))
+            client = socket.create_connection((params.DEFAULT_CONNECT, server.getsockname()[1]))
             fd = client.makefile(mode='rb')
             client.close()
-            assert fd.readline() == b'hello\n'
-            assert fd.read() == b''
+            self.assertEqual(fd.readline(), b'hello\n')
+            self.assertFalse(fd.read())
             fd.close()
 
-        server = tcp_listener(('0.0.0.0', 0))
+        server = tcp_listener()
         server_greenlet = gevent.spawn(accept_close_early, server)
         did_it_work(server)
         server_greenlet.kill()
 
-        server = tcp_listener(('0.0.0.0', 0))
+        server = tcp_listener()
         server_greenlet = gevent.spawn(accept_close_late, server)
         did_it_work(server)
         server_greenlet.kill()
 
     @skipOnPyPy("GC is different")
     def test_del_closes_socket(self):
-        timer = gevent.Timeout.start_new(0.5)
-
         def accept_once(listener):
             # delete/overwrite the original conn
             # object, only keeping the file object around
@@ -113,15 +114,15 @@ class TestGreenIo(TestCase):
                 if oconn is not None:
                     oconn.close()
 
-        server = tcp_listener(('0.0.0.0', 0))
+        server = tcp_listener()
         gevent.spawn(accept_once, server)
-        client = socket.create_connection(('127.0.0.1', server.getsockname()[1]))
-        fd = client.makefile()
-        client.close()
-        assert fd.read() == 'hello\n'
-        assert fd.read() == ''
+        client = socket.create_connection((params.DEFAULT_CONNECT, server.getsockname()[1]))
+        with gevent.Timeout.start_new(0.5):
+            fd = client.makefile()
+            client.close()
+            self.assertEqual(fd.read(), 'hello\n')
+            self.assertEqual(fd.read(), '')
 
-        timer.close()
 
 if __name__ == '__main__':
     main()

src/greentest/test__greenness.py

@@ -23,9 +23,11 @@
 To do that spawn a green server and then access it using a green socket.
 If either operation blocked the whole script would block and timeout.
 """
-import greentest
 from gevent import monkey
 monkey.patch_all()
+
+import greentest
+
 try:
     import urllib2
 except ImportError:
@@ -34,33 +36,37 @@ try:
     import BaseHTTPServer
 except ImportError:
     from http import server as BaseHTTPServer
+
 import gevent
+from greentest import params
 
 
 class TestGreenness(greentest.TestCase):
     check_totalrefcount = False
 
+    def setUp(self):
+        server_address = params.DEFAULT_BIND_ADDR_TUPLE
+        BaseHTTPServer.BaseHTTPRequestHandler.protocol_version = "HTTP/1.0"
+        self.httpd = BaseHTTPServer.HTTPServer(server_address, BaseHTTPServer.BaseHTTPRequestHandler)
+        self.httpd.request_count = 0
+
+    def tearDown(self):
+        self.httpd.server_close()
+        self.httpd = None
+
     def serve(self):
         self.httpd.handle_request()
         self.httpd.request_count += 1
 
     def test_urllib2(self):
-        server_address = ('', 0)
-        BaseHTTPServer.BaseHTTPRequestHandler.protocol_version = "HTTP/1.0"
-        self.httpd = BaseHTTPServer.HTTPServer(server_address, BaseHTTPServer.BaseHTTPRequestHandler)
-        self.httpd.request_count = 0
         server = gevent.spawn(self.serve)
 
         port = self.httpd.socket.getsockname()[1]
-        try:
+        with self.assertRaises(urllib2.HTTPError) as exc:
             urllib2.urlopen('http://127.0.0.1:%s' % port)
-            assert False, 'should not get there'
-        except urllib2.HTTPError as ex:
-            assert ex.code == 501, repr(ex)
+        self.assertEqual(exc.exception.code, 501)
         server.get(0.01)
         self.assertEqual(self.httpd.request_count, 1)
-        self.httpd.server_close()
-        self.httpd = None
 
 
 if __name__ == '__main__':