Fix several more socket-related tests under Py3: test__examples, test__os,...

Fix several more socket-related tests under Py3: test__examples, test__os, test__fileobject, test__subprocess_poll, test__server, test__execmodules.

gevent/_socket3.py

@@ -475,7 +475,7 @@ class _fileobject(object):
 
     def flush(self):
         if self._wbuf:
-            data = "".join(self._wbuf)
+            data = b"".join(self._wbuf)
             self._wbuf = []
             self._wbuf_len = 0
             buffer_size = max(self._rbufsize, self.default_bufsize)
@@ -497,13 +497,14 @@ class _fileobject(object):
         return self._sock.fileno()
 
     def write(self, data):
-        data = str(data) # XXX Should really reject non-string non-buffers
+        if not isinstance(data, bytes):
+            raise TypeError("Non-bytes data")
         if not data:
             return
         self._wbuf.append(data)
         self._wbuf_len += len(data)
         if (self._wbufsize == 0 or
-            (self._wbufsize == 1 and '\n' in data) or
+            (self._wbufsize == 1 and b'\n' in data) or
             (self._wbufsize > 1 and self._wbuf_len >= self._wbufsize)):
             self.flush()
 

gevent/fileobject.py

@@ -3,6 +3,7 @@ import sys
 import os
 from gevent.hub import get_hub
 from gevent.hub import integer_types
+from gevent.hub import PY3
 from gevent.socket import EBADF
 from gevent.os import _read, _write, ignored_errors
 from gevent.lock import Semaphore, DummySemaphore
@@ -10,6 +11,12 @@ from gevent.lock import Semaphore, DummySemaphore
 
 PYPY = hasattr(sys, 'pypy_version_info')
 
+if hasattr(sys, 'exc_clear'):
+    def _exc_clear():
+        sys.exc_clear()
+else:
+    def _exc_clear():
+        return
 
 try:
     from fcntl import fcntl
@@ -38,7 +45,10 @@ else:
         SocketAdapter__del__ = None
         noop = None
 
-    from types import UnboundMethodType
+    if PY3:
+        UnboundMethodType = None
+    else:
+        from types import UnboundMethodType
 
     class NA(object):
 
@@ -121,7 +131,7 @@ else:
                     code = ex.args[0]
                     if code not in ignored_errors:
                         raise
-                    sys.exc_clear()
+                    _exc_clear()
                 if bytes_written >= bytes_total:
                     return
                 self.hub.wait(self._write_event)
@@ -134,24 +144,24 @@ else:
                     code = ex.args[0]
                     if code not in ignored_errors:
                         raise
-                    sys.exc_clear()
+                    _exc_clear()
                 else:
                     if not self._translate or not data:
                         return data
                     if self._eat_newline:
                         self._eat_newline = False
-                        if data.startswith('\n'):
+                        if data.startswith(b'\n'):
                             data = data[1:]
                             if not data:
                                 return self.recv(size)
-                    if data.endswith('\r'):
+                    if data.endswith(b'\r'):
                         self._eat_newline = True
                     return self._translate_newlines(data)
                 self.hub.wait(self._read_event)
 
         def _translate_newlines(self, data):
-            data = data.replace("\r\n", "\n")
-            data = data.replace("\r", "\n")
+            data = data.replace(b"\r\n", b"\n")
+            data = data.replace(b"\r", b"\n")
             return data
 
         if not SocketAdapter__del__:
@@ -160,13 +170,15 @@ else:
                 fileno = self._fileno
                 if fileno is not None:
                     close(fileno)
-
-    if SocketAdapter__del__:
+        elif PY3:
+            def __del__(self, close=os.close):
+                SocketAdapter__del__(self, close=close)
+    if SocketAdapter__del__ and not PY3:
         SocketAdapter.__del__ = UnboundMethodType(SocketAdapter__del__, None, SocketAdapter)
 
     class FileObjectPosix(_fileobject):
 
-        def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
+        def __init__(self, fobj=None, mode='rb', bufsize=-1, close=True):
             if isinstance(fobj, integer_types):
                 fileno = fobj
                 fobj = None
@@ -212,13 +224,13 @@ else:
                 raise FileObjectClosed
             return getattr(self._fobj, item)
 
-        if not noop:
+        if not noop or PY3:
 
             def __del__(self):
                 # disable _fileobject's __del__
                 pass
 
-    if noop:
+    if noop and not PY3:
         FileObjectPosix.__del__ = UnboundMethodType(FileObjectPosix, None, noop)
 
 

gevent/server.py

@@ -98,7 +98,7 @@ class StreamServer(BaseServer):
             try:
                 fd, address = sock._accept()
             except BlockingIOError:
-                if sock.timeout == 0.0:
+                if not sock.timeout:
                     return
                 raise
             sock = socket(sock.family, sock.type, sock.proto, fileno=fd)

greentest/test__backdoor.py

@@ -24,24 +24,28 @@ def create_connection(address):
     conn.connect(address)
     return conn
 
+def readline(conn):
+    f = conn.makefile()
+    line = f.readline()
+    f.close()
+    return line
 
 class Test(greentest.TestCase):
 
     def test(self):
         server = backdoor.BackdoorServer(('127.0.0.1', 0))
+        server.start()
 
         def connect():
             conn = create_connection(('127.0.0.1', server.server_port))
             try:
                 read_until(conn, '>>> ')
                 conn.sendall(b'2+2\r\n')
-                with conn.makefile() as f:
-                    line = f.readline()
+                line = readline(conn)
                 self.assertEqual(line.strip(), '4', repr(line))
             finally:
                 conn.close()
 
-        server.start()
         try:
             jobs = [gevent.spawn(connect) for _ in xrange(10)]
             gevent.joinall(jobs, raise_error=True)
@@ -56,8 +60,7 @@ class Test(greentest.TestCase):
             conn = create_connection(('127.0.0.1', server.server_port))
             read_until(conn, '>>> ')
             conn.sendall(b'quit()\r\n')
-            with conn.makefile() as f:
-                line = f.read()
+            line = readline(conn)
             self.assertEqual(line, '')
         finally:
             conn.close()
@@ -70,8 +73,7 @@ class Test(greentest.TestCase):
             conn = create_connection(('127.0.0.1', server.server_port))
             read_until(conn, b'>>> ')
             conn.sendall(b'import sys; sys.exit(0)\r\n')
-            with conn.makefile() as f:
-                line = f.read()
+            line = readline(conn)
             self.assertEqual(line, '')
         finally:
             conn.close()

greentest/test__fileobject.py

@@ -13,7 +13,7 @@ class Test(greentest.TestCase):
     def _test_del(self, **kwargs):
         r, w = os.pipe()
         s = FileObject(w, 'wb')
-        s.write('x')
+        s.write(b'x')
         s.flush()
         if PYPY:
             s.close()
@@ -25,7 +25,7 @@ class Test(greentest.TestCase):
             pass  # expected, because SocketAdapter already closed it
         else:
             raise AssertionError('os.close(%r) must not succeed' % w)
-        self.assertEqual(FileObject(r).read(), 'x')
+        self.assertEqual(FileObject(r).read(), b'x')
 
     def test_del(self):
         self._test_del()
@@ -38,22 +38,22 @@ class Test(greentest.TestCase):
         def test_del_noclose(self):
             r, w = os.pipe()
             s = FileObject(w, 'wb', close=False)
-            s.write('x')
+            s.write(b'x')
             s.flush()
             if PYPY:
                 s.close()
             else:
                 del s
             os.close(w)
-            self.assertEqual(FileObject(r).read(), 'x')
+            self.assertEqual(FileObject(r).read(), b'x')
 
     def test_newlines(self):
         r, w = os.pipe()
-        lines = ['line1\n', 'line2\r', 'line3\r\n', 'line4\r\nline5', '\nline6']
+        lines = [b'line1\n', b'line2\r', b'line3\r\n', b'line4\r\nline5', b'\nline6']
         g = gevent.spawn(writer, FileObject(w, 'wb'), lines)
         try:
             result = FileObject(r, 'rU').read()
-            self.assertEqual('line1\nline2\nline3\nline4\nline5\nline6', result)
+            self.assertEqual(b'line1\nline2\nline3\nline4\nline5\nline6', result)
         finally:
             g.kill()
 
@@ -76,7 +76,7 @@ else:
         def _test_del(self, **kwargs):
             r, w = os.pipe()
             s = SocketAdapter(w)
-            s.sendall('x')
+            s.sendall(b'x')
             if PYPY:
                 s.close()
             else:
@@ -87,7 +87,7 @@ else:
                 pass  # expected, because SocketAdapter already closed it
             else:
                 raise AssertionError('os.close(%r) must not succeed' % w)
-            self.assertEqual(FileObject(r).read(), 'x')
+            self.assertEqual(FileObject(r).read(), b'x')
 
         def test_del(self):
             self._test_del()
@@ -98,10 +98,10 @@ else:
         def test_del_noclose(self):
             r, w = os.pipe()
             s = SocketAdapter(w, close=False)
-            s.sendall('x')
+            s.sendall(b'x')
             del s
             os.close(w)
-            self.assertEqual(FileObject(r).read(), 'x')
+            self.assertEqual(FileObject(r).read(), b'x')
 
 
 if __name__ == '__main__':

greentest/test__issue6.py

@@ -5,7 +5,7 @@ import sys
 if not sys.argv[1:]:
     from subprocess import Popen, PIPE
     p = Popen([sys.executable, __file__, 'subprocess'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
-    out, err = p.communicate('hello world\n')
+    out, err = p.communicate(b'hello world\n')
     code = p.poll()
     assert p.poll() == 0, (out, err, code)
     assert out.strip() == '11 chars.', (out, err, code)

greentest/test__os.py

@@ -32,7 +32,7 @@ class TestOS_tp(TestCase):
         r, w = self.pipe()
         # set nbytes such that for sure it is > maximum pipe buffer
         nbytes = 1000000
-        block = 'x' * 4096
+        block = b'x' * 4096
         buf = buffer_class(block)
         # Lack of "nonlocal" keyword in Python 2.x:
         bytesread = [0]

greentest/test__server.py

 from __future__ import print_function
 import greentest
+from gevent.hub import PY3
 from gevent import socket
 import gevent
 from gevent.server import StreamServer
@@ -21,15 +22,15 @@ class SimpleStreamServer(StreamServer):
                 print('Failed to parse request line: %r' % (request_line, ))
                 raise
             if path == '/ping':
-                client_socket.sendall('HTTP/1.0 200 OK\r\n\r\nPONG')
+                client_socket.sendall(b'HTTP/1.0 200 OK\r\n\r\nPONG')
             elif path in ['/long', '/short']:
-                client_socket.sendall('hello')
+                client_socket.sendall(b'hello')
                 while True:
                     data = client_socket.recv(1)
                     if not data:
                         break
             else:
-                client_socket.sendall('HTTP/1.0 404 WTF?\r\n\r\n')
+                client_socket.sendall(b'HTTP/1.0 404 WTF?\r\n\r\n')
         finally:
             fd.close()
 
@@ -82,10 +83,35 @@ class TestCase(greentest.TestCase):
     def makefile(self, timeout=0.1, bufsize=1):
         sock = socket.socket()
         sock.connect((self.server.server_host, self.server.server_port))
-        fobj = sock.makefile(bufsize=bufsize)
-        fobj._sock.settimeout(timeout)
+
+        if PY3:
+            kwargs = {'buffering': bufsize, 'mode': 'rb'}
+        else:
+            kwargs = {'bufsize': bufsize}
+
+        rconn = sock.makefile(**kwargs)
+        if PY3:
+            # Under Python3, you can't read and write to the same
+            # makefile() opened in r, and r+ is not allowed
+            kwargs['mode'] = 'wb'
+            wconn = sock.makefile(**kwargs)
+            rconn._sock = sock
+            _rconn_close = rconn.close
+            def write(data):
+                if isinstance(data, str):
+                    data = data.encode('ascii')
+                return wconn.write(data)
+            def flush():
+                return wconn.flush()
+            def close():
+                _rconn_close()
+                wconn.close()
+            rconn.write = write
+            rconn.flush = flush
+            rconn.close = close
+        rconn._sock.settimeout(timeout)
         sock.close()
-        return fobj
+        return rconn
 
     def send_request(self, url='/', timeout=0.1, bufsize=1):
         conn = self.makefile(timeout=timeout, bufsize=bufsize)
@@ -128,12 +154,14 @@ class TestCase(greentest.TestCase):
             assert not result, repr(result)
             return
         assert result.startswith('HTTP/1.0 500 Internal Server Error'), repr(result)
+        conn.close()
 
     def assertRequestSucceeded(self, timeout=0.1):
         conn = self.makefile(timeout=timeout)
-        conn.write('GET /ping HTTP/1.0\r\n\r\n')
+        conn.write(b'GET /ping HTTP/1.0\r\n\r\n')
         result = conn.read()
-        assert result.endswith('\r\n\r\nPONG'), repr(result)
+        conn.close()
+        assert result.endswith(b'\r\n\r\nPONG'), repr(result)
 
     def start_server(self):
         self.server.start()
@@ -270,6 +298,7 @@ class TestDefaultSpawn(TestCase):
                     raise
         finally:
             timeout.cancel()
+            conn.close()
         self.stop_server()
 
     def init_server(self):
@@ -319,6 +348,10 @@ class TestPoolSpawn(TestDefaultSpawn):
         self.assertPoolFull()
         self.assertPoolFull()
         short_request._sock.close()
+        if PY3:
+            # We use two makefiles to simulate reading/writing
+            # under py3
+            short_request.close()
         # gevent.http and gevent.wsgi cannot detect socket close, so sleep a little
         # to let /short request finish
         gevent.sleep(0.1)

known_failures.py

@@ -84,27 +84,20 @@ if PY3:
     # No idea / TODO
     FAILING_TESTS += '''
 test__example_udp_server.py
-test__examples.py
 test__pool.py
 FLAKY test___example_servers.py
-test__example_udp_client.py
-test__os.py
 test_threading_2.py
 test__refcount.py
 test__subprocess.py
 test__all__.py
-test__fileobject.py
 test__pywsgi.py
 test__socket_ex.py
 test__example_echoserver.py
-test__subprocess_poll.py
 test__makefile_ref.py
 test__socketpair.py
 test__server_pywsgi.py
 test__core_stat.py
-test__server.py
 test__example_portforwarder.py
-test__execmodules.py
 FLAKY test__greenio.py
 FLAKY test__socket_dns.py
 '''.strip().split('\n')