Additional tests for timeout/exception handling of ssl sockets; make behaviour...

Additional tests for timeout/exception handling of ssl sockets; make behaviour consistent with stdlib (sendall does what python 3.5 does); fixes #317

.pep8

 [pep8]
 ignore=E702,E265,E402,E731,E266,E261,W503,E129
 max_line_length=160
-exclude=.eggs,.tox,.git,build,2.6,2.7,2.7pypy,3.3,3.5,test_support.py,test_queue.py,patched_tests_setup.py,test_threading_2.py,lock_tests.py,_sslgte279.py,3.4
+exclude=.runtimes,.eggs,.tox,.git,build,2.6,2.7,2.7pypy,3.3,3.5,test_support.py,test_queue.py,patched_tests_setup.py,test_threading_2.py,lock_tests.py,_sslgte279.py,3.4

changelog.rst

@@ -7,7 +7,16 @@
 1.1rc2 (unreleased)
 ===================
 
-- TBD
+- Exceptions raised by gevent's SSL sockets are more consistent with
+  the standard library (e.g., gevent's Python 3 SSL sockets raise
+  :exc:`socket.timeout` instead of :exc:`ssl.SSLError`, a change
+  introduced in Python 3.2).
+- Python 2: gevent's socket's ``sendall`` method could ignore timeouts
+  in some cases. The timeout now refers to the total time taken by
+  ``sendall``.
+- gevent's SSL socket's ``sendall`` method should no longer raise ``SSL3_WRITE_PENDING``
+  in rare cases when sending large buffers. Reported in :issue:`317`.
+
 
 1.1rc1 (Nov 14, 2015)
 =====================

gevent/_socket2.py

@@ -341,13 +341,16 @@ class socket(object):
             chunk = data_memory[data_sent:]
             if timeleft is None:
                 data_sent += self.send(chunk, flags)
+            elif timeleft <= 0:
+                # Check before sending to guarantee a check
+                # happens even if each chunk successfully sends its data
+                # (especially important for SSL sockets since they have large
+                # buffers)
+                raise timeout('timed out')
             else:
                 data_sent += self.send(chunk, flags, timeout=timeleft)
-                if data_sent >= len_data_memory:
-                    return
                 timeleft = end - time.time()
-                if timeleft <= 0:
-                    raise timeout('timed out')
+
         return timeleft
 
     def sendall(self, data, flags=0):

gevent/_ssl2.py

@@ -19,6 +19,7 @@ import sys
 import errno
 from gevent.socket import socket, _fileobject, timeout_default
 from gevent.socket import error as socket_error, EWOULDBLOCK
+from gevent.socket import timeout as _socket_timeout
 from gevent.hub import string_types, PYPY
 
 try:
@@ -193,6 +194,17 @@ class SSLSocket(socket):
             return socket.send(self, data, flags, timeout)
     # is it possible for sendall() to send some data without encryption if another end shut down SSL?
 
+    def sendall(self, data, flags=0):
+        try:
+            socket.sendall(self, data)
+        except _socket_timeout as ex:
+            if self.timeout == 0.0:
+                # Python 2 simply *hangs* in this case, which is bad, but
+                # Python 3 raises SSLWantWriteError. We do the same.
+                raise SSLError(SSL_ERROR_WANT_WRITE)
+            # Convert the socket.timeout back to the sslerror
+            raise SSLError(*ex.args)
+
     def sendto(self, *args):
         if self._sslobj:
             raise ValueError("sendto not allowed on instances of %s" %

gevent/_ssl3.py

@@ -15,7 +15,7 @@ _ssl = __ssl__._ssl
 import errno
 from gevent.socket import socket, timeout_default
 from gevent.socket import error as socket_error
-
+from gevent.socket import timeout as _socket_timeout
 
 __implements__ = ['SSLContext',
                   'SSLSocket',
@@ -273,14 +273,14 @@ class SSLSocket(socket):
                 raise ValueError(
                     "non-zero flags not allowed in calls to sendall() on %s" %
                     self.__class__)
-            amount = len(data)
-            count = 0
-            while (count < amount):
-                v = self.send(data[count:])
-                count += v
-            return amount
-        else:
+
+        try:
             return socket.sendall(self, data, flags)
+        except _socket_timeout:
+            if self.timeout == 0.0:
+                # Raised by the stdlib on non-blocking sockets
+                raise SSLWantWriteError("The operation did not complete (write)")
+            raise
 
     def recv(self, buflen=1024, flags=0):
         self._checkClosed()
@@ -428,9 +428,13 @@ class SSLSocket(socket):
         return self._sslobj.tls_unique_cb()
 
 
-_SSLErrorReadTimeout = SSLError('The read operation timed out')
-_SSLErrorWriteTimeout = SSLError('The write operation timed out')
-_SSLErrorHandshakeTimeout = SSLError('The handshake operation timed out')
+# Python 3.2 onwards raise normal timeout errors, not SSLError.
+# See https://bugs.python.org/issue10272
+t = __import__('socket').timeout
+_SSLErrorReadTimeout = t('The read operation timed out')
+_SSLErrorWriteTimeout = t('The write operation timed out')
+_SSLErrorHandshakeTimeout = t('The handshake operation timed out')
+del t
 
 
 def wrap_socket(sock, keyfile=None, certfile=None,

gevent/_sslgte279.py

@@ -15,6 +15,7 @@ _ssl = __ssl__._ssl
 import errno
 from gevent.socket import socket, timeout_default
 from gevent.socket import error as socket_error
+from gevent.socket import timeout as _socket_timeout
 from gevent.hub import PYPY
 
 __implements__ = ['SSLContext',
@@ -364,29 +365,34 @@ class SSLSocket(socket):
         else:
             return self._sslobj.compression()
 
+    def __check_flags(self, meth, flags):
+        if flags != 0:
+            raise ValueError(
+                "non-zero flags not allowed in calls to %s on %s" %
+                (meth, self.__class__))
+
     def send(self, data, flags=0, timeout=timeout_default):
         self._checkClosed()
+        self.__check_flags('send', flags)
+
         if timeout is timeout_default:
             timeout = self.timeout
-        if self._sslobj:
-            if flags != 0:
-                raise ValueError(
-                    "non-zero flags not allowed in calls to send() on %s" %
-                    self.__class__)
-            while True:
-                try:
-                    return self._sslobj.write(data)
-                except SSLWantReadError:
-                    if self.timeout == 0.0:
-                        return 0
-                    self._wait(self._read_event)
-                except SSLWantWriteError:
-                    if self.timeout == 0.0:
-                        return 0
-                    self._wait(self._write_event)
-        else:
+
+        if not self._sslobj:
             return socket.send(self, data, flags, timeout)
 
+        while True:
+            try:
+                return self._sslobj.write(data)
+            except SSLWantReadError:
+                if self.timeout == 0.0:
+                    return 0
+                self._wait(self._read_event)
+            except SSLWantWriteError:
+                if self.timeout == 0.0:
+                    return 0
+                self._wait(self._write_event)
+
     def sendto(self, data, flags_or_addr, addr=None):
         self._checkClosed()
         if self._sslobj:
@@ -405,19 +411,17 @@ class SSLSocket(socket):
 
     def sendall(self, data, flags=0):
         self._checkClosed()
-        if self._sslobj:
-            if flags != 0:
-                raise ValueError(
-                    "non-zero flags not allowed in calls to sendall() on %s" %
-                    self.__class__)
-            amount = len(data)
-            count = 0
-            while (count < amount):
-                v = self.send(data[count:])
-                count += v
-            return amount
-        else:
-            return socket.sendall(self, data, flags)
+        self.__check_flags('sendall', flags)
+
+        try:
+            socket.sendall(self, data)
+        except _socket_timeout as ex:
+            if self.timeout == 0.0:
+                # Python 2 simply *hangs* in this case, which is bad, but
+                # Python 3 raises SSLWantWriteError. We do the same.
+                raise SSLWantWriteError("The operation did not complete (write)")
+            # Convert the socket.timeout back to the sslerror
+            raise SSLError(*ex.args)
 
     def recv(self, buflen=1024, flags=0):
         self._checkClosed()

greentest/test__socket.py

@@ -146,11 +146,13 @@ class TestTCP(greentest.TestCase):
     # On Windows send() accepts whatever is thrown at it
     if sys.platform != 'win32':
 
+        _test_sendall_timeout_check_time = True
+        # Travis-CI container infrastructure is configured with
+        # large socket buffers, at least 2MB, as-of Jun 3, 2015,
+        # so we must be sure to send more data than that.
+        _test_sendall_data = b'hello' * 1000000
+
         def test_sendall_timeout(self):
-            # Travis-CI container infrastructure is configured with
-            # large socket buffers, at least 2MB, as-of Jun 3, 2015,
-            # so we must be sure to send more data than that.
-            data_sent = b'hello' * 1000000
             client_sock = []
             acceptor = Thread(target=lambda: client_sock.append(self.listener.accept()))
             client = self.create_connection()
@@ -159,9 +161,10 @@ class TestTCP(greentest.TestCase):
             client.settimeout(0.1)
             start = time.time()
             try:
-                self.assertRaises(self.TIMEOUT_ERROR, client.sendall, data_sent)
-                took = time.time() - start
-                assert 0.1 - 0.01 <= took <= 0.1 + 0.1, took
+                self.assertRaises(self.TIMEOUT_ERROR, client.sendall, self._test_sendall_data)
+                if self._test_sendall_timeout_check_time:
+                    took = time.time() - start
+                    assert 0.09 <= took <= 0.2, took
             finally:
                 acceptor.join()
                 client.close()

greentest/test__ssl.py

 from gevent import monkey; monkey.patch_all()
 import os
+import sys
 import socket
 import greentest
-from test__socket import TestTCP
+# Be careful not to have TestTCP as a bare attribute in this module,
+# even aliased, to avoid running duplicate tests
+import test__socket
 import ssl
+import time
 
 
-class TestSSL(TestTCP):
+class TestSSL(test__socket.TestTCP):
 
     certfile = os.path.join(os.path.dirname(__file__), 'test_server.crt')
     privfile = os.path.join(os.path.dirname(__file__), 'test_server.key')
-    # Python 2.x has socket.sslerror, which we need to be sure is an alias for
-    # ssl.SSLError. That's gone in Py3 though.
-    TIMEOUT_ERROR = getattr(socket, 'sslerror', ssl.SSLError)
+    # Python 2.x has socket.sslerror (which  is an alias for
+    # ssl.SSLError); That's gone in Py3 though. In Python 2, most timeouts are raised
+    # as SSLError, but Python 3 raises the normal socket.timeout instead. So this has
+    # the effect of making TIMEOUT_ERROR be SSLError on Py2 and socket.timeout on Py3
+    # See https://bugs.python.org/issue10272
+    TIMEOUT_ERROR = getattr(socket, 'sslerror', socket.timeout)
 
     def setUp(self):
         greentest.TestCase.setUp(self)
-        self.listener, raw_listener = ssl_listener(('127.0.0.1', 0), self.privfile, self.certfile)
+        self.listener, _raw_listener = ssl_listener(('127.0.0.1', 0), self.privfile, self.certfile)
         self.port = self.listener.getsockname()[1]
 
     def create_connection(self):
         return ssl.wrap_socket(super(TestSSL, self).create_connection())
 
-    def test_sendall_timeout(self):
-        pass
+    if not sys.platform.startswith('win32'):
 
-del TestTCP
+        # The SSL library can take a long time to buffer the large amount of data we're trying
+        # to send, so we can't compare to the timeout values
+        _test_sendall_timeout_check_time = False
+
+        # The SSL layer has extra buffering, so test_sendall needs
+        # to send a very large amount to make it timeout
+        _test_sendall_data = data_sent = b'hello' * 100000000
+
+        def test_sendall_timeout0(self):
+            # Issue #317: SSL_WRITE_PENDING in some corner cases
+
+            server_sock = []
+            acceptor = test__socket.Thread(target=lambda: server_sock.append(self.listener.accept()))
+            client = self.create_connection()
+            client.setblocking(False)
+            try:
+                # Python 3 raises ssl.SSLWantWriteError; Python 2 simply *hangs*
+                # on non-blocking sockets because it's a simple loop around
+                # send(). Python 2.6 doesn't have SSLWantWriteError
+                expected = getattr(ssl, 'SSLWantWriteError', ssl.SSLError)
+                self.assertRaises(expected, client.sendall, self._test_sendall_data)
+            finally:
+                acceptor.join()
+                client.close()
+                server_sock[0][0].close()
 
 
 def ssl_listener(address, private_key, certificate):