bpo-32947: OpenSSL 1.1.1-pre1 / TLS 1.3 fixes (#5663)

* bpo-32947: OpenSSL 1.1.1-pre1 / TLS 1.3 fixes

Misc fixes and workarounds for compatibility with OpenSSL 1.1.1-pre1 and
TLS 1.3 support. With OpenSSL 1.1.1, Python negotiates TLS 1.3 by
default. Some test cases only apply to TLS 1.2. Other tests currently
fail because the threaded or async test servers stop after failure.

I'm going to address these issues when OpenSSL 1.1.1 reaches beta.

OpenSSL 1.1.1 has added a new option OP_ENABLE_MIDDLEBOX_COMPAT for TLS
1.3. The feature is enabled by default for maximum compatibility with
broken middle boxes. Users should be able to disable the hack and CPython's test suite needs
it to verify default options.
Signed-off-by: Christian Heimes <christian@python.org>

Doc/library/ssl.rst

@@ -831,6 +831,15 @@ Constants
 
    .. versionadded:: 3.3
 
+.. data:: OP_ENABLE_MIDDLEBOX_COMPAT
+
+   Send dummy Change Cipher Spec (CCS) messages in TLS 1.3 handshake to make
+   a TLS 1.3 connection look more like a TLS 1.2 connection.
+
+   This option is only available with OpenSSL 1.1.1 and later.
+
+   .. versionadded:: 3.8
+
 .. data:: OP_NO_COMPRESSION
 
    Disable compression on the SSL channel.  This is useful if the application

Doc/whatsnew/3.7.rst

@@ -669,6 +669,9 @@ expected hostname in A-label form (``"xn--pythn-mua.org"``), rather
 than the U-label form (``"pythön.org"``). (Contributed by
 Nathaniel J. Smith and Christian Heimes in :issue:`28414`.)
 
+The ssl module has preliminary and experimental support for TLS 1.3 and
+OpenSSL 1.1.1. (Contributed by Christian Heimes in :issue:`32947`,
+:issue:`20995`, :issue:`29136`, and :issue:`30622`)
 
 string
 ------

Lib/test/test_asyncio/utils.py

@@ -74,6 +74,8 @@ def simple_server_sslcontext():
     server_context.load_cert_chain(ONLYCERT, ONLYKEY)
     server_context.check_hostname = False
     server_context.verify_mode = ssl.CERT_NONE
+    # TODO: fix TLSv1.3 support
+    server_context.options |= ssl.OP_NO_TLSv1_3
     return server_context
 
 

Lib/test/test_ftplib.py

@@ -312,6 +312,8 @@ if ssl is not None:
 
         def secure_connection(self):
             context = ssl.SSLContext()
+            # TODO: fix TLSv1.3 support
+            context.options |= ssl.OP_NO_TLSv1_3
             context.load_cert_chain(CERTFILE)
             socket = context.wrap_socket(self.socket,
                                          suppress_ragged_eofs=False,
@@ -908,6 +910,8 @@ class TestTLS_FTPClass(TestCase):
     def test_context(self):
         self.client.quit()
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        # TODO: fix TLSv1.3 support
+        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.check_hostname = False
         ctx.verify_mode = ssl.CERT_NONE
         self.assertRaises(ValueError, ftplib.FTP_TLS, keyfile=CERTFILE,
@@ -940,6 +944,8 @@ class TestTLS_FTPClass(TestCase):
     def test_check_hostname(self):
         self.client.quit()
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        # TODO: fix TLSv1.3 support
+        ctx.options |= ssl.OP_NO_TLSv1_3
         self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED)
         self.assertEqual(ctx.check_hostname, True)
         ctx.load_verify_locations(CAFILE)

Lib/test/test_poplib.py

@@ -153,6 +153,8 @@ class DummyPOP3Handler(asynchat.async_chat):
             if self.tls_active is False:
                 self.push('+OK Begin TLS negotiation')
                 context = ssl.SSLContext()
+                # TODO: fix TLSv1.3 support
+                context.options |= ssl.OP_NO_TLSv1_3
                 context.load_cert_chain(CERTFILE)
                 tls_sock = context.wrap_socket(self.socket,
                                                server_side=True,
@@ -356,6 +358,8 @@ class TestPOP3Class(TestCase):
     def test_stls_context(self):
         expected = b'+OK Begin TLS negotiation'
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        # TODO: fix TLSv1.3 support
+        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.load_verify_locations(CAFILE)
         self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED)
         self.assertEqual(ctx.check_hostname, True)
@@ -396,6 +400,8 @@ class TestPOP3_SSLClass(TestPOP3Class):
 
     def test_context(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        # TODO: fix TLSv1.3 support
+        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.check_hostname = False
         ctx.verify_mode = ssl.CERT_NONE
         self.assertRaises(ValueError, poplib.POP3_SSL, self.server.host,

Misc/NEWS.d/next/Library/2018-02-25-13-06-21.bpo-32947.mqStVW.rst

+Add OP_ENABLE_MIDDLEBOX_COMPAT and test workaround for TLSv1.3 for future
+compatibility with OpenSSL 1.1.1.

Modules/_ssl.c

@@ -5681,6 +5681,10 @@ PyInit__ssl(void)
     PyModule_AddIntConstant(m, "OP_NO_COMPRESSION",
                             SSL_OP_NO_COMPRESSION);
 #endif
+#ifdef SSL_OP_ENABLE_MIDDLEBOX_COMPAT
+    PyModule_AddIntConstant(m, "OP_ENABLE_MIDDLEBOX_COMPAT",
+                            SSL_OP_ENABLE_MIDDLEBOX_COMPAT);
+#endif
 
 #ifdef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT
     PyModule_AddIntConstant(m, "HOSTFLAG_ALWAYS_CHECK_SUBJECT",

Tools/ssl/multissltests.py

@@ -41,24 +41,20 @@ import tarfile
 log = logging.getLogger("multissl")
 
 OPENSSL_OLD_VERSIONS = [
-     # "0.9.8zh",
-     # "1.0.1u",
+    "1.0.2",
 ]
 
 OPENSSL_RECENT_VERSIONS = [
-     "1.0.2",
-     "1.0.2m",
-     "1.1.0g",
+    "1.0.2n",
+    "1.1.0g",
+    "1.1.1-pre1",
 ]
 
 LIBRESSL_OLD_VERSIONS = [
-    # "2.3.10",
-    # "2.4.5",
 ]
 
 LIBRESSL_RECENT_VERSIONS = [
-    "2.5.3",
-    "2.5.5",
+    # "2.6.5",
 ]
 
 # store files in ../multissl