[2.7] bpo-33570: TLS 1.3 ciphers for OpenSSL 1.1.1 (GH-6976) (GH-8760) (GH-10607)

Change TLS 1.3 cipher suite settings for compatibility with OpenSSL
1.1.1-pre6 and newer. OpenSSL 1.1.1 will have TLS 1.3 cipers enabled by
default.

Also update multissltests to test with latest OpenSSL.

Signed-off-by: Christian Heimes <christian@python.org>.
(cherry picked from commit 3e630c541b35c96bfe5619165255e559f577ee71)
Co-authored-by: Christian Heimes <christian@python.org>

Doc/library/ssl.rst

@@ -294,11 +294,6 @@ purposes.
 
      3DES was dropped from the default cipher string.
 
-   .. versionchanged:: 2.7.15
-
-     TLS 1.3 cipher suites TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384,
-     and TLS_CHACHA20_POLY1305_SHA256 were added to the default cipher string.
-
 .. function:: _https_verify_certificates(enable=True)
 
    Specifies whether or not server certificates are verified when creating
@@ -1179,6 +1174,9 @@ to speed up repeated connections from the same clients.
       when connected, the :meth:`SSLSocket.cipher` method of SSL sockets will
       give the currently selected cipher.
 
+      OpenSSL 1.1.1 has TLS 1.3 cipher suites enabled by default. The suites
+      cannot be disabled with :meth:`~SSLContext.set_ciphers`.
+
 .. method:: SSLContext.set_alpn_protocols(protocols)
 
    Specify which protocols the socket should advertise during the SSL/TLS

Lib/test/test_ssl.py

@@ -2796,19 +2796,24 @@ else:
                     sock.do_handshake()
                 self.assertEqual(cm.exception.errno, errno.ENOTCONN)
 
-        def test_default_ciphers(self):
-            context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
-            try:
-                # Force a set of weak ciphers on our client context
-                context.set_ciphers("DES")
-            except ssl.SSLError:
-                self.skipTest("no DES cipher available")
-            with ThreadedEchoServer(CERTFILE,
-                                    ssl_version=ssl.PROTOCOL_SSLv23,
-                                    chatty=False) as server:
-                with closing(context.wrap_socket(socket.socket())) as s:
-                    with self.assertRaises(ssl.SSLError):
-                        s.connect((HOST, server.port))
+        def test_no_shared_ciphers(self):
+            server_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+            server_context.load_cert_chain(SIGNED_CERTFILE)
+            client_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+            client_context.verify_mode = ssl.CERT_REQUIRED
+            client_context.check_hostname = True
+
+            # OpenSSL enables all TLS 1.3 ciphers, enforce TLS 1.2 for test
+            client_context.options |= ssl.OP_NO_TLSv1_3
+            # Force different suites on client and master
+            client_context.set_ciphers("AES128")
+            server_context.set_ciphers("AES256")
+            with ThreadedEchoServer(context=server_context) as server:
+                s = client_context.wrap_socket(
+                        socket.socket(),
+                        server_hostname="localhost")
+                with self.assertRaises(ssl.SSLError):
+                    s.connect((HOST, server.port))
             self.assertIn("no shared cipher", str(server.conn_errors[0]))
 
         def test_version_basic(self):
@@ -2839,9 +2844,9 @@ else:
                 with context.wrap_socket(socket.socket()) as s:
                     s.connect((HOST, server.port))
                     self.assertIn(s.cipher()[0], [
-                        'TLS13-AES-256-GCM-SHA384',
-                        'TLS13-CHACHA20-POLY1305-SHA256',
-                        'TLS13-AES-128-GCM-SHA256',
+                        'TLS_AES_256_GCM_SHA384',
+                        'TLS_CHACHA20_POLY1305_SHA256',
+                        'TLS_AES_128_GCM_SHA256',
                     ])
 
         @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL")

Misc/NEWS.d/next/Library/2018-05-18-21-50-47.bpo-33570.7CZy4t.rst

+Change TLS 1.3 cipher suite settings for compatibility with OpenSSL
+1.1.1-pre6 and newer. OpenSSL 1.1.1 will have TLS 1.3 cipers enabled by
+default.

Tools/ssl/multissltests.py

@@ -41,15 +41,14 @@ import tarfile
 log = logging.getLogger("multissl")
 
 OPENSSL_OLD_VERSIONS = [
-     "0.9.8zc",
-     "0.9.8zh",
      "1.0.1u",
+     "1.0.2o",
 ]
 
 OPENSSL_RECENT_VERSIONS = [
-     "1.0.2",
-     "1.0.2l",
-     "1.1.0f",
+     "1.0.2p",
+     "1.1.0i",
+#     "1.1.1",
 ]
 
 LIBRESSL_OLD_VERSIONS = [
@@ -59,13 +58,15 @@ LIBRESSL_OLD_VERSIONS = [
 
 LIBRESSL_RECENT_VERSIONS = [
     "2.5.5",
-    "2.6.4",
-    "2.7.1",
+    "2.6.5",
+    "2.7.4",
 ]
 
 # store files in ../multissl
-HERE = os.path.abspath(os.getcwd())
-MULTISSL_DIR = os.path.abspath(os.path.join(HERE, '..', 'multissl'))
+HERE = os.path.dirname(os.path.abspath(__file__))
+PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..'))
+MULTISSL_DIR = os.path.abspath(os.path.join(PYTHONROOT, '..', 'multissl'))
+
 
 parser = argparse.ArgumentParser(
     prog='multissl',
@@ -77,7 +78,7 @@ parser = argparse.ArgumentParser(
 parser.add_argument(
     '--debug',
     action='store_true',
-    help="Enable debug mode",
+    help="Enable debug logging",
 )
 parser.add_argument(
     '--disable-ancient',
@@ -120,32 +121,49 @@ parser.add_argument(
     help="Disable network tests."
 )
 parser.add_argument(
-    '--compile-only',
-    action='store_true',
-    help="Don't run tests, only compile _ssl.c and _hashopenssl.c."
+    '--steps',
+    choices=['library', 'modules', 'tests'],
+    default='tests',
+    help=(
+        "Which steps to perform. 'library' downloads and compiles OpenSSL "
+        "or LibreSSL. 'module' also compiles Python modules. 'tests' builds "
+        "all and runs the test suite."
+    )
 )
 
+parser.add_argument(
+    '--force',
+    action='store_true',
+    dest='force',
+    help="Force build and installation."
+)
+parser.add_argument(
+    '--keep-sources',
+    action='store_true',
+    dest='keep_sources',
+    help="Keep original sources for debugging."
+)
 
 class AbstractBuilder(object):
     library = None
     url_template = None
     src_template = None
     build_template = None
+    install_target = 'install'
 
     module_files = ("Modules/_ssl.c",
                     "Modules/_hashopenssl.c")
     module_libs = ("_ssl", "_hashlib")
 
-    def __init__(self, version, compile_args=(),
-                 basedir=MULTISSL_DIR):
+    def __init__(self, version, args):
         self.version = version
-        self.compile_args = compile_args
+        self.args = args
         # installation directory
         self.install_dir = os.path.join(
-            os.path.join(basedir, self.library.lower()), version
+            os.path.join(args.base_directory, self.library.lower()), version
         )
         # source file
-        self.src_dir = os.path.join(basedir, 'src')
+        self.src_dir = os.path.join(args.base_directory, 'src')
         self.src_file = os.path.join(
             self.src_dir, self.src_template.format(version))
         # build directory (removed after install)
@@ -252,21 +270,29 @@ class AbstractBuilder(object):
     def _build_src(self):
         """Now build openssl"""
         log.info("Running build in {}".format(self.build_dir))
-        cwd = self.build_dir
-        cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)]
-        cmd.extend(self.compile_args)
+        cmd = [
+            "./config",
+            "shared", "--debug",
+            "--prefix={}".format(self.install_dir)
+        ]
+        env = os.environ.copy()
+        # set rpath
+        env["LD_RUN_PATH"] = self.lib_dir
         self._subprocess_call(cmd, cwd=cwd)
         # Old OpenSSL versions do not support parallel builds.
         self._subprocess_call(["make", "-j1"], cwd=cwd)
 
-    def _make_install(self, remove=True):
-        self._subprocess_call(["make", "-j1", "install"], cwd=self.build_dir)
-        if remove:
+    def _make_install(self):
+        self._subprocess_call(
+            ["make", "-j1", self.install_target],
+            cwd=self.build_dir
+        )
+        if not self.args.keep_sources:
             shutil.rmtree(self.build_dir)
 
     def install(self):
         log.info(self.openssl_cli)
-        if not self.has_openssl:
+        if not self.has_openssl or self.args.force:
             if not self.has_src:
                 self._download_src()
             else:
@@ -332,6 +358,8 @@ class BuildOpenSSL(AbstractBuilder):
     url_template = "https://www.openssl.org/source/openssl-{}.tar.gz"
     src_template = "openssl-{}.tar.gz"
     build_template = "openssl-{}"
+    # only install software, skip docs
+    install_target = 'install_sw'
 
 
 class BuildLibreSSL(AbstractBuilder):
@@ -370,57 +398,64 @@ def main():
 
     start = datetime.now()
 
-    for name in ['python', 'setup.py', 'Modules/_ssl.c']:
-        if not os.path.isfile(name):
+    if args.steps in {'modules', 'tests'}:
+        for name in ['setup.py', 'Modules/_ssl.c']:
+            if not os.path.isfile(os.path.join(PYTHONROOT, name)):
+                parser.error(
+                    "Must be executed with ./python from CPython build dir"
+                )
+        if not os.path.samefile('python', sys.executable):
             parser.error(
-                "Must be executed from CPython build dir"
-            )
-    if not os.path.samefile('python', sys.executable):
-        parser.error(
             "Must be executed with ./python from CPython build dir"
         )
 
-    # check for configure and run make
-    configure_make()
+        # check for configure and run make
+        configure_make()
 
     # download and register builder
     builds = []
 
     for version in args.openssl:
-        build = BuildOpenSSL(version)
+        build = BuildOpenSSL(
+            version,
+            args
+        )
         build.install()
         builds.append(build)
 
     for version in args.libressl:
-        build = BuildLibreSSL(version)
+        build = BuildLibreSSL(
+            version,
+            args
+        )
         build.install()
         builds.append(build)
 
-    for build in builds:
-        try:
-            build.recompile_pymods()
-            build.check_pyssl()
-            if not args.compile_only:
-                build.run_python_tests(
-                    tests=args.tests,
-                    network=args.network,
-                )
-        except Exception as e:
-            log.exception("%s failed", build)
-            print("{} failed: {}".format(build, e), file=sys.stderr)
-            sys.exit(2)
-
-    print("\n{} finished in {}".format(
-        "Tests" if not args.compile_only else "Builds",
-        datetime.now() - start
-    ))
+    if args.steps in {'modules', 'tests'}:
+        for build in builds:
+            try:
+                build.recompile_pymods()
+                build.check_pyssl()
+                if args.steps == 'tests':
+                    build.run_python_tests(
+                        tests=args.tests,
+                        network=args.network,
+                    )
+            except Exception as e:
+                log.exception("%s failed", build)
+                print("{} failed: {}".format(build, e), file=sys.stderr)
+                sys.exit(2)
+
+    log.info("\n{} finished in {}".format(
+            args.steps.capitalize(),
+            datetime.now() - start
+        ))
     print('Python: ', sys.version)
-    if args.compile_only:
-        print('Build only')
-    elif args.tests:
-        print('Executed Tests:', ' '.join(args.tests))
-    else:
-        print('Executed all SSL tests.')
+    if args.steps == 'tests':
+        if args.tests:
+            print('Executed Tests:', ' '.join(args.tests))
+        else:
+            print('Executed all SSL tests.')
 
     print('OpenSSL / LibreSSL versions:')
     for build in builds: