Issue #26268: Update the prepare_ssl.py script

It can now handle OpenSSL versions 1.0.2e and greater, which don't
include include files in include/.

Note that sources prepared by this script no longer support the old
project files for 2.7; you now have to have Perl available to use
the old build_ssl.py script with sources from svn.python.org.

Misc/NEWS

@@ -263,6 +263,10 @@ Build
 Windows
 -------
 
+- Issue #26268: Update the prepare_ssl.py script to handle OpenSSL releases
+  that don't include the contents of the include directory (that is, 1.0.2e
+  and later).
+
 - Issue #26071: bdist_wininst created binaries fail to start and find
   32bit Python
 

PCbuild/prepare_ssl.py

@@ -21,11 +21,11 @@
 import os
 import re
 import sys
-import shutil
 import subprocess
+from shutil import copy
 
 # Find all "foo.exe" files on the PATH.
-def find_all_on_path(filename, extras = None):
+def find_all_on_path(filename, extras=None):
     entries = os.environ["PATH"].split(os.pathsep)
     ret = []
     for p in entries:
@@ -39,6 +39,7 @@ def find_all_on_path(filename, extras = None):
                 ret.append(fname)
     return ret
 
+
 # Find a suitable Perl installation for OpenSSL.
 # cygwin perl does *not* work.  ActivePerl does.
 # Being a Perl dummy, the simplest way I can check is if the "Win32" package
@@ -61,37 +62,15 @@ def find_working_perl(perls):
         print("NO perl interpreters were found on this machine at all!")
     print(" Please install ActivePerl and ensure it appears on your path")
 
-def create_makefile64(makefile, m32):
-    """Create and fix makefile for 64bit
 
-    Replace 32 with 64bit directories
-    """
-    if not os.path.isfile(m32):
-        return
-    with open(m32) as fin:
-        with open(makefile, 'w') as fout:
-            for line in fin:
-                line = line.replace("=tmp32", "=tmp64")
-                line = line.replace("=out32", "=out64")
-                line = line.replace("=inc32", "=inc64")
-                # force 64 bit machine
-                line = line.replace("MKLIB=lib", "MKLIB=lib /MACHINE:X64")
-                line = line.replace("LFLAGS=", "LFLAGS=/MACHINE:X64 ")
-                # don't link against the lib on 64bit systems
-                line = line.replace("bufferoverflowu.lib", "")
-                fout.write(line)
-    os.unlink(m32)
-
-def create_asms(makefile):
+def create_asms(makefile, tmp_d):
     #create a custom makefile out of the provided one
     asm_makefile = os.path.splitext(makefile)[0] + '.asm.mak'
-    with open(makefile) as fin:
-        with open(asm_makefile, 'w') as fout:
+    with open(makefile) as fin, open(asm_makefile, 'w') as fout:
         for line in fin:
             # Keep everything up to the install target (it's convenient)
             if line.startswith('install: all'):
                 break
-                else:
             fout.write(line)
         asms = []
         for line in fin:
@@ -105,38 +84,25 @@ def create_asms(makefile):
         fout.write('asms: $(TMP_D) ')
         fout.write(' '.join(asms))
         fout.write('\n')
-
-    os.system('nmake /f {} PERL=perl asms'.format(asm_makefile))
-    os.unlink(asm_makefile)
-
+    os.system('nmake /f {} PERL=perl TMP_D={} asms'.format(asm_makefile, tmp_d))
 
 
-def fix_makefile(makefile):
-    """Fix some stuff in all makefiles
-    """
-    if not os.path.isfile(makefile):
-        return
+def copy_includes(makefile, suffix):
+    dir = 'include'+suffix+'\\openssl'
+    os.makedirs(dir, exist_ok=True)
     copy_if_different = r'$(PERL) $(SRC_D)\util\copy-if-different.pl'
     with open(makefile) as fin:
-        lines = fin.readlines()
-    with open(makefile, 'w') as fout:
-        for line in lines:
-            if line.startswith("PERL="):
-                continue
-            if line.startswith("CP="):
-                line = "CP=copy\n"
-            if line.startswith("MKDIR="):
-                line = "MKDIR=mkdir\n"
-            if line.startswith("CFLAG="):
-                line = line.strip()
-                for algo in ("RC5", "MDC2", "IDEA"):
-                    noalgo = " -DOPENSSL_NO_%s" % algo
-                    if noalgo not in line:
-                        line = line + noalgo
-                line = line + '\n'
+        for line in fin:
             if copy_if_different in line:
-                line = line.replace(copy_if_different, 'copy /Y')
-            fout.write(line)
+                perl, script, src, dest = line.split()
+                if not '$(INCO_D)' in dest:
+                    continue
+                # We're in the root of the source tree
+                src = src.replace('$(SRC_D)', '.').strip('"')
+                dest = dest.strip('"').replace('$(INCO_D)', dir)
+                print('copying', src, 'to', dest)
+                copy(src, dest)
+
 
 def run_configure(configure, do_script):
     print("perl Configure "+configure+" no-idea no-mdc2")
@@ -144,61 +110,37 @@ def run_configure(configure, do_script):
     print(do_script)
     os.system(do_script)
 
-def cmp(f1, f2):
-    bufsize = 1024 * 8
-    with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2:
-        while True:
-            b1 = fp1.read(bufsize)
-            b2 = fp2.read(bufsize)
-            if b1 != b2:
-                return False
-            if not b1:
-                return True
-
-def copy(src, dst):
-    if os.path.isfile(dst) and cmp(src, dst):
-        return
-    shutil.copy(src, dst)
 
 def prep(arch):
+    makefile_template = "ms\\nt{}.mak"
+    generated_makefile = makefile_template.format('')
     if arch == "x86":
         configure = "VC-WIN32"
         do_script = "ms\\do_nasm"
-        makefile="ms\\nt.mak"
-        m32 = makefile
-        dirsuffix = "32"
+        suffix = "32"
     elif arch == "amd64":
         configure = "VC-WIN64A"
         do_script = "ms\\do_win64a"
-        makefile = "ms\\nt64.mak"
-        m32 = makefile.replace('64', '')
-        dirsuffix = "64"
+        suffix = "64"
         #os.environ["VSEXTCOMP_USECL"] = "MS_OPTERON"
     else:
         raise ValueError('Unrecognized platform: %s' % arch)
 
-    # rebuild makefile when we do the role over from 32 to 64 build
-    if arch == "amd64" and os.path.isfile(m32) and not os.path.isfile(makefile):
-        os.unlink(m32)
-
-    # If the ssl makefiles do not exist, we invoke Perl to generate them.
-    # Due to a bug in this script, the makefile sometimes ended up empty
-    # Force a regeneration if it is.
-    if not os.path.isfile(makefile) or os.path.getsize(makefile)==0:
     print("Creating the makefiles...")
     sys.stdout.flush()
+    # run configure, copy includes, create asms
     run_configure(configure, do_script)
-
-        if arch == "amd64":
-            create_makefile64(makefile, m32)
-        fix_makefile(makefile)
-        copy(r"crypto\buildinf.h", r"crypto\buildinf_%s.h" % arch)
-        copy(r"crypto\opensslconf.h", r"crypto\opensslconf_%s.h" % arch)
-    else:
-        print(makefile, 'already exists!')
+    makefile = makefile_template.format(suffix)
+    try:
+        os.unlink(makefile)
+    except FileNotFoundError:
+        pass
+    os.rename(generated_makefile, makefile)
+    copy_includes(makefile, suffix)
 
     print('creating asms...')
-    create_asms(makefile)
+    create_asms(makefile, 'tmp'+suffix)
+
 
 def main():
     if len(sys.argv) == 1:
@@ -229,6 +171,9 @@ def main():
         print("Found a working perl at '%s'" % (perl,))
     else:
         sys.exit(1)
+    if not find_all_on_path('nmake.exe'):
+        print('Could not find nmake.exe, try running env.bat')
+        sys.exit(1)
     sys.stdout.flush()
 
     # Put our working Perl at the front of our path