New version, with contributions from Sjoerd Mullender and Mark Hammond.

Sjoerd writes:

This version of freeze creates one file per Python module, instead of
one humongous file for all Python modules.
bkfile: new module to used to write files with backups.  No new file
is produced if the new contents is identical to the old.
New option "-x excluded-module" for modulefinder test program.
New option "-i filename" for freeze main program to include a list of
options in place of the -i option.

Tools/freeze/README

@@ -77,10 +77,12 @@ such as /usr/joe/python/Tools/freeze/freeze.py).
 What do I do next?
 ------------------
 
-Freeze creates three files: frozen.c, config.c and Makefile.  To
-produce the frozen version of your program, you can simply type
-"make".  This should produce a binary file.  If the filename argument
-to Freeze was "hello.py", the binary will be called "hello".
+Freeze creates a number of files: frozen.c, config.c and Makefile,
+plus one file for each Python module that gets included named
+M_<module>.c.  To produce the frozen version of your program, you can
+simply type "make".  This should produce a binary file.  If the
+filename argument to Freeze was "hello.py", the binary will be called
+"hello".
 
 Note: you can use the -o option to freeze to specify an alternative
 directory where these files are created. This makes it easier to

Tools/freeze/bkfile.py

+_orig_open = open
+
+class _BkFile:
+	def __init__(self, file, mode, bufsize):
+		import os
+		self.__filename = file
+		self.__backup = file + '~'
+		try:
+			os.unlink(self.__backup)
+		except os.error:
+			pass
+		try:
+			os.rename(file, self.__backup)
+		except os.error:
+			self.__backup = None
+		self.__file = _orig_open(file, mode, bufsize)
+		self.closed = self.__file.closed
+		self.fileno = self.__file.fileno
+		self.flush = self.__file.flush
+		self.isatty = self.__file.isatty
+		self.mode = self.__file.mode
+		self.name = self.__file.name
+		self.read = self.__file.read
+		self.readinto = self.__file.readinto
+		self.readline = self.__file.readline
+		self.readlines = self.__file.readlines
+		self.seek = self.__file.seek
+		self.softspace = self.__file.softspace
+		self.tell = self.__file.tell
+		self.truncate = self.__file.truncate
+		self.write = self.__file.write
+		self.writelines = self.__file.writelines
+
+	def close(self):
+		self.__file.close()
+		if self.__backup is None:
+			return
+		import cmp
+		# don't use cmp.cmp because of NFS bugs :-( and
+		# anyway, the stat mtime values differ so do_cmp will
+		# most likely be called anyway
+		if cmp.do_cmp(self.__backup, self.__filename):
+			import os
+			os.unlink(self.__filename)
+			os.rename(self.__backup, self.__filename)
+
+def open(file, mode = 'r', bufsize = -1):
+	if 'w' not in mode:
+		return _orig_open(file, mode, bufsize)
+	return _BkFile(file, mode, bufsize)

Tools/freeze/checkextensions_win32.py

@@ -9,11 +9,13 @@ options anyway (eg, to enable or disable specific functionality)
 
 So my basic stragtegy is:
 
-* Have a Windows INI file which "describes" an extension module.
+* Have some Windows INI files which "describe" one or more extension modules.
+  (Freeze comes with a default one for all known modules - but you can specify
+  your own).
 * This description can include:
   - The MSVC .dsp file for the extension.  The .c source file names
     are extraced from there.
-  - Specific compiler options
+  - Specific compiler/linker options
   - Flag to indicate if Unicode compilation is expected.
 
 At the moment the name and location of this INI file is hardcoded,
@@ -52,31 +54,52 @@ class CExtension:
 	def GetLinkerLibs(self):
 		return self.linkerLibs
 
-def checkextensions(unknown, ignored):
+def checkextensions(unknown, extra_inis):
         # Create a table of frozen extensions
 
-	mapFileName = os.path.join( os.path.split(sys.argv[0])[0], "extensions_win32.ini")
+	defaultMapName = os.path.join( os.path.split(sys.argv[0])[0], "extensions_win32.ini")
+	if not os.path.isfile(defaultMapName):
+		sys.stderr.write("WARNING: %s can not be found - standard extensions may not be found" % mapFileName)
+	else:
+		# must go on end, so other inis can override.
+		extra_inis.append(defaultMapName)
+
 	ret = []
 	for mod in unknown:
-		defn = get_extension_defn( mod, mapFileName )
-		if defn is not None:
-			ret.append( defn )
+		for ini in extra_inis:
+#			print "Looking for", mod, "in", win32api.GetFullPathName(ini),"...",
+			defn = get_extension_defn( mod, ini )
+			if defn is not None:
+#				print "Yay - found it!"
+				ret.append( defn )
+				break
+#			print "Nope!"
+		else: # For not broken!
+			sys.stderr.write("No definition of module %s in any specified map file.\n" % (mod))
+		
 	return ret
 
 def get_extension_defn(moduleName, mapFileName):
 	if win32api is None: return None
 	dsp = win32api.GetProfileVal(moduleName, "dsp", "", mapFileName)
 	if dsp=="":
-		sys.stderr.write("No definition of module %s in map file '%s'\n" % (moduleName, mapFileName))
 		return None
 
 	# We allow environment variables in the file name
 	dsp = win32api.ExpandEnvironmentStrings(dsp)
+	# If the path to the .DSP file is not absolute, assume it is relative
+	# to the description file.
+	if not os.path.isabs(dsp):
+		dsp = os.path.join( os.path.split(mapFileName)[0], dsp)
+	# Parse it to extract the source files.
 	sourceFiles = parse_dsp(dsp)
 	if sourceFiles is None:
 		return None
 
 	module = CExtension(moduleName, sourceFiles)
+	# Put the path to the DSP into the environment so entries can reference it.
+	os.environ['dsp_path'] = os.path.split(dsp)[0]
+	os.environ['ini_path'] = os.path.split(mapFileName)[0]
 
 	cl_options = win32api.GetProfileVal(moduleName, "cl", "", mapFileName)
 	if cl_options:
@@ -90,7 +113,7 @@ def get_extension_defn(moduleName, mapFileName):
 
 	libs = string.split(win32api.GetProfileVal(moduleName, "libs", "", mapFileName))
 	for lib in libs:
-		module.AddLinkerLib(lib)
+		module.AddLinkerLib(win32api.ExpandEnvironmentStrings(lib))
 
 	for exc in exclude:
 		if exc in module.sourceFiles:

Tools/freeze/extensions_win32.ini

@@ -112,6 +112,6 @@ Unicode = 1
 ; Pythonwin
 [win32ui]
 dsp=%PYTHONEX%\Pythonwin\win32ui.dsp
-cl=/I %PYTHONEX%\win32\src
+cl=/D _AFXDLL /D FREEZE_WIN32UI /GX /I %PYTHONEX%\win32\src
 libs=mfc42.lib
 

Tools/freeze/freeze.py

@@ -5,7 +5,6 @@
 usage: freeze [options...] script [module]...
 
 Options:
-
 -p prefix:    This is the prefix used when you ran ``make install''
               in the Python build directory.
               (If you never ran this, freeze won't work.)
@@ -22,6 +21,8 @@ Options:
 -e extension: A directory containing additional .o files that
               may be used to resolve modules.  This directory
               should also have a Setup file describing the .o files.
+              On Windows, the name of a .INI file describing one
+              or more extensions is passed.
               More than one -e option may be given.
 
 -o dir:       Directory where the output files are created; default '.'.
@@ -41,15 +42,20 @@ Options:
 
 -h:           Print this help message.
 
--w:           Toggle Windows (NT or 95) behavior.
-              (For debugging only -- on a win32 platform, win32 behaviour
-              is automatic.)
-
 -x module     Exclude the specified module.
 
+-i filename:  Include a file with additional command line options.  Used
+              to prevent command lines growing beyond the capabilities of
+              the shell/OS.  All arguments specified in filename
+              are read and the -i option replaced with the parsed
+              params (note - quoting args in this file is NOT supported)
+
 -s subsystem: Specify the subsystem (For Windows only.); 
               'console' (default), 'windows', 'service' or 'com_dll'
               
+-w:           Toggle Windows (NT or 95) behavior.
+              (For debugging only -- on a win32 platform, win32 behaviour
+              is automatic.)
 
 Arguments:
 
@@ -87,6 +93,7 @@ import makeconfig
 import makefreeze
 import makemakefile
 import parsesetup
+import bkfile
 
 
 # Main program
@@ -105,8 +112,8 @@ def main():
     win = sys.platform[:3] == 'win'
 
     # default the exclude list for each platform
-##     if win: exclude = exclude + [
-##         'dos', 'dospath', 'mac', 'macpath', 'MACFS', 'posix', 'os2']
+    if win: exclude = exclude + [
+        'dos', 'dospath', 'mac', 'macpath', 'macfs', 'MACFS', 'posix', 'os2']
 
     # modules that are imported by the Python runtime
     implicits = ["site", "exceptions"]
@@ -118,7 +125,20 @@ def main():
     makefile = 'Makefile'
     subsystem = 'console'
 
-    # parse command line
+    # parse command line by first replacing any "-i" options with the file contents.
+    pos = 1
+    while pos < len(sys.argv)-1: # last option can not be "-i", so this ensures "pos+1" is in range!
+        if sys.argv[pos] == '-i':
+            try:
+                options = string.split(open(sys.argv[pos+1]).read())
+            except IOError, why:
+                usage("File name '%s' specified with the -i option can not be read - %s" % (sys.argv[pos+1], why) )
+            # Replace the '-i' and the filename with the read params.
+            sys.argv[pos:pos+2] = options
+            pos = pos + len(options) - 1 # Skip the name and the included args.
+        pos = pos + 1
+
+    # Now parse the command line with the extras inserted.
     try:
         opts, args = getopt.getopt(sys.argv[1:], 'a:de:hmo:p:P:qs:wx:l:')
     except getopt.error, msg:
@@ -197,13 +217,15 @@ def main():
     includes = ['-I' + incldir, '-I' + config_h_dir]
 
     # sanity check of directories and files
-    for dir in [prefix, exec_prefix, binlib, incldir] + extensions:
+    check_dirs = [prefix, exec_prefix, binlib, incldir]
+    if not win: check_dirs = check_dirs + extensions # These are not directories on Windows.
+    for dir in check_dirs:
         if not os.path.exists(dir):
             usage('needed directory %s not found' % dir)
         if not os.path.isdir(dir):
             usage('%s: not a directory' % dir)
     if win:
-        files = supp_sources
+        files = supp_sources + extensions # extensions are files on Windows.
     else:
         files = [config_c_in, makefile_in] + supp_sources
     for file in supp_sources:
@@ -260,7 +282,9 @@ def main():
             print "Created output directory", odir
         except os.error, msg:
             usage('%s: mkdir failed (%s)' % (odir, str(msg)))
+    base = ''
     if odir:
+        base = os.path.join(odir, '')
         frozen_c = os.path.join(odir, frozen_c)
         config_c = os.path.join(odir, config_c)
         target = os.path.join(odir, target)
@@ -323,21 +347,7 @@ def main():
     dict = mf.modules
 
     # generate output for frozen modules
-    backup = frozen_c + '~'
-    try:
-        os.rename(frozen_c, backup)
-    except os.error:
-        backup = None
-    outfp = open(frozen_c, 'w')
-    try:
-        makefreeze.makefreeze(outfp, dict, debug, custom_entry_point)
-    finally:
-        outfp.close()
-    if backup:
-        if cmp.cmp(backup, frozen_c):
-            sys.stderr.write('%s not changed, not written\n' % frozen_c)
-            os.unlink(frozen_c)
-            os.rename(backup, frozen_c)
+    files = makefreeze.makefreeze(base, dict, debug, custom_entry_point)
 
     # look for unfrozen modules (builtin and of unknown origin)
     builtins = []
@@ -387,7 +397,7 @@ def main():
                                                     frozen_extensions)
         # Create a module definition for the bootstrap C code.
         xtras = [frozenmain_c, os.path.basename(frozen_c),
-                 frozendllmain_c, extensions_c]
+                 frozendllmain_c, extensions_c] + files
         maindefn = checkextensions_win32.CExtension( '__main__', xtras )
         frozen_extensions.append( maindefn )
         outfp = open(makefile, 'w')
@@ -403,22 +413,12 @@ def main():
     # generate config.c and Makefile
     builtins.sort()
     infp = open(config_c_in)
-    backup = config_c + '~'
-    try:
-        os.rename(config_c, backup)
-    except os.error:
-        backup = None
-    outfp = open(config_c, 'w')
+    outfp = bkfile.open(config_c, 'w')
     try:
         makeconfig.makeconfig(infp, outfp, builtins)
     finally:
         outfp.close()
     infp.close()
-    if backup:
-        if cmp.cmp(backup, config_c):
-            sys.stderr.write('%s not changed, not written\n' % config_c)
-            os.unlink(config_c)
-            os.rename(backup, config_c)
 
     cflags = defines + includes + ['$(OPT)']
     libs = [os.path.join(binlib, 'libpython$(VERSION).a')]
@@ -431,31 +431,14 @@ def main():
 
     somevars['CFLAGS'] = string.join(cflags) # override
     files = ['$(OPT)', '$(LDFLAGS)', base_config_c, base_frozen_c] + \
-            supp_sources +  addfiles + libs + \
+            files + supp_sources +  addfiles + libs + \
             ['$(MODLIBS)', '$(LIBS)', '$(SYSLIBS)']
 
-    backup = makefile + '~'
-    if os.path.exists(makefile):
-        try:
-            os.unlink(backup)
-        except os.error:
-            pass
-    try:
-        os.rename(makefile, backup)
-    except os.error:
-        backup = None
-    outfp = open(makefile, 'w')
+    outfp = bkfile.open(makefile, 'w')
     try:
         makemakefile.makemakefile(outfp, somevars, files, base_target)
     finally:
         outfp.close()
-    if backup:
-        if not cmp.cmp(backup, makefile):
-            print 'previous Makefile saved as', backup
-        else:
-            sys.stderr.write('%s not changed, not written\n' % makefile)
-            os.unlink(makefile)
-            os.rename(backup, makefile)
 
     # Done!
 

Tools/freeze/makefreeze.py

 import marshal
 import string
+import bkfile
 
 
 # Write a file containing frozen code for the modules in the dictionary.
@@ -30,15 +31,19 @@ main(argc, argv)
 
 """
 
-def makefreeze(outfp, dict, debug=0, entry_point = None):
+def makefreeze(base, dict, debug=0, entry_point = None):
     if entry_point is None: entry_point = default_entry_point
     done = []
+    files = []
     mods = dict.keys()
     mods.sort()
     for mod in mods:
         m = dict[mod]
         mangled = string.join(string.split(mod, "."), "__")
         if m.__code__:
+            file = 'M_' + mangled + '.c'
+            outfp = bkfile.open(base + file, 'w')
+            files.append(file)
             if debug:
                 print "freezing", mod, "..."
             str = marshal.dumps(m.__code__)
@@ -48,13 +53,19 @@ def makefreeze(outfp, dict, debug=0, entry_point = None):
                 size = -size
             done.append((mod, mangled, size))
             writecode(outfp, mangled, str)
+            outfp.close()
     if debug:
         print "generating table of frozen modules"
+    outfp = bkfile.open(base + 'frozen.c', 'w')
+    for mod, mangled, size in done:
+        outfp.write('extern unsigned char M_%s[];\n' % mangled)
     outfp.write(header)
     for mod, mangled, size in done:
         outfp.write('\t{"%s", M_%s, %d},\n' % (mod, mangled, size))
     outfp.write(trailer)
     outfp.write(entry_point)
+    outfp.close()
+    return files
 
 
 
@@ -62,9 +73,13 @@ def makefreeze(outfp, dict, debug=0, entry_point = None):
 # The array is called M_<mod>.
 
 def writecode(outfp, mod, str):
-    outfp.write('static unsigned char M_%s[] = {' % mod)
+    outfp.write('unsigned char M_%s[] = {' % mod)
     for i in range(0, len(str), 16):
         outfp.write('\n\t')
         for c in str[i:i+16]:
             outfp.write('%d,' % ord(c))
     outfp.write('\n};\n')
+
+## def writecode(outfp, mod, str):
+##     outfp.write('unsigned char M_%s[%d] = "%s";\n' % (mod, len(str),
+##     string.join(map(lambda s: `s`[1:-1], string.split(str, '"')), '\\"')))

Tools/freeze/modulefinder.py

@@ -359,14 +359,16 @@ class ModuleFinder:
         keys = self.badmodules.keys()
         keys.sort()
         for key in keys:
-            print "?", key
+            # ... but not if they were explicitely excluded.
+            if key not in self.excludes:
+                print "?", key
 
 
 def test():
     # Parse command line
     import getopt
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "dmp:q")
+        opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:")
     except getopt.error, msg:
         print msg
         return
@@ -375,6 +377,7 @@ def test():
     debug = 1
     domods = 0
     addpath = []
+    exclude = []
     for o, a in opts:
         if o == '-d':
             debug = debug + 1
@@ -384,6 +387,8 @@ def test():
             addpath = addpath + string.split(a, os.pathsep)
         if o == '-q':
             debug = 0
+        if o == '-x':
+            exclude.append(a)
 
     # Provide default arguments
     if not args:
@@ -401,7 +406,7 @@ def test():
             print "   ", `item`
 
     # Create the module finder and turn its crank
-    mf = ModuleFinder(path, debug)
+    mf = ModuleFinder(path, debug, exclude)
     for arg in args[1:]:
         if arg == '-m':
             domods = 1

Tools/freeze/winmakemakefile.py

@@ -50,13 +50,29 @@ def makemakefile(outfp, vars, files, target):
         sys.stdout = save
 
 def realwork(vars, moddefns, target):
-    print "# Makefile for Windows (NT or 95) generated by freeze.py script"
+    print "# Makefile for Microsoft Visual C++ generated by freeze.py script"
     print
     print 'target = %s' % target
     print 'pythonhome = "%s"' % vars['prefix']
-    # XXX The following line is fishy and may need manual fixing
-    print 'pythonlib = "%s"' % (vars['exec_prefix'] +
-                                "/pcbuild/release/python15.lib")
+    print
+    print 'DEBUG=0 # Set to 1 to use the _d versions of Python.'
+    print '!IF $(DEBUG)'
+    print 'debug_suffix=_d'
+    print 'c_debug=/Zi /Od /DDEBUG /D_DEBUG'
+    print 'l_debug=/DEBUG'
+    print 'temp_dir=Build\\Debug'
+    print '!ELSE'
+    print 'debug_suffix='
+    print 'c_debug=/Ox'
+    print 'l_debug='
+    print 'temp_dir=Build\\Release'
+    print '!ENDIF'
+    print
+
+    print '# The following line assumes you have built Python using the standard instructions'
+    print '# Otherwise fix the following line to point to the library.'
+    print 'pythonlib = "$(pythonhome)/pcbuild/python15$(debug_suffix).lib"'
+    print
 
     # We only ever write one "entry point" symbol - either
     # "main" or "WinMain".  Therefore, there is no need to
@@ -69,21 +85,27 @@ def realwork(vars, moddefns, target):
         target_link_flags = "-dll"
         target_ext = ".dll"
 
-    print "cdl = /MD" # XXX - Should this come from vars?  User may have specific requirements...
+
+    print "# As the target uses Python15.dll, we must use this compiler option!"
+    print "cdl = /MD"
+    print
+    print "all: $(target)$(debug_suffix)%s" % (target_ext)
     print
-    print "all: $(target)%s" % (target_ext)
+
+    print '$(temp_dir):'
+    print '  if not exist $(temp_dir)\. mkdir $(temp_dir)'
     print
 
     objects = []
-    libs = ["shell32.lib", "comdlg32.lib", "wsock32.lib", "user32.lib"]
+    libs = ["shell32.lib", "comdlg32.lib", "wsock32.lib", "user32.lib", "oleaut32.lib"]
     for moddefn in moddefns:
         print "# Module", moddefn.name
         for file in moddefn.sourceFiles:
             base = os.path.basename(file)
             base, ext = os.path.splitext(base)
             objects.append(base + ".obj")
-            print '%s.obj: "%s"' % (base, file)
-            print "\t@$(CC) -c -nologo $(cdl) /D BUILD_FREEZE",
+            print '$(temp_dir)\%s.obj: "%s"' % (base, file)
+            print "\t@$(CC) -c -nologo /Fo$* $(cdl) $(c_debug) /D BUILD_FREEZE",
             print "-I$(pythonhome)/Include  -I$(pythonhome)/PC \\"
             print "\t\t$(cflags) $(cdebug) $(cinclude) \\"
             extra = moddefn.GetCompilerOptions()
@@ -102,20 +124,20 @@ def realwork(vars, moddefns, target):
     print ; print
 
     print "OBJS=",
-    for obj in objects: print '"%s"' % (obj),
+    for obj in objects: print '"$(temp_dir)\%s"' % (obj),
     print ; print
 
     print "LIBS=",
     for lib in libs: print '"%s"' % (lib),
     print ; print
 
-    print "$(target)%s: $(OBJS)" % (target_ext)
-    print "\tlink -out:$(target)%s %s" % (target_ext, target_link_flags),
+    print "$(target)$(debug_suffix)%s: $(temp_dir) $(OBJS)" % (target_ext)
+    print "\tlink -out:$(target)$(debug_suffix)%s %s" % (target_ext, target_link_flags),
     print "\t$(OBJS) \\"
     print "\t$(LIBS) \\"
     print "\t$(ADDN_LINK_FILES) \\"
-    print "\t\t$(pythonlib) $(lcustom)\\"
-    print "\t\t$(resources)"
+    print "\t$(pythonlib) $(lcustom) $(l_debug)\\"
+    print "\t$(resources)"
     print
     print "clean:"
     print "\t-rm -f *.obj"