Jack's last version (now I'm supposed to get it working :-)

Tools/freeze/freeze.py

+#! /usr/local/bin/python
+
+# Given a Python script, create a binary that runs the script.
+# The binary is 100% independent of Python libraries and binaries.
+# It will not contain any Python source code -- only "compiled" Python
+# (as initialized static variables containing marshalled code objects).
+# It even does the right thing for dynamically loaded modules!
+# The module search path of the binary is set to the current directory.
+#
+# Some problems remain:
+# - You need to have the Python source tree lying around as well as
+#   the various libraries used to generate the Python binary.
+# - For scripts that use many modules it generates absurdly large
+#   files (frozen.c and config.o as well as the final binary),
+#   and is consequently rather slow.
+#
+# Caveats:
+# - The search for modules sometimes finds modules that are never
+#   actually imported since the code importing them is never executed.
+# - If an imported module isn't found, you get a warning but the
+#   process of freezing continues.  The binary will fail if it
+#   actually tries to import one of these modules.
+# - This often happens with the module 'mac', which module 'os' tries
+#   to import (to determine whether it is running on a Macintosh).
+#   You can ignore the warning about this.
+# - If the program dynamically reads or generates Python code and
+#   executes it, this code may reference built-in or library modules
+#   that aren't present in the frozen binary, and this will fail.
+# - Your program may be using external data files, e.g. compiled
+#   forms definitions (*.fd).  These aren't incorporated. By default,
+#   the sys.path in the resulting binary is only '.' (but you can override
+#   that with the -P option).
+#
+# Usage hints:
+# - If you have a bunch of scripts that you want to freeze, instead
+#   of freezing each of them separately, you might consider writing
+#   a tiny main script that looks at sys.argv[0] and then imports
+#   the corresponding module.  You can then make links to the
+#   frozen binary named after the various scripts you support.
+#   Pass the additional scripts as arguments after the main script.
+#   A minimal script to do this is the following.
+#       import sys, posixpath
+#       exec('import ' + posixpath.basename(sys.argv[0]) + '\n')
+#
+# Mods by Jack, August 94:
+# - Removed all static configuration stuff. Now, Setup and Makefile files
+#   are parsed to obtain the linking info for the libraries. You have to
+#   supply the -B option, though.
+# - Added -P (set sys.path) and -I/-D/-L/-l options (passed on to cc and
+#   ld).
+
+import os
+import sys
+import regex
+import getopt
+import regsub
+import string
+import marshal
+
+# Exception used when scanfile fails
+NoSuchFile = 'NoSuchFile'
+
+# Global options
+builddir = ''				# -B dir
+quiet = 0				# -q
+verbose = 0				# -v
+noexec = 0				# -n
+nowrite = 0				# -N
+ofile = 'a.out'				# -o file
+path = '\'"."\''			# -P path
+
+cc_options = []				# Collects cc options
+ld_options = []				# Collects ld options
+module_libraries = {}			# ld options for each module
+global_libraries = []			# Libraries we always need
+include_path = ''			# Include path, from Makefile 
+lib_path = ''				# and lib path, ditto
+compiler = 'cc'				# and compiler
+
+# Main program -- argument parsing etc.
+def main():
+	global quiet, verbose, noexec, nowrite, ofile, builddir, path
+	try:
+		opts, args = getopt.getopt(sys.argv[1:], 'B:nNo:P:qvI:D:L:l:')
+	except getopt.error, msg:
+		usage(str(msg))
+		sys.exit(2)
+	for o, a in opts:
+	        if o == '-B': builddir = a
+		if o == '-n': noexec = 1
+		if o == '-N': nowrite = 1
+		if o == '-o': ofile = a
+		if o == '-P':
+		    if '"' in a:
+			usage('sorry, cannot have " in -P option')
+			sys.exit(2)
+		    path = `'"' + a + '"'`
+		if o == '-q': verbose = 0; quiet = 1
+		if o == '-v': verbose = verbose + 1; quiet = 0
+		if o in ('-I', '-D'): cc_options.append(o+a)
+		if o in ('-L', '-l'): ld_options.append(o+a)
+	if not builddir:
+	    usage('sorry, you have to pass a -B option')
+	    sys.exit(2)
+	if len(args) < 1:
+		usage('please pass at least one file argument')
+		sys.exit(2)
+	process(args[0], args[1:])
+
+# Print usage message to stderr
+def usage(*msgs):
+	sys.stdout = sys.stderr
+	for msg in msgs: print msg
+	print 'Usage: freeze [options] scriptfile [modulefile ...]'
+	print '-B dir  : name of python build dir (no default)'
+	print '-n      : generate the files but don\'t compile and link'
+	print '-N      : don\'t write frozen.c (do compile unless -n given)'
+	print '-o file : binary output file (default a.out)'
+	print '-P path : set sys.path for program (default ".")'
+	print '-q      : quiet (no messages at all except errors)'
+	print '-v      : verbose (lots of extra messages)'
+	print '-D and -I options are passed to cc, -L and -l to ld'
+
+# Process the script file
+def process(filename, addmodules):
+	global noexec
+	#
+	if not quiet: print 'Computing needed modules ...'
+	todo = {}
+	todo['__main__'] = filename
+	for name in addmodules:
+		mod = os.path.basename(name)
+		if mod[-3:] == '.py': mod = mod[:-3]
+		todo[mod] = name
+	try:
+		dict = closure(todo)
+	except NoSuchFile, filename:
+		sys.stderr.write('Can\'t open file %s\n' % filename)
+		sys.exit(1)
+	#
+	mods = dict.keys()
+	mods.sort()
+	#
+	if verbose:
+		print '%-15s %s' % ('Module', 'Filename')
+		for mod in mods:
+			print '%-15s %s' % (`mod`, dict[mod])
+	#
+	if not quiet: print 'Looking for dynamically linked modules ...'
+	dlmodules = []
+	objs = []
+	libs = []
+	for mod in mods:
+		if dict[mod][-2:] == '.o':
+			if verbose: print 'Found', mod, dict[mod]
+			dlmodules.append(mod)
+			objs.append(dict[mod])
+			libsname = dict[mod][:-2] + '.libs'
+			try:
+				f = open(libsname, 'r')
+			except IOError:
+				f = None
+			if f:
+				libtext = f.read()
+				f.close()
+				for lib in string.split(libtext):
+					if lib in libs: libs.remove(lib)
+					libs.append(lib)
+	#
+	if not nowrite:
+		if not quiet: print 'Writing frozen.c ...'
+		writefrozen('frozen.c', dict)
+	else:
+		if not quiet: print 'NOT writing frozen.c ...'
+	#
+	if not quiet:
+	    print 'Deducing compile/link options from', builddir
+	#
+	# Parse the config info
+	#
+	parse(builddir)
+	CONFIG_IN = lib_path + '/config.c.in'
+	FMAIN = lib_path + '/frozenmain.c'
+	CC = compiler
+	#
+##	if not dlmodules:
+	if 0:
+		config = CONFIG
+		if not quiet: print 'Using existing', config, '...'
+	else:
+		config = 'tmpconfig.c'
+		if nowrite:
+			if not quiet: print 'NOT writing config.c ...'
+		else:
+			if not quiet:
+				print 'Writing config.c with dl modules ...'
+			f = open(CONFIG_IN, 'r')
+			g = open(config, 'w')
+			m1 = regex.compile('-- ADDMODULE MARKER 1 --')
+			m2 = regex.compile('-- ADDMODULE MARKER 2 --')
+			builtinmodules = []
+			stdmodules = ('sys', '__main__', '__builtin__',
+				      'marshal')
+			for mod in dict.keys():
+				if dict[mod] == '<builtin>' and \
+					  mod not in stdmodules:
+					builtinmodules.append(mod)
+			todomodules = builtinmodules + dlmodules
+			while 1:
+				line = f.readline()
+				if not line: break
+				g.write(line)
+				if m1.search(line) >= 0:
+					if verbose: print 'Marker 1 ...'
+					for mod in todomodules:
+						g.write('extern void init' + \
+						  mod + '();\n')
+				if m2.search(line) >= 0:
+					if verbose: print 'Marker 2 ...'
+					for mod in todomodules:
+						g.write('{"' + mod + \
+						  '", init' + mod + '},\n')
+			g.close()
+	#
+	if not quiet:
+		if noexec: print 'Generating compilation commands ...'
+		else: print 'Starting compilation ...'
+	defs = ['-DNO_MAIN', '-DUSE_FROZEN']
+	defs.append('-DPYTHONPATH='+path)
+	#
+	incs = ['-I.', '-I' + include_path]
+#	if dict.has_key('stdwin'):
+#		incs.append('-I' + j(STDWIN, 'H'))
+	#
+	srcs = [config, FMAIN]
+	#
+	modlibs = module_libraries
+	
+	for mod in dict.keys():
+	    if modlibs.has_key(mod):
+		libs = libs + modlibs[mod]
+
+	libs = libs + global_libraries
+	#
+	# remove dups:
+	# XXXX Not well tested...
+	nskip = 0
+	newlibs = []
+	while libs:
+	    l = libs[0]
+	    del libs[0]
+	    if l[:2] == '-L' and l in newlibs:
+		nskip = nskip + 1
+		continue
+	    if (l[:2] == '-l' or l[-2:] == '.a') and l in libs:
+		nskip = nskip + 1
+		continue
+	    newlibs.append(l)
+	libs = newlibs
+	if nskip and not quiet:
+	    print 'Removed %d duplicate libraries'%nskip
+	#
+	sts = 0
+	#
+	cmd = CC + ' -c'
+	if cc_options:
+	    cmd = cmd + ' ' + string.join(cc_options)
+	cmd = cmd + ' ' + string.join(defs)
+	cmd = cmd + ' ' + string.join(incs)
+	cmd = cmd + ' ' + string.join(srcs)
+	print cmd
+	#
+	if not noexec:
+		sts = os.system(cmd)
+		if sts:
+			print 'Exit status', sts, '-- turning on -n'
+			noexec = 1
+	#
+	for s in srcs:
+		s = os.path.basename(s)
+		if s[-2:] == '.c': s = s[:-2]
+		o = s + '.o'
+		objs.insert(0, o)
+	#
+	cmd = CC
+	cmd = cmd + ' ' + string.join(objs)
+	cmd = cmd + ' ' + string.join(libs)
+	if ld_options:
+	    cmd = cmd + ' ' + string.join(ld_options)
+	cmd = cmd + ' -o ' + ofile
+	print cmd
+	#
+	if not noexec:
+		sts = os.system(cmd)
+		if sts:
+			print 'Exit status', sts
+		else:
+			print 'Done.'
+	#
+	if not quiet and not noexec and sts == 0:
+		print 'Note: consider this:'; print '\tstrip', ofile
+	#
+	sys.exit(sts)
+
+
+# Generate code for a given module
+def makecode(filename):
+	if filename[-2:] == '.o':
+		return None
+	try:
+		f = open(filename, 'r')
+	except IOError:
+		return None
+	if verbose: print 'Making code from', filename, '...'
+	text = f.read()
+	code = compile(text, filename, 'exec')
+	f.close()
+	return marshal.dumps(code)
+
+
+# Write the C source file containing the frozen Python code
+def writefrozen(filename, dict):
+	f = open(filename, 'w')
+	codelist = []
+	for mod in dict.keys():
+		codestring = makecode(dict[mod])
+		if codestring is not None:
+			codelist.append((mod, codestring))
+	write = sys.stdout.write
+	save_stdout = sys.stdout
+	try:
+		sys.stdout = f
+		for mod, codestring in codelist:
+			if verbose:
+				write('Writing initializer for %s\n'%mod)
+			print 'static unsigned char M_' + mod + '[' + \
+				  str(len(codestring)) + '+1] = {'
+			for i in range(0, len(codestring), 16):
+				for c in codestring[i:i+16]:
+					print str(ord(c)) + ',',
+				print
+			print '};'
+		print 'struct frozen {'
+		print '  char *name;'
+		print '  unsigned char *code;'
+		print '  int size;'
+		print '} frozen_modules[] = {'
+		for mod, codestring in codelist:
+			print '  {"' + mod + '",',
+			print 'M_' + mod + ',',
+			print str(len(codestring)) + '},'
+		print '  {0, 0, 0} /* sentinel */'
+		print '};'
+	finally:
+		sys.stdout = save_stdout
+	f.close()
+
+
+# Determine the names and filenames of the modules imported by the
+# script, recursively.  This is done by scanning for lines containing
+# import statements.  (The scanning has only superficial knowledge of
+# Python syntax and no knowledge of semantics, so in theory the result
+# may be incorrect -- however this is quite unlikely if you don't
+# intentionally obscure your Python code.)
+
+# Compute the closure of scanfile() -- special first file because of script
+def closure(todo):
+	done = {}
+	while todo:
+		newtodo = {}
+		for modname in todo.keys():
+			if not done.has_key(modname):
+				filename = todo[modname]
+				if filename is None:
+					filename = findmodule(modname)
+				done[modname] = filename
+				if filename in ('<builtin>', '<unknown>'):
+					continue
+				modules = scanfile(filename)
+				for m in modules:
+					if not done.has_key(m):
+						newtodo[m] = None
+		todo = newtodo
+	return done
+
+# Scan a file looking for import statements
+importstr = '\(^\|:\)[ \t]*import[ \t]+\([a-zA-Z0-9_, \t]+\)'
+fromstr   = '\(^\|:\)[ \t]*from[ \t]+\([a-zA-Z0-9_]+\)[ \t]+import[ \t]+'
+isimport = regex.compile(importstr)
+isfrom = regex.compile(fromstr)
+def scanfile(filename):
+	allmodules = {}
+	try:
+		f = open(filename, 'r')
+	except IOError, msg:
+		raise NoSuchFile, filename
+	while 1:
+		line = f.readline()
+		if not line: break # EOF
+		while line[-2:] == '\\\n': # Continuation line
+			line = line[:-2] + ' '
+			line = line + f.readline()
+		if isimport.search(line) >= 0:
+			rawmodules = isimport.group(2)
+			modules = string.splitfields(rawmodules, ',')
+			for i in range(len(modules)):
+				modules[i] = string.strip(modules[i])
+		elif isfrom.search(line) >= 0:
+			modules = [isfrom.group(2)]
+		else:
+			continue
+		for mod in modules:
+			allmodules[mod] = None
+	f.close()
+	return allmodules.keys()
+
+# Find the file containing a module, given its name; None if not found
+builtins = sys.builtin_module_names + ['sys']
+def findmodule(modname):
+	if modname in builtins: return '<builtin>'
+	for dirname in sys.path:
+		dlfullname = os.path.join(dirname, modname + 'module.o')
+		try:
+			f = open(dlfullname, 'r')
+		except IOError:
+			f = None
+		if f:
+			f.close()
+			return dlfullname
+		fullname = os.path.join(dirname, modname + '.py')
+		try:
+			f = open(fullname, 'r')
+		except IOError:
+			continue
+		f.close()
+		return fullname
+	if not quiet:
+		sys.stderr.write('Warning: module %s not found\n' % modname)
+	return '<unknown>'
+#
+# Parse a setup file. Returns two dictionaries, one containing variables
+# defined with their values and one containing module definitions
+#
+def parse_setup(fp):
+    modules = {}
+    variables = {}
+    for line in fp.readlines():
+	if '#' in line:				# Strip comments
+	    line = string.splitfields(line, '#')[0]
+	line = string.strip(line[:-1])		# Strip whitespace
+	if not line:
+	    continue
+	words = string.split(line)
+	if '=' in words[0]:
+	    #
+	    # equal sign before first space. Definition
+	    #
+	    pos = string.index(line, '=')
+	    name = line[:pos]
+	    value = string.strip(line[pos+1:])
+	    variables[name] = value
+	else:
+	    modules[words[0]] = words[1:]
+    return modules, variables
+#
+# Parse a makefile. Returns a list of the variables defined.
+#
+def parse_makefile(fp):
+    variables = {}
+    for line in fp.readlines():
+	if '#' in line:				# Strip comments
+	    line = string.splitfields(line, '#')[0]
+	if not line:
+	    continue
+	if line[0] in string.whitespace:
+	    continue
+	line = string.strip(line[:-1])		# Strip whitespace
+	if not line:
+	    continue
+	if '=' in string.splitfields(line, ':')[0]: 
+	    #
+	    # equal sign before first colon. Definition
+	    #
+	    pos = string.index(line, '=')
+	    name = line[:pos]
+	    value = string.strip(line[pos+1:])
+	    variables[name] = value
+    return variables
+
+#
+# Recursively add loader options from Setup files in extension
+# directories.
+#
+def add_extension_directory(name, isinstalldir):
+    if verbose:
+	print 'Adding extension directory', name
+    fp = open(name + '/Setup', 'r')
+    modules, variables = parse_setup(fp)
+    #
+    # Locate all new modules and remember the ld flags needed for them
+    #
+    for m in modules.keys():
+	if module_libraries.has_key(m):
+	    continue
+	options = modules[m]
+	if isinstalldir:
+	    ld_options = []
+	else:
+	    ld_options = [name + '/lib.a']
+	for o in options:
+	    # ld options are all capital except DUIC and l
+	    if o[:-2] == '.a':
+		ld_options.append(o)
+	    elif o[0] == '-':
+		if o[1] == 'l':
+		    ld_options.append(o)
+		elif o[1] in string.uppercase and not o[1] in 'DUIC':
+		    ld_options.append(o)
+	module_libraries[m] = ld_options
+    #
+    # See if we have to bother with base setups
+    #
+    if variables.has_key('BASESETUP'):
+	if isinstalldir:
+	    raise 'installdir has base setup'
+	setupfiles = string.split(variables['BASESETUP'])
+	for s in setupfiles:
+	    if s[-6:] <> '/Setup':
+		raise 'Incorrect BASESETUP', s
+	    s = s[:-6]
+	    if s[0] <> '/':
+		s = name + '/' + s
+		s = os.path.normpath(s)
+	    add_extension_directory(s, 0)
+#
+# Main routine for this module: given a build directory, get all
+# information needed for the linker.
+#
+def parse(dir):
+    global include_path
+    global lib_path
+    global compiler
+    
+    fp = open(dir + '/Makefile', 'r')
+    #
+    # First find the global libraries and the base python
+    #
+    vars = parse_makefile(fp)
+    if vars.has_key('CC'):
+	compiler = vars['CC']
+    if not vars.has_key('installdir'):
+	raise 'No $installdir in Makefile'
+    include_path = vars['installdir'] + '/include/Py'
+    lib_path = vars['installdir'] + '/lib/python/lib'
+    global_libraries.append('-L' + lib_path)
+    global_libraries.append('-lPython')
+    global_libraries.append('-lParser')
+    global_libraries.append('-lObjects')
+    global_libraries.append('-lModules')
+    for name in ('LIBS', 'LIBM', 'LIBC'):
+	if not vars.has_key(name):
+	    raise 'Missing required def in Makefile', name
+	for lib in string.split(vars[name]):
+	    global_libraries.append(lib)
+    #
+    # Next, parse the modules from the base python
+    #
+    add_extension_directory(lib_path, 1)
+    #
+    # Finally, parse the modules from the extension python
+    #
+    if dir <> lib_path:
+	add_extension_directory(dir, 0)
+
+# Call the main program
+main()