Commit dc34f555 authored by Gary Poster's avatar Gary Poster

an interpreter variant that supports all interpreter flags (AFAIK). this...

an interpreter variant that supports all interpreter flags (AFAIK).  this commit contains core implementation (in easy_install module) and tests of all functionality except import_sitecustomize.  Integration into zc.recipe.egg and tests for import_sitecustomize (which will be more involved, and in tests.py) will follow.
parent 0ee68e53
...@@ -968,17 +968,91 @@ def scripts(reqs, working_set, executable, dest, ...@@ -968,17 +968,91 @@ def scripts(reqs, working_set, executable, dest,
return generated return generated
import_site_snippet = '''\ def interpreter(name, working_set, executable, dest, site_py_dest,
# We have to import pkg_resources before namespace extra_paths=(),
# package .pth files are processed or else the distribution's namespace initialization='',
# packages will mask all of the egg-based packages in the same namespace relative_paths=False,
# package. import_site=False,
try: import_sitecustomize=False
import pkg_resources ):
except ImportError: generated = []
# Write sitecustomize.py.
sitecustomize_path = os.path.join(site_py_dest, 'sitecustomize.py')
sitecustomize = open(sitecustomize_path, 'w')
if initialization:
sitecustomize.write(initialization + '\n')
if import_sitecustomize:
real_sitecustomize_path = _get_module_file(
executable, 'sitecustomize')
if real_sitecustomize_path:
sitecustomize.write('execfile(%s)\n' % (real_sitecustomize_path,))
sitecustomize.close()
generated.append(sitecustomize_path)
# Write site.py.
path = [dist.location for dist in working_set]
path.extend(extra_paths)
path = map(realpath, path)
site_path = os.path.join(site_py_dest, 'site.py')
path_string, rpsetup = _relative_path_and_setup(
site_path, path, relative_paths)
site = open(site_path, 'w')
site.write(rpsetup)
site.write(sys_path_template % (path_string,))
if import_site:
real_site_path = _get_module_file(executable, 'site')
if real_site_path:
site.write(import_site_template % (real_site_path,))
else:
site.write('import sitecustomize\n')
generated.append(site_path)
# Write interpreter script.
script_name = full_name = os.path.join(dest, name)
site_py_dest_string, rpsetup = _relative_path_and_setup(
full_name, [site_py_dest], relative_paths)
if is_win32:
script_name += '-script.py'
contents = interpreter_template % dict(
python = _safe_arg(executable),
site_dest = site_py_dest_string,
relative_paths_setup = rpsetup,
)
changed = not (os.path.exists(script_name) and
open(script_name).read() == contents)
if is_win32:
# Generate exe file and give the script a magic name.
exe = full_name + '.exe'
open(exe, 'wb').write(
pkg_resources.resource_string('setuptools', 'cli.exe')
)
generated.append(exe)
if changed:
open(script_name, 'w').write(contents)
try:
os.chmod(script_name,0755)
except (AttributeError, os.error):
pass pass
import site logger.info("Generated interpreter %r.", name)
''' generated.append(script_name)
return generated
def _get_module_file(executable, name):
cmd = [executable, "-c",
"import imp; fp, path, desc = imp.find_module(%r); fp.close; print path" % (name,)]
_proc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = _proc.communicate();
if _proc.returncode:
logger.info(
'Could not find file for module %s:\n%s', name, stderr)
return None
# else: ...
res = stdout.strip()
if res.endswith('.pyc') or res.endswith('.pyo'):
raise RuntimeError('Cannot find uncompiled version of %s' % (name,))
if not os.path.exists(res):
raise RuntimeError(
'File does not exist for module %s:\n%s' % (name, res))
return res
def _relative_path_and_setup(sname, path, relative_paths): def _relative_path_and_setup(sname, path, relative_paths):
if relative_paths: if relative_paths:
...@@ -996,7 +1070,6 @@ def _relative_path_and_setup(sname, path, relative_paths): ...@@ -996,7 +1070,6 @@ def _relative_path_and_setup(sname, path, relative_paths):
rpsetup = '' rpsetup = ''
return spath, rpsetup return spath, rpsetup
def _relative_depth(common, path): def _relative_depth(common, path):
n = 0 n = 0
while 1: while 1:
...@@ -1033,7 +1106,6 @@ def _relativitize(path, script, relative_paths): ...@@ -1033,7 +1106,6 @@ def _relativitize(path, script, relative_paths):
else: else:
return repr(path) return repr(path)
relative_paths_setup = """ relative_paths_setup = """
import os import os
...@@ -1090,6 +1162,25 @@ if is_jython and jython_os_name == 'linux': ...@@ -1090,6 +1162,25 @@ if is_jython and jython_os_name == 'linux':
else: else:
script_header = '#!%(python)s -S' script_header = '#!%(python)s -S'
_import_site_start = '''\
# We have to import pkg_resources before namespace
# package .pth files are processed or else the distribution's namespace
# packages will mask all of the egg-based packages in the same namespace
# package.
try:
import pkg_resources
except ImportError:
pass
'''
import_site_snippet = _import_site_start + 'import site\n'
import_site_template = _import_site_start + 'execfile(%r)\n'
sys_path_template = '''\
import sys
sys.path[0:0] = [
%s,
]
'''
script_template = script_header + '''\ script_template = script_header + '''\
...@@ -1106,6 +1197,20 @@ if __name__ == '__main__': ...@@ -1106,6 +1197,20 @@ if __name__ == '__main__':
%(module_name)s.%(attrs)s(%(arguments)s) %(module_name)s.%(attrs)s(%(arguments)s)
''' '''
interpreter_template = script_header + '''\
%(relative_paths_setup)s
import os
import sys
argv = [sys.executable] + sys.argv[1:]
environ = os.environ.copy()
path = %(site_dest)s
if environ.get('PYTHONPATH'):
path = os.pathsep.join([path, environ['PYTHONPATH']])
environ['PYTHONPATH'] = path
os.execve(sys.executable, argv, environ)
'''
def _pyscript(path, dest, executable, rsetup, import_site): def _pyscript(path, dest, executable, rsetup, import_site):
generated = [] generated = []
......
...@@ -999,6 +999,233 @@ We specified an interpreter and its paths are adjusted too: ...@@ -999,6 +999,233 @@ We specified an interpreter and its paths are adjusted too:
__import__("code").interact(banner="", local=globals()) __import__("code").interact(banner="", local=globals())
Interpreter generation
----------------------
The interpreter created above is a script that emulates the Python
interpreter. Sometimes it is useful to have a full-fledged Python interpreter
available. The ``interpreter`` function supports this.
It works by creating site.py and sitecustomize.py files that set up the
desired paths and initialization. These must be placed within an otherwise
empty directory. Typically this is in a recipe's parts directory.
Here's the simplest example.
>>> interpreter_dir = tmpdir('interpreter')
>>> mkdir(interpreter_dir, 'bin')
>>> mkdir(interpreter_dir, 'eggs')
>>> mkdir(interpreter_dir, 'parts')
>>> mkdir(interpreter_dir, 'parts', 'interpreter')
>>> ws = zc.buildout.easy_install.install(
... ['demo'], join(interpreter_dir, 'eggs'), links=[link_server],
... index=link_server+'index/')
>>> generated = zc.buildout.easy_install.interpreter(
... 'py', ws, sys.executable, join(interpreter_dir, 'bin'),
... join(interpreter_dir, 'parts', 'interpreter'))
On non-Windows systems, this creates three files.
>>> len(generated)
3
>>> import pprint
>>> pprint.pprint(sorted(generated))
['/interpreter/bin/py',
'/interpreter/parts/interpreter/site.py',
'/interpreter/parts/interpreter/sitecustomize.py']
These are the three generated files.
>>> cat(interpreter_dir, 'bin', 'py')
#!/usr/bin/python2.4 -S
<BLANKLINE>
import os
import sys
<BLANKLINE>
argv = [sys.executable] + sys.argv[1:]
environ = os.environ.copy()
path = '/interpreter/parts/interpreter'
if environ.get('PYTHONPATH'):
path = os.pathsep.join([path, environ['PYTHONPATH']])
environ['PYTHONPATH'] = path
os.execve(sys.executable, argv, environ)
>>> ls(interpreter_dir, 'parts', 'interpreter')
- site.py
- sitecustomize.py
>>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
import sys
sys.path[0:0] = [
'/interpreter/eggs/demo-0.3-py2.4.egg',
'/interpreter/eggs/demoneeded-1.1-py2.4.egg',
]
import sitecustomize
>>> cat(interpreter_dir, 'parts', 'interpreter', 'sitecustomize.py')
Here are some examples of the interpreter in use.
>>> print system(join(interpreter_dir, 'bin', 'py') + ' -c "print 16+26"')
42
<BLANKLINE>
>>> res = system(join(interpreter_dir, 'bin', 'py') +
... ' -c "import sys, pprint; pprint.pprint(sys.path)"')
>>> print res # doctest: +ELLIPSIS
['',
'/interpreter/eggs/demo-0.3-py2.4.egg',
'/interpreter/eggs/demoneeded-1.1-py2.4.egg',
'/interpreter/parts/interpreter',
...]
<BLANKLINE>
>>> clean_paths = eval(res.strip()) # This is used later for comparison.
If you provide initialization, it goes in sitecustomize.py.
>>> def reset_interpreter():
... # This is necessary because, in our tests, the timestamps of the
... # .pyc files are not outdated when we want them to be.
... rmdir(interpreter_dir, 'bin')
... mkdir(interpreter_dir, 'bin')
... rmdir(interpreter_dir, 'parts', 'interpreter')
... mkdir(interpreter_dir, 'parts', 'interpreter')
...
>>> reset_interpreter()
>>> initialization_string = """\
... import os
... os.environ['FOO'] = 'bar baz bing shazam'"""
>>> generated = zc.buildout.easy_install.interpreter(
... 'py', ws, sys.executable, join(interpreter_dir, 'bin'),
... join(interpreter_dir, 'parts', 'interpreter'),
... initialization=initialization_string)
>>> cat(interpreter_dir, 'parts', 'interpreter', 'sitecustomize.py')
import os
os.environ['FOO'] = 'bar baz bing shazam'
>>> print system(join(interpreter_dir, 'bin', 'py') +
... """ -c 'import os; print os.environ["FOO"]'""")
bar baz bing shazam
<BLANKLINE>
If you use relative paths, this affects the interpreter and site.py.
>>> reset_interpreter()
>>> generated = zc.buildout.easy_install.interpreter(
... 'py', ws, sys.executable, join(interpreter_dir, 'bin'),
... join(interpreter_dir, 'parts', 'interpreter'),
... relative_paths=interpreter_dir)
>>> cat(interpreter_dir, 'bin', 'py')
#!/usr/bin/python2.4 -S
<BLANKLINE>
import os
<BLANKLINE>
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
<BLANKLINE>
import os
import sys
<BLANKLINE>
argv = [sys.executable] + sys.argv[1:]
environ = os.environ.copy()
path = join(base, 'parts/interpreter')
if environ.get('PYTHONPATH'):
path = os.pathsep.join([path, environ['PYTHONPATH']])
environ['PYTHONPATH'] = path
os.execve(sys.executable, argv, environ)
>>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
<BLANKLINE>
import os
<BLANKLINE>
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
base = os.path.dirname(base)
import sys
sys.path[0:0] = [
join(base, 'eggs/demo-0.3-pyN.N.egg'),
join(base, 'eggs/demoneeded-1.1-pyN.N.egg'),
]
import sitecustomize
>>> print system(join(interpreter_dir, 'bin', 'py') +
... ' -c "import sys, pprint; pprint.pprint(sys.path)"')
... # doctest: +ELLIPSIS
['',
'/interpreter/eggs/demo-0.3-py2.4.egg',
'/interpreter/eggs/demoneeded-1.1-py2.4.egg',
'/interpreter/parts/interpreter',
...]
<BLANKLINE>
The ``extra_paths`` argument affects the path in site.py.
>>> reset_interpreter()
>>> generated = zc.buildout.easy_install.interpreter(
... 'py', ws, sys.executable, join(interpreter_dir, 'bin'),
... join(interpreter_dir, 'parts', 'interpreter'),
... extra_paths=[join(interpreter_dir, 'other')])
>>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
import sys
sys.path[0:0] = [
'/interpreter/eggs/demo-0.3-py2.4.egg',
'/interpreter/eggs/demoneeded-1.1-py2.4.egg',
'/interpreter/other',
]
import sitecustomize
>>> print system(join(interpreter_dir, 'bin', 'py') +
... ' -c "import sys, pprint; pprint.pprint(sys.path)"')
... # doctest: +ELLIPSIS
['',
'/interpreter/eggs/demo-0.3-py2.4.egg',
'/interpreter/eggs/demoneeded-1.1-py2.4.egg',
'/interpreter/other',
'/interpreter/parts/interpreter',
...]
<BLANKLINE>
The ``import_site`` argument also affects site.py. It causes the executable's
real site.py to be executed after our customizations have run.
>>> reset_interpreter()
>>> generated = zc.buildout.easy_install.interpreter(
... 'py', ws, sys.executable, join(interpreter_dir, 'bin'),
... join(interpreter_dir, 'parts', 'interpreter'),
... import_site=True)
>>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
import sys
sys.path[0:0] = [
'/interpreter/eggs/demo-0.3-py2.4.egg',
'/interpreter/eggs/demoneeded-1.1-py2.4.egg',
]
# We have to import pkg_resources before namespace
# package .pth files are processed or else the distribution's namespace
# packages will mask all of the egg-based packages in the same namespace
# package.
try:
import pkg_resources
except ImportError:
pass
execfile('/usr/lib/python/site.py')
>>> res = system(join(interpreter_dir, 'bin', 'py') +
... ' -c "import sys, pprint; pprint.pprint(sys.path)"')
>>> print res # doctest: +ELLIPSIS
['',
'/interpreter/eggs/demo-0.3-py2.4.egg',
'/interpreter/eggs/demoneeded-1.1-py2.4.egg',
'/interpreter/parts/interpreter',
...]
<BLANKLINE>
The clean_paths gathered earlier is a subset of this full list of paths.
>>> full_paths = eval(res.strip())
>>> len(clean_paths) < len(full_paths)
True
>>> set(os.path.normpath(p) for p in clean_paths).difference(
... os.path.normpath(p) for p in full_paths)
set([])
The ``import_sitecustomize`` argument does the same thing for the
sitecustomize module.
Handling custom build options for extensions provided in source distributions Handling custom build options for extensions provided in source distributions
----------------------------------------------------------------------------- -----------------------------------------------------------------------------
......
...@@ -2933,6 +2933,8 @@ def test_suite(): ...@@ -2933,6 +2933,8 @@ def test_suite():
(re.compile('extdemo[.]pyd'), 'extdemo.so'), (re.compile('extdemo[.]pyd'), 'extdemo.so'),
(re.compile('[-d] setuptools-\S+[.]egg'), 'setuptools.egg'), (re.compile('[-d] setuptools-\S+[.]egg'), 'setuptools.egg'),
(re.compile(r'\\[\\]?'), '/'), (re.compile(r'\\[\\]?'), '/'),
(re.compile('''execfile\(['"].+site.py['"]\)'''),
"execfile('/usr/lib/python/site.py')"),
]+(sys.version_info < (2, 5) and [ ]+(sys.version_info < (2, 5) and [
(re.compile('.*No module named runpy.*', re.S), ''), (re.compile('.*No module named runpy.*', re.S), ''),
(re.compile('.*usage: pdb.py scriptfile .*', re.S), ''), (re.compile('.*usage: pdb.py scriptfile .*', re.S), ''),
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment