Skip to content

S
slapos.buildout
Commit 1fae78f7 authored by Gary Poster's avatar Gary Poster
Browse files
Options

simplify resulting code a bit more and try again to remove warnings

parent 7e825d61
Hide whitespace changes
Inline Side-by-side
Showing with 191 additions and 140 deletions
src/zc/buildout/easy_install.py
View file @ 1fae78f7
......@@ -1063,7 +1063,8 @@ def _generate_scripts(reqs, working_set, dest, path, scripts, relative_paths,
return generated
def _relative_path_and_setup(sname, path,
relative_paths=False, indent_level=1):
relative_paths=False, indent_level=1,
omit_os_import=False):
"""Return a string of code of paths and of setup if appropriate.
- sname is the full path to the script name to be created.
......@@ -1080,8 +1081,10 @@ def _relative_path_and_setup(sname, path,
[_relativitize(os.path.normcase(path_item), sname, relative_paths)
for path_item in path], indent_level=indent_level)
rpsetup = relative_paths_setup
if not omit_os_import:
rpsetup = '\n\nimport os\n' + rpsetup
for i in range(_relative_depth(relative_paths, sname)):
rpsetup += "base = os.path.dirname(base)\n"
rpsetup += "\nbase = os.path.dirname(base)"
else:
spath = _format_paths((repr(p) for p in path),
indent_level=indent_level)
......@@ -1143,11 +1146,8 @@ def _relativitize(path, script, relative_paths):
return repr(path)
relative_paths_setup = """
import os
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
"""
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))"""
def _write_script(full_name, contents, logged_type):
"""Write contents of script in full_name, logging the action.
......@@ -1214,8 +1214,8 @@ sys.path[0:0] = [
'''
script_template = script_header + '''\
%(relative_paths_setup)s
import sys
sys.path[0:0] = [
%(path)s,
......@@ -1240,8 +1240,8 @@ def _pyscript(path, dest, executable, rsetup):
return _write_script(dest, contents, 'interpreter')
py_script_template = script_header + '''\
%(relative_paths_setup)s
import sys
sys.path[0:0] = [
......@@ -1384,29 +1384,32 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
"""
path = _get_path(working_set, extra_paths)
site_path = os.path.join(dest, 'site.py')
path_string, rpsetup = _relative_path_and_setup(
site_path, path, relative_paths, indent_level=2)
if rpsetup:
rpsetup = '\n'.join(
egg_path_string, preamble = _relative_path_and_setup(
site_path, path, relative_paths, indent_level=2, omit_os_import=True)
if preamble:
preamble = '\n'.join(
[(line and ' %s' % (line,) or line)
for line in rpsetup.split('\n')])
namespace_setup = ''
addsitedir = addsitedir_snippet
for line in preamble.split('\n')])
original_path_setup = ''
if add_site_packages:
stdlib, site_paths = _get_system_paths(executable)
path_string = ''.join([
path_string,
(",\n"
" # These are the underlying Python's site-packages.\n"
" "),
_format_paths((repr(p) for p in site_paths), 2)])
original_path_setup = original_path_snippet % (
_format_paths((repr(p) for p in site_paths), 2),)
distribution = working_set.find(
pkg_resources.Requirement.parse('setuptools'))
if distribution is not None:
# We need to worry about namespace packages.
namespace_setup = namespace_add_site_packages_setup % (
distribution.location,)
addsitedir = addsitedir_namespace_add_site_packages_snippet
if relative_paths:
location = _relativitize(
distribution.location,
os.path.normcase(os.path.abspath(site_path)),
relative_paths)
else:
location = repr(distribution.location)
preamble += namespace_add_site_packages_setup % (location,)
original_path_setup = (
addsitedir_namespace_originalpackages_snippet +
original_path_setup)
addsitepackages_marker = 'def addsitepackages('
enableusersite_marker = 'ENABLE_USER_SITE = '
successful_rewrite = False
......@@ -1420,7 +1423,7 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
site.write('False # buildout does not support user sites.\n')
elif line.startswith(addsitepackages_marker):
site.write(addsitepackages_script % (
namespace_setup, rpsetup, path_string, addsitedir))
preamble, egg_path_string, original_path_setup))
site.write(line[len(addsitepackages_marker):])
successful_rewrite = True
else:
......@@ -1433,49 +1436,41 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
return site_path
namespace_add_site_packages_setup = '''
setuptools_path = %r
setuptools_path = %s
sys.path.append(setuptools_path)
known_paths.add(setuptools_path)
known_paths.add(os.path.normcase(setuptools_path))
import pkg_resources'''
addsitedir_snippet = '''
for path in paths:
addsitedir(path, known_paths)'''
addsitedir_namespace_originalpackages_snippet = '''
for dist in pkg_resources.find_distributions(sitedir, True):
pkg_resources.fixup_namespace_packages(dist.location)
if dist.has_metadata('namespace_packages.txt'):
for namespace in dist.get_metadata_lines(
'namespace_packages.txt'):
pkg_resources.declare_namespace(namespace)'''
addsitedir_namespace_add_site_packages_snippet = '''
dotpth = os.extsep + "pth"
for path in paths:
# This duplicates addsitedir except for adding the pkg_resources call.
sitedir, sitedircase = makepath(path)
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.append(sitedir)
pkg_resources.working_set.add_entry(sitedir)
known_paths.add(sitedircase)
try:
names = os.listdir(sitedir)
except os.error:
continue
names = [name for name in names if name.endswith(dotpth)]
names.sort()
for name in names:
addpackage(sitedir, name, known_paths)'''
original_path_snippet = '''
original_paths = [
%s
]
for path in original_paths:
addsitedir(path, known_paths)'''
addsitepackages_script = '''\
def addsitepackages(known_paths):
"""Add site packages.
"""Add site packages, as determined by zc.buildout.
This function is written by buildout. See original_addsitepackages,
below, for the original version."""%s
%s paths = [
# Eggs.
See original_addsitepackages, below, for the original version."""%s
buildout_paths = [
%s
]%s
global addsitepackages
addsitepackages = original_addsitepackages
]
for path in buildout_paths:
sitedir, sitedircase = makepath(path)
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.append(sitedir)
known_paths.add(sitedircase)%s
return known_paths
buildout_addsitepackages = addsitepackages
def original_addsitepackages('''
def _generate_interpreter(name, dest, executable, site_py_dest,
......@@ -1483,7 +1478,9 @@ def _generate_interpreter(name, dest, executable, site_py_dest,
"""Write an interpreter script, using the site.py approach."""
full_name = os.path.join(dest, name)
site_py_dest_string, rpsetup = _relative_path_and_setup(
full_name, [site_py_dest], relative_paths)
full_name, [site_py_dest], relative_paths, omit_os_import=True)
if rpsetup:
rpsetup += "\n"
if sys.platform == 'win32':
windows_import = '\nimport subprocess'
# os.exec* is a mess on Windows, particularly if the path
......@@ -1534,12 +1531,10 @@ def _generate_interpreter(name, dest, executable, site_py_dest,
)
return _write_script(full_name, contents, 'interpreter')
interpreter_template = script_header + '''\
%(relative_paths_setup)s
interpreter_template = script_header + '''
import os
import sys%(windows_import)s
%(relative_paths_setup)s
argv = [sys.executable] + sys.argv[1:]
environ = os.environ.copy()
path = %(site_dest)s
......
src/zc/buildout/easy_install.txt
View file @ 1fae78f7
......@@ -997,8 +997,7 @@ The interpreter script is simple. It puts the directory with the
site.py and sitecustomize.py on the PYTHONPATH and (re)starts Python.
>>> cat(py_path)
#!/usr/bin/python2.4 -S
<BLANKLINE>
#!/usr/bin/python -S
import os
import sys
<BLANKLINE>
......@@ -1012,31 +1011,28 @@ site.py and sitecustomize.py on the PYTHONPATH and (re)starts Python.
The site.py file is a modified version of the underlying Python's site.py.
The most important modification is that it has a different version of the
addsitepackages function. It has all of the trickier bits, and sets up the
Python path, similarly to the behavior of the function it replaces. The
following shows the part that buildout inserts.
addsitepackages function. It sets up the Python path, similarly to the
behavior of the function it replaces. The following shows the part that
buildout inserts, in the simplest case.
>>> sys.stdout.write('#\n'); cat(site_path)
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
#...
def addsitepackages(known_paths):
"""Add site packages.
"""Add site packages, as determined by zc.buildout.
<BLANKLINE>
This function is written by buildout. See original_addsitepackages,
below, for the original version."""
paths = [
# Eggs.
See original_addsitepackages, below, for the original version."""
buildout_paths = [
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
]
for path in paths:
addsitedir(path, known_paths)
global addsitepackages
addsitepackages = original_addsitepackages
for path in buildout_paths:
sitedir, sitedircase = makepath(path)
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.append(sitedir)
known_paths.add(sitedircase)
return known_paths
<BLANKLINE>
buildout_addsitepackages = addsitepackages
<BLANKLINE>
def original_addsitepackages(known_paths):...
Here are some examples of the interpreter in use.
......@@ -1079,24 +1075,23 @@ If you provide initialization, it goes in sitecustomize.py.
bar baz bing shazam
<BLANKLINE>
If you use relative paths, this affects the interpreter and site.py.
If you use relative paths, this affects the interpreter and site.py. (This is
again the UNIX version; the Windows version uses subprocess instead of
os.execve.)
>>> reset_interpreter()
>>> generated = zc.buildout.easy_install.generate_scripts(
... interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
... interpreter='py', relative_paths=interpreter_dir)
>>> cat(py_path)
#!/usr/bin/python2.4 -S
<BLANKLINE>
#!/usr/bin/python -S
import os
import sys
<BLANKLINE>
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
<BLANKLINE>
import os
import sys
<BLANKLINE>
argv = [sys.executable] + sys.argv[1:]
environ = os.environ.copy()
path = join(base, 'parts/interpreter')
......@@ -1112,19 +1107,14 @@ paths join a base to a path, as with the use of this argument in the
>>> sys.stdout.write('#\n'); cat(site_path) # doctest: +ELLIPSIS
#...
def addsitepackages(known_paths):
"""Add site packages.
<BLANKLINE>
This function is written by buildout. See original_addsitepackages,
below, for the original version."""
<BLANKLINE>
import os
"""Add site packages, as determined by zc.buildout.
<BLANKLINE>
See original_addsitepackages, below, for the original version."""
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
base = os.path.dirname(base)
paths = [
# Eggs.
buildout_paths = [
join(base, 'eggs/demo-0.3-pyN.N.egg'),
join(base, 'eggs/demoneeded-1.1-pyN.N.egg')
]...
......@@ -1152,12 +1142,10 @@ The ``extra_paths`` argument affects the path in site.py. Notice that
>>> sys.stdout.write('#\n'); cat(site_path) # doctest: +ELLIPSIS
#...
def addsitepackages(known_paths):
"""Add site packages.
"""Add site packages, as determined by zc.buildout.
<BLANKLINE>
This function is written by buildout. See original_addsitepackages,
below, for the original version."""
paths = [
# Eggs.
See original_addsitepackages, below, for the original version."""
buildout_paths = [
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
'/interpreter/other'
......@@ -1213,19 +1201,29 @@ this is very straightforward.
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
#...
def addsitepackages(known_paths):
"""Add site packages.
"""Add site packages, as determined by zc.buildout.
<BLANKLINE>
This function is written by buildout. See original_addsitepackages,
below, for the original version."""
paths = [
# Eggs.
See original_addsitepackages, below, for the original version."""
buildout_paths = [
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
# These are the underlying Python's site-packages.
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
]
for path in buildout_paths:
sitedir, sitedircase = makepath(path)
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.append(sitedir)
known_paths.add(sitedircase)
original_paths = [
...
]...
]
for path in original_paths:
addsitedir(path, known_paths)
return known_paths
<BLANKLINE>
def original_addsitepackages(known_paths):...
It simply adds the site-packages after the eggs.
It simply adds the original paths using addsitedir after the code to add the
buildout paths.
Here's an example of the new script in use. Other documents and tests in
this package give the feature a more thorough workout, but this should
......@@ -1271,45 +1269,37 @@ call to another text fixture to create.
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
#...
def addsitepackages(known_paths):
"""Add site packages.
"""Add site packages, as determined by zc.buildout.
<BLANKLINE>
This function is written by buildout. See original_addsitepackages,
below, for the original version."""
See original_addsitepackages, below, for the original version."""
setuptools_path = '...setuptools...'
sys.path.append(setuptools_path)
known_paths.add(setuptools_path)
known_paths.add(os.path.normcase(setuptools_path))
import pkg_resources
paths = [
# Eggs.
buildout_paths = [
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/tellmy.fortune-1.0-pyN.N.egg',
'...setuptools...',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
# These are the underlying Python's site-packages.
...
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
]
dotpth = os.extsep + "pth"
for path in paths:
# This duplicates addsitedir except for adding the pkg_resources call.
for path in buildout_paths:
sitedir, sitedircase = makepath(path)
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.append(sitedir)
pkg_resources.working_set.add_entry(sitedir)
known_paths.add(sitedircase)
try:
names = os.listdir(sitedir)
except os.error:
continue
names = [name for name in names if name.endswith(dotpth)]
names.sort()
for name in names:
addpackage(sitedir, name, known_paths)
global addsitepackages
addsitepackages = original_addsitepackages
for dist in pkg_resources.find_distributions(sitedir, True):
pkg_resources.fixup_namespace_packages(dist.location)
if dist.has_metadata('namespace_packages.txt'):
for namespace in dist.get_metadata_lines(
'namespace_packages.txt'):
pkg_resources.declare_namespace(namespace)
original_paths = [
...
]
for path in original_paths:
addsitedir(path, known_paths)
return known_paths
<BLANKLINE>
buildout_addsitepackages = addsitepackages
<BLANKLINE>
def original_addsitepackages(known_paths):...
>>> print call_py(interpreter_path, "import sys; print sys.path")
......@@ -1330,6 +1320,66 @@ feature--lines that might be part of the setuptools namespace package
implementation for system packages, as mentioned above, and that must
come after processing egg namespaces.
The most complex that this function gets is if you use namespace packages,
include site-packages, and use relative paths. For completeness, we'll look
at that result.
>>> generated = zc.buildout.easy_install.generate_scripts(
... interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
... interpreter='py', add_site_packages=True,
... relative_paths=interpreter_dir)
>>> sys.stdout.write('#\n'); cat(site_path)
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
#...
def addsitepackages(known_paths):
"""Add site packages, as determined by zc.buildout.
<BLANKLINE>
See original_addsitepackages, below, for the original version."""
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
base = os.path.dirname(base)
setuptools_path = '...setuptools...'
sys.path.append(setuptools_path)
known_paths.add(os.path.normcase(setuptools_path))
import pkg_resources
buildout_paths = [
join(base, 'eggs/demo-0.3-pyN.N.egg'),
join(base, 'eggs/tellmy.fortune-1.0-pyN.N.egg'),
'...setuptools...',
join(base, 'eggs/demoneeded-1.1-pyN.N.egg')
]
for path in buildout_paths:
sitedir, sitedircase = makepath(path)
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.append(sitedir)
known_paths.add(sitedircase)
for dist in pkg_resources.find_distributions(sitedir, True):
pkg_resources.fixup_namespace_packages(dist.location)
if dist.has_metadata('namespace_packages.txt'):
for namespace in dist.get_metadata_lines(
'namespace_packages.txt'):
pkg_resources.declare_namespace(namespace)
original_paths = [
...
]
for path in original_paths:
addsitedir(path, known_paths)
return known_paths
<BLANKLINE>
def original_addsitepackages(known_paths):...
>>> print call_py(interpreter_path, "import sys; print sys.path")
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
['',
'/interpreter/parts/interpreter',
...,
'...setuptools...',
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/tellmy.fortune-1.0-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
...]
The ``exec_sitecustomize`` argument does the same thing for the
sitecustomize module--it allows you to include the code from the
sitecustomize module in the underlying Python if you set the argument to
......
src/zc/buildout/testing.py
View file @ 1fae78f7
......@@ -310,6 +310,16 @@ def buildoutSetUp(test):
old_wd = os.getcwd()
os.chdir(buildout)
make_buildout()
# Normally we don't process .pth files in extra-paths. We want to
# in this case so that we can test with setuptools system installs
# (--single-version-externally-managed), which use .pth files.
initialization = (
('import sys\n'
'import site\n'
'known_paths = set(sys.path)\n'
'site_packages_dir = %r\n'
'site.addsitedir(site_packages_dir, known_paths)\n'
) % (site_packages_dir,)) + initialization
initialization = '\n'.join(
' ' + line for line in initialization.split('\n'))
install_develop(
......
z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt
View file @ 1fae78f7
......@@ -273,11 +273,11 @@ into the sitecustomize.
Generated interpreter '/sample-buildout/bin/py'.
>>> cat(sample_buildout, 'parts', 'py', 'sitecustomize.py')
... # doctest: +NORMALIZE_WHITESPACE
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
<BLANKLINE>
# The following is from
# /executable_buildout/parts/py/sitecustomize.py
<BLANKLINE>
...
import os
os.environ['zc.buildout'] = 'foo bar baz shazam'
......
z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py
View file @ 1fae78f7
......@@ -175,14 +175,14 @@ custom Python.
Generated interpreter '/sample-buildout/bin/py'.
>>> cat(sample_buildout, 'parts', 'py', 'sitecustomize.py')
... # doctest: +NORMALIZE_WHITESPACE
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
<BLANKLINE>
import os
os.environ['zc.recipe.egg'] = 'baLOOba'
<BLANKLINE>
# The following is from
# /executable_buildout/parts/py/sitecustomize.py
<BLANKLINE>
...
import os
os.environ['zc.buildout'] = 'foo bar baz shazam'
......@@ -227,15 +227,11 @@ Let's look at the site.py that was generated:
#...
def addsitepackages(known_paths):
"..."
<BLANKLINE>
import os
<BLANKLINE>
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
base = os.path.dirname(base)
paths = [
# Eggs.
buildout_paths = [
'/foo/bar',
join(base, 'spam')
]...
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7