Commit 8d04ac68 authored by jim's avatar jim

Refactored to do more work in buildout and less work in easy_install.

This makes things go a little faster, makes errors a little easier to
handle, and allows extensions (like the sftp extension) to influence
more of the process.


git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@70184 62d5b8a3-27da-0310-9561-8e5933582275
parent bdd5c8ed
...@@ -20,8 +20,12 @@ installed. ...@@ -20,8 +20,12 @@ installed.
$Id$ $Id$
""" """
import logging, os, re, tempfile, sys import glob, logging, os, re, tempfile, shutil, sys, zipimport
import pkg_resources, setuptools.command.setopt, setuptools.package_index import distutils.errors
import pkg_resources
import setuptools.command.setopt
import setuptools.package_index
import setuptools.archive_util
import zc.buildout import zc.buildout
default_index_url = os.environ.get('buildout-testing-index-url') default_index_url = os.environ.get('buildout-testing-index-url')
...@@ -171,8 +175,7 @@ _easy_install_cmd = _safe_arg( ...@@ -171,8 +175,7 @@ _easy_install_cmd = _safe_arg(
'from setuptools.command.easy_install import main; main()' 'from setuptools.command.easy_install import main; main()'
) )
def _call_easy_install(spec, dest, links=(), def _call_easy_install(spec, dest,
index=None,
executable=sys.executable, executable=sys.executable,
always_unzip=False, always_unzip=False,
): ):
...@@ -180,10 +183,6 @@ def _call_easy_install(spec, dest, links=(), ...@@ -180,10 +183,6 @@ def _call_easy_install(spec, dest, links=(),
path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)]) path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)])
args = ('-c', _easy_install_cmd, '-mUNxd', _safe_arg(dest)) args = ('-c', _easy_install_cmd, '-mUNxd', _safe_arg(dest))
if links:
args += ('-f', _safe_arg(' '.join(links)))
if index:
args += ('-i', index)
if always_unzip: if always_unzip:
args += ('-Z', ) args += ('-Z', )
level = logger.getEffectiveLevel() level = logger.getEffectiveLevel()
...@@ -220,15 +219,63 @@ def _get_dist(requirement, env, ws, ...@@ -220,15 +219,63 @@ def _get_dist(requirement, env, ws,
if dest is not None: if dest is not None:
logger.info("Getting new distribution for %s", requirement) logger.info("Getting new distribution for %s", requirement)
# Retrieve the dist:
index = _get_index(executable, index, links)
dist = index.obtain(requirement)
if dist is None:
raise zc.buildout.UserError(
"Couln't find a distribution for %s."
% requirement)
if dist.location.endswith('.egg'):
# It's already an egg, just fetch it into the dest
tmp = tempfile.mkdtemp('get_dist')
try:
dist = index.fetch_distribution(requirement, tmp)
if dist is None:
raise zc.buildout.UserError(
"Couln't download a distribution for %s."
% requirement)
metadata = pkg_resources.EggMetadata(
zipimport.zipimporter(dist.location)
)
if (always_unzip
or
metadata.has_metadata('not-zip-safe')
or
not metadata.has_metadata('zip-safe')
):
setuptools.archive_util.unpack_archive(
dist.location,
os.path.join(dest, os.path.basename(dist.location)
),
)
else:
shutil.move(
dist.location,
os.path.join(dest, os.path.basename(dist.location)
),
)
finally:
shutil.rmtree(tmp)
else:
# It's some other kind of dist. We'll download it to
# a temporary directory and let easy_install have it's
# way with it:
tmp = tempfile.mkdtemp('get_dist')
try:
dist = index.fetch_distribution(requirement, tmp)
# May need a new one. Call easy_install # May need a new one. Call easy_install
_call_easy_install(str(requirement), dest, links, index, _call_easy_install(dist.location, dest,
executable, always_unzip) executable, always_unzip)
finally:
shutil.rmtree(tmp)
# Because we may have added new eggs, we need to rescan # Because we have added a new egg, we need to rescan
# the destination directory. A possible optimization # the destination directory.
# is to get easy_install to recod the files installed
# and either firgure out the distribution added, or
# only rescan if any files have been added.
env.scan([dest]) env.scan([dest])
dist = env.best_match(requirement, ws) dist = env.best_match(requirement, ws)
logger.info("Got %s", dist) logger.info("Got %s", dist)
...@@ -302,32 +349,6 @@ def install(specs, dest, ...@@ -302,32 +349,6 @@ def install(specs, dest,
return ws return ws
def _editable(spec, dest, links=(), index = None, executable=sys.executable):
prefix = sys.exec_prefix + os.path.sep
path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)])
args = ('-c', _easy_install_cmd, '-eb', _safe_arg(dest))
if links:
args += ('-f', ' '.join(links))
if index:
args += ('-i', index)
level = logger.getEffectiveLevel()
if level > logging.DEBUG:
args += ('-q', )
elif level < logging.DEBUG:
args += ('-v', )
args += (spec, )
if level <= logging.DEBUG:
logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
executable, '" "'.join(args), path)
args += (dict(os.environ, PYTHONPATH=path), )
sys.stdout.flush() # We want any pending output first
exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
assert exit_code == 0
def build(spec, dest, build_ext, def build(spec, dest, build_ext,
links=(), index=None, links=(), index=None,
executable=sys.executable, executable=sys.executable,
...@@ -355,18 +376,45 @@ def build(spec, dest, build_ext, ...@@ -355,18 +376,45 @@ def build(spec, dest, build_ext,
# Get an editable version of the package to a temporary directory: # Get an editable version of the package to a temporary directory:
tmp = tempfile.mkdtemp('editable') tmp = tempfile.mkdtemp('editable')
_editable(spec, tmp, links, index, executable) tmp2 = tempfile.mkdtemp('editable')
try:
index = _get_index(executable, index, links)
dist = index.fetch_distribution(requirement, tmp2, False, True)
if dist is None:
raise zc.buildout.UserError(
"Couln't find a source distribution for %s."
% requirement)
setuptools.archive_util.unpack_archive(dist.location, tmp)
if os.path.exists(os.path.join(tmp, 'setup.py')):
base = tmp
else:
setups = glob.glob(os.path.join(tmp, '*', 'setup.py'))
if not setups:
raise distutils.errors.DistutilsError(
"Couldn't find a setup script in %s"
% os.path.basename(dist.location)
)
if len(setups) > 1:
raise distutils.errors.DistutilsError(
"Multiple setup scripts in %s"
% os.path.basename(dist.location)
)
base = os.path.dirname(setups[0])
setup_cfg = os.path.join(tmp, requirement.key, 'setup.cfg') setup_cfg = os.path.join(base, 'setup.cfg')
if not os.path.exists(setup_cfg): if not os.path.exists(setup_cfg):
f = open(setup_cfg, 'w') f = open(setup_cfg, 'w')
f.close() f.close()
setuptools.command.setopt.edit_config(setup_cfg, dict(build_ext=build_ext)) setuptools.command.setopt.edit_config(
setup_cfg, dict(build_ext=build_ext))
# Now run easy_install for real: # Now run easy_install for real:
_call_easy_install( _call_easy_install(base, dest, executable, True)
os.path.join(tmp, requirement.key), finally:
dest, links, index, executable, True) shutil.rmtree(tmp)
shutil.rmtree(tmp2)
def working_set(specs, executable, path): def working_set(specs, executable, path):
return install(specs, None, executable=executable, path=path) return install(specs, None, executable=executable, path=path)
......
...@@ -75,10 +75,8 @@ We have a link server that has a number of eggs: ...@@ -75,10 +75,8 @@ We have a link server that has a number of eggs:
<a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br> <a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
<a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br> <a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br>
<a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br> <a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
<a href="demoneeded-1.0-py2.3.egg">demoneeded-1.0-py2.3.egg</a><br> <a href="demoneeded-1.0.tar.gz">demoneeded-1.0.tar.gz</a><br>
<a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br> <a href="demoneeded-1.1.tar.gz">demoneeded-1.1.tar.gz</a><br>
<a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br>
<a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br>
<a href="extdemo-1.4.tar.gz">extdemo-1.4.tar.gz</a><br> <a href="extdemo-1.4.tar.gz">extdemo-1.4.tar.gz</a><br>
<a href="index/">index/</a><br> <a href="index/">index/</a><br>
<a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br> <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
......
...@@ -125,15 +125,16 @@ from pkg_resources import load_entry_point ...@@ -125,15 +125,16 @@ from pkg_resources import load_entry_point
sys.exit(load_entry_point('zc.buildout', 'console_scripts', 'buildout')()) sys.exit(load_entry_point('zc.buildout', 'console_scripts', 'buildout')())
''' '''
def runsetup(d, executable): def runsetup(d, executable, type='bdist_egg'):
here = os.getcwd() here = os.getcwd()
try: try:
os.chdir(d) os.chdir(d)
os.spawnle( os.spawnle(
os.P_WAIT, executable, executable, os.P_WAIT, executable, executable,
'setup.py', '-q', 'bdist_egg', 'setup.py', '-q', type,
{'PYTHONPATH': os.path.dirname(pkg_resources.__file__)}, {'PYTHONPATH': os.path.dirname(pkg_resources.__file__)},
) )
if os.path.exists('build'):
shutil.rmtree('build') shutil.rmtree('build')
finally: finally:
os.chdir(here) os.chdir(here)
...@@ -152,10 +153,11 @@ def create_sample_eggs(test, executable=sys.executable): ...@@ -152,10 +153,11 @@ def create_sample_eggs(test, executable=sys.executable):
sample, 'setup.py', sample, 'setup.py',
"from setuptools import setup\n" "from setuptools import setup\n"
"setup(name='demoneeded', py_modules=['eggrecipedemobeeded']," "setup(name='demoneeded', py_modules=['eggrecipedemobeeded'],"
" zip_safe=True, version='1.%s')\n" " zip_safe=True, version='1.%s', author='bob', url='bob', "
"author_email='bob')\n"
% i % i
) )
runsetup(sample, executable) runsetup(sample, executable, 'sdist')
write( write(
sample, 'setup.py', sample, 'setup.py',
......
...@@ -58,8 +58,8 @@ We have a link server that has a number of eggs: ...@@ -58,8 +58,8 @@ We have a link server that has a number of eggs:
<a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br> <a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br>
<a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br> <a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br>
<a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br> <a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br>
<a href="demoneeded-1.0-py2.3.egg">demoneeded-1.0-py2.3.egg</a><br> <a href="demoneeded-1.0.tar.gz">demoneeded-1.0.tar.gz</a><br>
<a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br> <a href="demoneeded-1.1.tar.gz">demoneeded-1.1.tar.gz</a><br>
<a href="index/">index/</a><br> <a href="index/">index/</a><br>
<a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br> <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
</body></html> </body></html>
......
...@@ -19,10 +19,8 @@ We have a link server: ...@@ -19,10 +19,8 @@ We have a link server:
<a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br> <a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
<a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br> <a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br>
<a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br> <a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
<a href="demoneeded-1.0-py2.3.egg">demoneeded-1.0-py2.3.egg</a><br> <a href="demoneeded-1.0.tar.gz">demoneeded-1.0.tar.gz</a><br>
<a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br> <a href="demoneeded-1.1.tar.gz">demoneeded-1.1.tar.gz</a><br>
<a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br>
<a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br>
<a href="index/">index/</a><br> <a href="index/">index/</a><br>
<a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br> <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
<a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br> <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment