Commit 2530eeb2 authored by Jim Fulton's avatar Jim Fulton

Major refactoring. The original motivation was to get the newest

distributions available. This required working around issues with
easy_install's --upgrade option:

- Upgrade is not recursive. Upgrading a distro doesn't update it's
  dependencies.

- Upgrade doesn't try very hard to avoid searching.  If we require a
  specific version of a distribution, and we already have that
  distribution, there's no point in looking for a newer one.

- easy_install has kind of odd rules for deciding when to look at an
  index.   Now that we use upgrade all the time, easy_install always
  wants to look at an index.

- We get warnings when connecting to index servers, like PyPI that 
  return text/plain not found messages.

We now have much greater control over how dependencies are
managed. We've essentially taken this over from easy_install.

Because we now always talk to an index server and because we want to
control anything we do in a test, many of the tests actually run their
own web servers.

Anyway:

- Now handle upgrades correctly, I think.

- The egg recipe can now install multiple distributions.

- We have the beginnings of offline mode.

- The internal architeture is much cleaner.

- We've merged the easy_install and egglinker modules, tossing
  some superfluois apis in the egglinker module.
parent e74cdf1a
......@@ -25,42 +25,17 @@ for d in 'eggs', 'develop-eggs', 'bin':
ez = {}
exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
).read() in ez
ez['use_setuptools'](to_dir='eggs', download_delay=0)
import setuptools.command.easy_install
import pkg_resources
import setuptools.package_index
import distutils.dist
os.spawnle(os.P_WAIT, sys.executable, sys.executable, 'setup.py',
'-q', 'develop', '-m', '-x', '-d', 'develop-eggs',
{'PYTHONPATH': os.path.dirname(pkg_resources.__file__)},
)
pkg_resources.working_set.add_entry('src')
## easy = setuptools.command.easy_install.easy_install(
## distutils.dist.Distribution(),
## multi_version=True,
## exclude_scripts=True,
## sitepy_installed=True,
## install_dir='eggs',
## outputs=[],
## quiet=True,
## zip_ok=True,
## args=['zc.buildout'],
## )
## easy.finalize_options()
## easy.easy_install('zc.buildout')
env = pkg_resources.Environment(['develop-eggs', 'eggs'])
ws = pkg_resources.WorkingSet()
sys.path[0:0] = [
d.location
for d in ws.resolve([pkg_resources.Requirement.parse('zc.buildout')], env)
]
import zc.buildout.egglinker
zc.buildout.egglinker.scripts(['zc.buildout'], 'bin', ['eggs'])
import zc.buildout.easy_install
zc.buildout.easy_install.scripts(
['zc.buildout'], pkg_resources.working_set , sys.executable, 'bin')
sys.exit(os.spawnl(os.P_WAIT, 'bin/buildout', 'bin/buildout'))
......@@ -2,10 +2,12 @@
develop = eggrecipe testrunnerrecipe
parts = test
# prevent slow access to cheeseshop:
index = http://download.zope.org
[test]
recipe = zc.recipe.testrunner
distributions =
zc.buildout
zc.recipe.egg
zc.recipe.testrunner
......@@ -7,7 +7,7 @@ setup(
include_package_data = True,
package_dir = {'':'src'},
namespace_packages = ['zc', 'zc.recipe'],
install_requires = ['zc.buildout'],
install_requires = ['zc.buildout', 'setuptools'],
tests_require = ['zope.testing'],
test_suite = 'zc.recipe.eggs.tests.test_suite',
author = "Jim Fulton",
......
......@@ -11,9 +11,23 @@ distribution
If not specified, the distribution defaults to the part name.
Multiple requirements can be given, separated by newlines. Each
requirement has to be on a separate line.
find-links
A list of URLs, files, or directories to search for distributions.
index
The URL of an index server, or almost any other valid URL. :)
If not specified, the Python Package Index,
http://cheeseshop.python.org/pypi, is used. You can specify an
alternate index with this option. If you use the links option and
if the links point to the needed distributions, then the index can
be anything and will be largely ignored. In the examples, here,
we'll just point to an empty directory on our link server. This
will make our examples run a little bit faster.
python
The name of a section to get the Python executable from.
If not specified, then the buildout python option is used. The
......@@ -26,13 +40,20 @@ unzip
only effective when an egg is installed. If a zipped egg already
exists in the eggs directory, it will not be unzipped.
To illustrate this, we've created a directory with some sample eggs:
>>> ls(sample_eggs)
- demo-0.1-py2.3.egg
- demo-0.2-py2.3.egg
- demo-0.3-py2.3.egg
- demoneeded-1.0-py2.3.egg
We have a link server that has a number of eggs:
>>> print get(link_server),
<html><body>
<a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br>
<a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br>
<a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br>
<a href="demoneeded-1.0-py2.3.egg">demoneeded-1.0-py2.3.egg</a><br>
<a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br>
<a href="index/">index/</a><br>
<a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
</body></html>
We have a sample buildout. Let's update it's configuration file to
install the demo package.
......@@ -44,9 +65,10 @@ install the demo package.
...
... [demo]
... recipe = zc.recipe.egg
... distribution = demo <0.3
... find-links = %s
... """ % sample_eggs)
... distribution = demo<0.3
... find-links = %(server)s
... index = %(server)s/index
... """ % dict(server=link_server))
In this example, we limited ourself to revisions before 0.3. We also
specified where to find distributions using the find-links option.
......@@ -55,14 +77,14 @@ Let's run the buildout:
>>> import os
>>> os.chdir(sample_buildout)
>>> runscript = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print system(runscript),
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print system(buildout),
Now, if we look at the buildout eggs directory:
>>> ls(sample_buildout, 'eggs')
- demo-0.2-py2.3.egg
- demoneeded-1.0-py2.3.egg
- demoneeded-1.1-py2.3.egg
We see that we got an egg for demo that met the requirement, as well
as the egg for demoneeded, wich demo requires. (We also see an egg
......@@ -114,21 +136,22 @@ specification. For example, We remove the restriction on demo:
...
... [demo]
... recipe = zc.recipe.egg
... find-links = %s
... find-links = %(server)s
... index = %(server)s/index
... unzip = true
... """ % sample_eggs)
... """ % dict(server=link_server))
We also used the unzip uption to request a directory, rather than
a zip file.
>>> print system(runscript),
>>> print system(buildout),
Then we'll get a new demo egg:
>>> ls(sample_buildout, 'eggs')
- demo-0.2-py2.3.egg
d demo-0.3-py2.3.egg
- demoneeded-1.0-py2.3.egg
d demoneeded-1.0-py2.3.egg
Note that we removed the distribution option, and the distribution
defaulted to the part name.
......@@ -150,12 +173,13 @@ arguments:
...
... [demo]
... recipe = zc.recipe.egg
... find-links = %s
... find-links = %(server)s
... index = %(server)s/index
... scripts =
... """ % sample_eggs)
... """ % dict(server=link_server))
>>> print system(runscript),
>>> print system(buildout),
>>> ls(sample_buildout, 'bin')
- buildout
......@@ -169,11 +193,12 @@ You can also control the name used for scripts:
...
... [demo]
... recipe = zc.recipe.egg
... find-links = %s
... find-links = %(server)s
... index = %(server)s/index
... scripts = demo=foo
... """ % sample_eggs)
... """ % dict(server=link_server))
>>> print system(runscript),
>>> print system(buildout),
>>> ls(sample_buildout, 'bin')
- buildout
......
......@@ -16,8 +16,7 @@
$Id$
"""
import os, zipfile
import zc.buildout.egglinker
import os, re, zipfile
import zc.buildout.easy_install
class Egg:
......@@ -29,14 +28,17 @@ class Egg:
links = options.get('find-links',
buildout['buildout'].get('find-links'))
if links:
buildout_directory = buildout['buildout']['directory']
links = [os.path.join(buildout_directory, link)
for link in links.split()]
links = links.split()
options['find-links'] = '\n'.join(links)
else:
links = ()
self.links = links
index = options.get('index', buildout['buildout'].get('index'))
if index is not None:
options['index'] = index
self.index = index
options['_b'] = buildout['buildout']['bin-directory']
options['_e'] = buildout['buildout']['eggs-directory']
options['_d'] = buildout['buildout']['develop-eggs-directory']
......@@ -48,13 +50,20 @@ class Egg:
def install(self):
options = self.options
distribution = options.get('distribution', self.name)
distributions = [
r.strip()
for r in options.get('distribution', self.name).split('\n')
if r.strip()]
zc.buildout.easy_install.install(
distribution, options['_e'], self.links, options['executable'],
always_unzip=options.get('unzip') == 'true')
ws = zc.buildout.easy_install.install(
distributions, options['_e'],
links = self.links,
index = self.index,
executable = options['executable'],
always_unzip=options.get('unzip') == 'true',
path=[options['_d']]
)
eggss = [options['_d'], options['_e']]
scripts = options.get('scripts')
if scripts or scripts is None:
if scripts is not None:
......@@ -63,7 +72,7 @@ class Egg:
('=' in s) and s.split('=', 1) or (s, s)
for s in scripts
])
return zc.buildout.egglinker.scripts(
[distribution], options['_b'], eggss,
scripts=scripts, executable=options['executable'])
return zc.buildout.easy_install.scripts(
distributions, ws, options['executable'],
options['_b'], scripts=scripts)
......@@ -9,17 +9,24 @@ We can specify the python to use by specifying the name of a section
to read the Python executable from. The default is the section
defined by the python buildout option.
We have a directory with some sample eggs:
>>> ls(sample_eggs)
- demo-0.1-py2.3.egg
- demo-0.1-py2.4.egg
- demo-0.2-py2.3.egg
- demo-0.2-py2.4.egg
- demo-0.3-py2.3.egg
- demo-0.3-py2.4.egg
- demoneeded-1.0-py2.3.egg
- demoneeded-1.0-py2.4.egg
We have a link server:
>>> print get(link_server),
<html><body>
<a href="demo-0.1-py2.3.egg">demo-0.1-py2.3.egg</a><br>
<a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
<a href="demo-0.2-py2.3.egg">demo-0.2-py2.3.egg</a><br>
<a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
<a href="demo-0.3-py2.3.egg">demo-0.3-py2.3.egg</a><br>
<a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
<a href="demoneeded-1.0-py2.3.egg">demoneeded-1.0-py2.3.egg</a><br>
<a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br>
<a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br>
<a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br>
<a href="index/">index/</a><br>
<a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
<a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
</body></html>
We have a sample buildout. Let's update it's configuration file to
install the demo package using Python 2.3.
......@@ -33,9 +40,10 @@ install the demo package using Python 2.3.
... [demo]
... recipe = zc.recipe.egg
... distribution = demo <0.3
... find-links = %s
... find-links = %(server)s
... index = %(server)s/index
... python = python2.3
... """ % sample_eggs)
... """ % dict(server=link_server))
In our default.cfg file in the .buildout subdirectiry of our
directory, we have something like::
......@@ -59,7 +67,7 @@ we'll get the Python 2.3 eggs for demo and demoneeded:
>>> ls(sample_buildout, 'eggs')
- demo-0.2-py2.3.egg
- demoneeded-1.0-py2.3.egg
- demoneeded-1.1-py2.3.egg
And the generated scripts invoke Python 2.3:
......@@ -71,7 +79,7 @@ And the generated scripts invoke Python 2.3:
import sys
sys.path[0:0] = [
'/private/tmp/tmpOEtRO8sample-buildout/eggs/demo-0.2-py2.3.egg',
'/private/tmp/tmpOEtRO8sample-buildout/eggs/demoneeded-1.0-py2.3.egg'
'/private/tmp/tmpOEtRO8sample-buildout/eggs/demoneeded-1.1-py2.3.egg'
]
<BLANKLINE>
import eggrecipedemo
......@@ -87,7 +95,7 @@ And the generated scripts invoke Python 2.3:
import sys
sys.path[0:0] = [
'/tmp/tmpOBTxDMsample-buildout/eggs/demo-0.2-py2.3.egg',
'/tmp/tmpOBTxDMsample-buildout/eggs/demoneeded-1.0-py2.3.egg'
'/tmp/tmpOBTxDMsample-buildout/eggs/demoneeded-1.1-py2.3.egg'
]
If we change the Python version to 2.4, we'll use Python 2.4 eggs:
......@@ -101,17 +109,18 @@ If we change the Python version to 2.4, we'll use Python 2.4 eggs:
... [demo]
... recipe = zc.recipe.egg
... distribution = demo <0.3
... find-links = %s
... find-links = %(server)s
... index = %(server)s/index
... python = python2.4
... """ % sample_eggs)
... """ % dict(server=link_server))
>>> print system(buildout),
>>> ls(sample_buildout, 'eggs')
- demo-0.2-py2.3.egg
- demo-0.2-py2.4.egg
- demoneeded-1.0-py2.3.egg
- demoneeded-1.0-py2.4.egg
- demoneeded-1.1-py2.3.egg
- demoneeded-1.1-py2.4.egg
>>> f = open(os.path.join(sample_buildout, 'bin', 'demo'))
>>> f.readline().strip() == '#!' + python2_4_executable
......@@ -121,7 +130,7 @@ If we change the Python version to 2.4, we'll use Python 2.4 eggs:
import sys
sys.path[0:0] = [
'/private/tmp/tmpOEtRO8sample-buildout/eggs/demo-0.2-py2.4.egg',
'/private/tmp/tmpOEtRO8sample-buildout/eggs/demoneeded-1.0-py2.4.egg'
'/private/tmp/tmpOEtRO8sample-buildout/eggs/demoneeded-1.1-py2.4.egg'
]
<BLANKLINE>
import eggrecipedemo
......@@ -137,7 +146,7 @@ If we change the Python version to 2.4, we'll use Python 2.4 eggs:
import sys
sys.path[0:0] = [
'/tmp/tmpOBTxDMsample-buildout/eggs/demo-0.2-py2.4.egg',
'/tmp/tmpOBTxDMsample-buildout/eggs/demoneeded-1.0-py2.4.egg'
'/tmp/tmpOBTxDMsample-buildout/eggs/demoneeded-1.1-py2.4.egg'
]
......@@ -29,10 +29,16 @@ def setUp(test):
'develop-eggs', 'zc.recipe.egg.egg-link'),
'w').write(dirname(__file__, 4))
zc.buildout.testing.create_sample_eggs(test)
test.globs['link_server'] = (
'http://localhost:%s/'
% zc.buildout.testing.start_server(zc.buildout.testing.make_tree(test))
)
def tearDown(test):
shutil.rmtree(test.globs['_sample_eggs_container'])
zc.buildout.testing.buildoutTearDown(test)
zc.buildout.testing.stop_server(test.globs['link_server'])
def setUpPython(test):
zc.buildout.testing.buildoutSetUp(test, clear_home=False)
......@@ -42,6 +48,10 @@ def setUpPython(test):
'w').write(dirname(__file__, 4))
zc.buildout.testing.multi_python(test)
test.globs['link_server'] = (
'http://localhost:%s/'
% zc.buildout.testing.start_server(zc.buildout.testing.make_tree(test))
)
def test_suite():
return unittest.TestSuite((
......@@ -54,7 +64,8 @@ def test_suite():
'(\\w+-)[^ \t\n%(sep)s/]+.egg'
% dict(sep=os.path.sep)
),
'\\2-VVV-egg')
'\\2-VVV-egg'),
(re.compile('-py\d[.]\d.egg'), '-py2.4.egg'),
])
),
doctest.DocFileSuite(
......
......@@ -28,7 +28,6 @@ import ConfigParser
import zc.buildout.easy_install
import pkg_resources
import zc.buildout.easy_install
import zc.buildout.egglinker
class MissingOption(KeyError):
"""A required option was missing
......@@ -262,32 +261,50 @@ class Buildout(dict):
os.chdir(os.path.dirname(setup))
os.spawnle(
os.P_WAIT, sys.executable, sys.executable,
setup, '-q', 'develop', '-m', '-x',
setup, '-q', 'develop', '-m', '-x', '-N',
'-f', ' '.join(self._links),
'-d', self['buildout']['develop-eggs-directory'],
{'PYTHONPATH':
os.path.dirname(pkg_resources.__file__)},
)
finally:
os.chdir(os.path.dirname(here))
os.chdir(here)
def _load_recipes(self, parts):
recipes = {}
if not parts:
return recipes
recipes_requirements = []
pkg_resources.working_set.add_entry(
self['buildout']['develop-eggs-directory'])
pkg_resources.working_set.add_entry(self['buildout']['eggs-directory'])
# Install the recipe distros
# Gather requirements
for part in parts:
options = self.get(part)
if options is None:
options = self[part] = {}
recipe, entry = self._recipe(part, options)
zc.buildout.easy_install.install(
recipe, self['buildout']['eggs-directory'], self._links)
recipes_requirements.append(recipe)
# Install the recipe distros
offline = self['buildout'].get('offline', 'false')
if offline not in ('true', 'false'):
self._error('Invalif value for offline option: %s', offline)
if offline == 'true':
ws = zc.buildout.easy_install.working_set(
recipes_requirements, sys.executable,
[self['buildout']['eggs-directory'],
self['buildout']['develop-eggs-directory'],
],
)
else:
ws = zc.buildout.easy_install.install(
recipes_requirements, self['buildout']['eggs-directory'],
links=self._links, index=self['buildout'].get('index'),
path=[self['buildout']['develop-eggs-directory']])
# Add the distros to the working set
pkg_resources.require(recipes_requirements)
......@@ -503,6 +520,10 @@ def main(args=None):
else:
verbosity -= 10
op = op[1:]
if op == 'd':
op = op[1:]
import pdb; pdb.set_trace()
if op[:1] == 'c':
op = op[1:]
if op:
......
......@@ -20,18 +20,299 @@ installed.
$Id$
"""
import os, sys
import logging, os, re, sys
import pkg_resources
import zc.buildout
def install(spec, dest, links, executable=sys.executable, always_unzip=False):
logger = logging.getLogger('zc.buildout.easy_install')
# Include buildout and setuptools eggs in paths
buildout_and_setuptools_path = [
(('.egg' in m.__file__)
and m.__file__[:m.__file__.rfind('.egg')+4]
or os.path.dirname(m.__file__)
)
for m in (pkg_resources,)
]
buildout_and_setuptools_path += [
(('.egg' in m.__file__)
and m.__file__[:m.__file__.rfind('.egg')+4]
or os.path.dirname(os.path.dirname(os.path.dirname(m.__file__)))
)
for m in (zc.buildout,)
]
_versions = {sys.executable: '%d.%d' % sys.version_info[:2]}
def _get_version(executable):
try:
return _versions[executable]
except KeyError:
i, o = os.popen4(executable + ' -V')
i.close()
version = o.read().strip()
o.close()
pystring, version = version.split()
assert pystring == 'Python'
version = re.match('(\d[.]\d)[.]\d$', version).group(1)
_versions[executable] = version
return version
def _satisfied(req, env):
dists = env[req.project_name]
best = None
for dist in dists:
if (dist.precedence == pkg_resources.DEVELOP_DIST) and (dist in req):
if best is not None and best.location != dist.location:
raise ValueError('Multiple devel eggs for', req)
best = dist
if best is not None:
return best
specs = [(pkg_resources.parse_version(v), op) for (op, v) in req.specs]
specs.sort()
maxv = None
greater = False
lastv = None
for v, op in specs:
if op == '==' and not greater:
maxv = v
elif op in ('>', '>=', '!='):
maxv = None
greater == True
elif op == '<':
maxv = None
greater == False
elif op == '<=':
maxv = v
greater == False
if v == lastv:
# Repeated versions values are undefined, so
# all bets are off
maxv = None
greater = True
else:
lastv = v
if maxv is not None:
for dist in dists:
if dist.parsed_version == maxv:
return dist
return None
def _call_easy_install(spec, dest, links=(),
index = None,
executable=sys.executable,
always_unzip=False,
):
prefix = sys.exec_prefix + os.path.sep
path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)])
args = (
'-c', 'from setuptools.command.easy_install import main; main()',
'-mqxd', dest)
'-mUNxd', dest)
if links:
args += ('-f', ' '.join(links))
if index:
args += ('-i', index)
if always_unzip:
args += ('-Z', )
args += (spec, dict(PYTHONPATH=path))
level = logger.getEffectiveLevel()
if level > logging.DEBUG:
args += ('-q', )
elif level < logging.DEBUG:
args += ('-v', )
args += (spec, )
if level <= logging.DEBUG:
logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
executable, '" "'.join(args), path)
os.spawnle(os.P_WAIT, executable, executable, *args)
args += (dict(PYTHONPATH=path), )
sys.stdout.flush() # We want any pending output first
exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
# We may overwrite distributions, so clear importer
# cache.
sys.path_importer_cache.clear()
assert exit_code == 0
def _get_dist(requirement, env, ws,
dest, links, index, executable, always_unzip):
# Maybe an existing dist is already the best dist that satisfies the
# requirement
dist = _satisfied(requirement, env)
# XXX Special case setuptools because:
# 1. Almost everything depends on it and
# 2. It is expensive to checl for.
# Need to think of a cleaner way to handle this.
# If we already have a satisfactory version, use it.
if dist is None and requirement.project_name == 'setuptools':
dist = env.best_match(requirement, ws)
if dist is None:
if dest is not None:
# May need a new one. Call easy_install
_call_easy_install(str(requirement), dest, links, index,
executable, always_unzip)
# Because we may have added new eggs, we need to rescan
# the destination directory. A possible optimization
# is to get easy_install to recod the files installed
# and either firgure out the distribution added, or
# only rescan if any files have been added.
env.scan([dest])
dist = env.best_match(requirement, ws)
# XXX Need test for this
if dist.has_metadata('dependency_links.txt'):
for link in dist.get_metadata_lines('dependency_links.txt'):
link = link.strip()
if link not in links:
links.append(link)
return dist
def install(specs, dest,
links=(), index=None,
executable=sys.executable, always_unzip=False,
path=None):
logger.debug('Installing %r', specs)
path = path and path[:] or []
if dest is not None:
path.insert(0, dest)
path += buildout_and_setuptools_path
links = list(links) # make copy, because we may need to mutate
# For each spec, see if it is already installed. We create a working
# set to keep track of what we've collected and to make sue than the
# distributions assembled are consistent.
env = pkg_resources.Environment(path, python=_get_version(executable))
requirements = [pkg_resources.Requirement.parse(spec) for spec in specs]
ws = pkg_resources.WorkingSet([])
for requirement in requirements:
ws.add(_get_dist(requirement, env, ws,
dest, links, index, executable, always_unzip)
)
# OK, we have the requested distributions and they're in the working
# set, but they may have unmet requirements. We'll simply keep
# trying to resolve requirements, adding missing requirements as they
# are reported.
#
# Note that we don't pass in the environment, because we
# want to look for new eggs unless what we have is the best that matches
# the requirement.
while 1:
try:
ws.resolve(requirements)
except pkg_resources.DistributionNotFound, err:
[requirement] = err
if dest:
logger.debug('Getting required %s', requirement)
ws.add(_get_dist(requirement, env, ws,
dest, links, index, executable, always_unzip)
)
else:
break
return ws
def working_set(specs, executable, path):
return install(specs, None, executable=executable, path=path)
def scripts(reqs, working_set, executable, dest, scripts=None):
reqs = [pkg_resources.Requirement.parse(r) for r in reqs]
projects = [r.project_name for r in reqs]